gridscore - mnistpct20 - kNN-Lwplsrda

using Jchemo, JchemoData
using JLD2, CairoMakie
using FreqTables

Data importation

path_jdat = dirname(dirname(pathof(JchemoData)))
db = joinpath(path_jdat, "data/mnist20pct.jld2") 
@load db dat
@names dat

(:Xtrain, :ytrain, :Xtest, :ytest)

Xtrain = dat.Xtrain
ytrain = dat.ytrain
Xtest = dat.Xtest
ytest = dat.ytest
@head Xtrain
@head Xtest
tab(ytrain)
tab(ytest)
ntrain, p = size(Xtrain)
ntest = nro(Xtest)
ntot = ntrain + ntest
(ntot = ntot, ntrain, ntest)

... (12000, 784)
 
... (2000, 784)
 
(ntot = 14000, ntrain = 12000, ntest = 2000)

3×784 DataFrame

684 columns omitted

Row	1x1	1x2	1x3	1x4	1x5	1x6	1x7	1x8	1x9	1x10	1x11	1x12	1x13	1x14	1x15	1x16	1x17	1x18	1x19	1x20	1x21	1x22	1x23	1x24	1x25	1x26	1x27	1x28	2x1	2x2	2x3	2x4	2x5	2x6	2x7	2x8	2x9	2x10	2x11	2x12	2x13	2x14	2x15	2x16	2x17	2x18	2x19	2x20	2x21	2x22	2x23	2x24	2x25	2x26	2x27	2x28	3x1	3x2	3x3	3x4	3x5	3x6	3x7	3x8	3x9	3x10	3x11	3x12	3x13	3x14	3x15	3x16	3x17	3x18	3x19	3x20	3x21	3x22	3x23	3x24	3x25	3x26	3x27	3x28	4x1	4x2	4x3	4x4	4x5	4x6	4x7	4x8	4x9	4x10	4x11	4x12	4x13	4x14	4x15	4x16	⋯
	Float32	Float32	Float32	Float32	Float32	Float32	Float32	Float32	Float32	Float32	Float32	Float32	Float32	Float32	Float32	Float32	Float32	Float32	Float32	Float32	Float32	Float32	Float32	Float32	Float32	Float32	Float32	Float32	Float32	Float32	Float32	Float32	Float32	Float32	Float32	Float32	Float32	Float32	Float32	Float32	Float32	Float32	Float32	Float32	Float32	Float32	Float32	Float32	Float32	Float32	Float32	Float32	Float32	Float32	Float32	Float32	Float32	Float32	Float32	Float32	Float32	Float32	Float32	Float32	Float32	Float32	Float32	Float32	Float32	Float32	Float32	Float32	Float32	Float32	Float32	Float32	Float32	Float32	Float32	Float32	Float32	Float32	Float32	Float32	Float32	Float32	Float32	Float32	Float32	Float32	Float32	Float32	Float32	Float32	Float32	Float32	Float32	Float32	Float32	Float32	⋯
1	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	⋯
2	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	⋯
3	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	⋯

3×784 DataFrame

684 columns omitted

Row	1x1	1x2	1x3	1x4	1x5	1x6	1x7	1x8	1x9	1x10	1x11	1x12	1x13	1x14	1x15	1x16	1x17	1x18	1x19	1x20	1x21	1x22	1x23	1x24	1x25	1x26	1x27	1x28	2x1	2x2	2x3	2x4	2x5	2x6	2x7	2x8	2x9	2x10	2x11	2x12	2x13	2x14	2x15	2x16	2x17	2x18	2x19	2x20	2x21	2x22	2x23	2x24	2x25	2x26	2x27	2x28	3x1	3x2	3x3	3x4	3x5	3x6	3x7	3x8	3x9	3x10	3x11	3x12	3x13	3x14	3x15	3x16	3x17	3x18	3x19	3x20	3x21	3x22	3x23	3x24	3x25	3x26	3x27	3x28	4x1	4x2	4x3	4x4	4x5	4x6	4x7	4x8	4x9	4x10	4x11	4x12	4x13	4x14	4x15	4x16	⋯
	Float32	Float32	Float32	Float32	Float32	Float32	Float32	Float32	Float32	Float32	Float32	Float32	Float32	Float32	Float32	Float32	Float32	Float32	Float32	Float32	Float32	Float32	Float32	Float32	Float32	Float32	Float32	Float32	Float32	Float32	Float32	Float32	Float32	Float32	Float32	Float32	Float32	Float32	Float32	Float32	Float32	Float32	Float32	Float32	Float32	Float32	Float32	Float32	Float32	Float32	Float32	Float32	Float32	Float32	Float32	Float32	Float32	Float32	Float32	Float32	Float32	Float32	Float32	Float32	Float32	Float32	Float32	Float32	Float32	Float32	Float32	Float32	Float32	Float32	Float32	Float32	Float32	Float32	Float32	Float32	Float32	Float32	Float32	Float32	Float32	Float32	Float32	Float32	Float32	Float32	Float32	Float32	Float32	Float32	Float32	Float32	Float32	Float32	Float32	Float32	⋯
1	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	⋯
2	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	⋯
3	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	0.0	⋯

Grey levels 0-255 standardized between 0-1 (not required here but used when fitting deep learning models)

Xtrain = Matrix(Xtrain) / 255
Xtest = Matrix(Xtest) / 255

2000×784 Matrix{Float32}:
 0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  …  0.0  0.0  0.0  0.0  0.0  0.0  0.0
 0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0     0.0  0.0  0.0  0.0  0.0  0.0  0.0
 0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0     0.0  0.0  0.0  0.0  0.0  0.0  0.0
 0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0     0.0  0.0  0.0  0.0  0.0  0.0  0.0
 0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0     0.0  0.0  0.0  0.0  0.0  0.0  0.0
 0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  …  0.0  0.0  0.0  0.0  0.0  0.0  0.0
 0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0     0.0  0.0  0.0  0.0  0.0  0.0  0.0
 0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0     0.0  0.0  0.0  0.0  0.0  0.0  0.0
 0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0     0.0  0.0  0.0  0.0  0.0  0.0  0.0
 0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0     0.0  0.0  0.0  0.0  0.0  0.0  0.0
 ⋮                        ⋮              ⋱                 ⋮              
 0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0     0.0  0.0  0.0  0.0  0.0  0.0  0.0
 0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0     0.0  0.0  0.0  0.0  0.0  0.0  0.0
 0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0     0.0  0.0  0.0  0.0  0.0  0.0  0.0
 0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0     0.0  0.0  0.0  0.0  0.0  0.0  0.0
 0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  …  0.0  0.0  0.0  0.0  0.0  0.0  0.0
 0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0     0.0  0.0  0.0  0.0  0.0  0.0  0.0
 0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0     0.0  0.0  0.0  0.0  0.0  0.0  0.0
 0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0     0.0  0.0  0.0  0.0  0.0  0.0  0.0
 0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0     0.0  0.0  0.0  0.0  0.0  0.0  0.0

Example of one sample (= one unfolded image)

plotsp(Xtrain, 1:p; nsamp = 1, xlabel = "Pixel", ylabel = "Grey level").f

Split Train to Cal/Val for model tuning

Below, Cal and Val are built by random sampling (other designs could be used)

nval = 1000
nval / ntrain # sampling proportion 
s = samprand(ntrain, nval)

(train = [1, 2, 3, 4, 5, 6, 8, 10, 11, 12  …  11991, 11992, 11993, 11994, 11995, 11996, 11997, 11998, 11999, 12000], test = [7, 9, 20, 30, 42, 55, 70, 84, 93, 96  …  11839, 11845, 11853, 11869, 11876, 11877, 11900, 11905, 11925, 11934])

Xcal = Xtrain[s.train, :]
ycal = ytrain[s.train]
Xval = Xtrain[s.test, :]
yval = ytrain[s.test]
ncal = ntrain - nval 
(ntot = ntot, ntrain, ncal, nval, ntest)

(ntot = 14000, ntrain = 12000, ncal = 11000, nval = 1000, ntest = 2000)

Grid-search

nlvdis = [10; 20]; metric = [:mah]
h = [1; 2; 5; Inf]; k = [200; 300; 500; 1000]  
nlv = 0:15
pars = mpar(nlvdis = nlvdis, metric = metric, h = h, k = k) 
length(pars[1])

model = lwplsrda()
res = gridscore(model, Xcal, ycal, Xval, yval; score = errp, pars, nlv)

512×6 DataFrame

487 rows omitted

Row	nlvdis	metric	h	k	nlv	y1
	Any	Any	Any	Any	Int64	Float32
1	10	mah	1.0	200	0	0.083
2	10	mah	1.0	200	1	0.048
3	10	mah	1.0	200	2	0.041
4	10	mah	1.0	200	3	0.036
5	10	mah	1.0	200	4	0.031
6	10	mah	1.0	200	5	0.031
7	10	mah	1.0	200	6	0.037
8	10	mah	1.0	200	7	0.037
9	10	mah	1.0	200	8	0.039
10	10	mah	1.0	200	9	0.039
11	10	mah	1.0	200	10	0.042
12	10	mah	1.0	200	11	0.041
13	10	mah	1.0	200	12	0.037
⋮	⋮	⋮	⋮	⋮	⋮	⋮
501	20	mah	Inf	1000	4	0.064
502	20	mah	Inf	1000	5	0.062
503	20	mah	Inf	1000	6	0.061
504	20	mah	Inf	1000	7	0.061
505	20	mah	Inf	1000	8	0.056
506	20	mah	Inf	1000	9	0.056
507	20	mah	Inf	1000	10	0.06
508	20	mah	Inf	1000	11	0.06
509	20	mah	Inf	1000	12	0.06
510	20	mah	Inf	1000	13	0.057
511	20	mah	Inf	1000	14	0.062
512	20	mah	Inf	1000	15	0.061

group = string.("nvldis=", res.nlvdis, " h=", res.h, " k=", res.k)
plotgrid(res.nlv, res.y1, group; step = 2, xlabel = "Nb. LVs", ylabel = "ERRP-Val").f

Selection of the best parameter combination

u = findall(res.y1 .== minimum(res.y1))[1] 
res[u, :]

DataFrameRow (6 columns)

Row	nlvdis	metric	h	k	nlv	y1
	Any	Any	Any	Any	Int64	Float32
153	20	mah	1.0	300	8	0.026

Final prediction (Test) using the optimal model

model = lwplsrda(nlvdis = res.nlvdis[u], metric = res.metric[u], h = res.h[u], 
    k = res.k[u], nlv = res.nlv[u])
fit!(model, Xtrain, ytrain)
@head pred = predict(model, Xtest).pred

3×1 Matrix{Float32}:
 1.0
 0.0
 4.0
... (2000, 1)

Generalization error

errp(pred, ytest)

1×1 Matrix{Float64}:
 0.0275

merrp(pred, ytest)

1×1 Matrix{Float64}:
 0.02810010015224467

cf = conf(pred, ytest)
@names cf

(:cnt, :pct, :A, :Apct, :diagpct, :accpct, :lev)

cf.cnt

10×11 DataFrame

Row	y	pred_0.0	pred_1.0	pred_2.0	pred_3.0	pred_4.0	pred_5.0	pred_6.0	pred_7.0	pred_8.0	pred_9.0
	String	Int64	Int64	Int64	Int64	Int64	Int64	Int64	Int64	Int64	Int64
1	0.0	195	0	0	0	0	0	1	0	0	0
2	1.0	0	225	0	2	0	0	0	0	0	0
3	2.0	0	1	204	1	0	0	0	1	0	0
4	3.0	0	0	1	196	0	1	0	0	3	0
5	4.0	1	1	1	0	192	0	0	0	0	2
6	5.0	2	0	0	2	0	166	3	1	3	1
7	6.0	1	0	0	0	1	1	189	0	0	0
8	7.0	1	2	4	0	0	0	0	197	0	1
9	8.0	0	0	0	2	1	0	1	0	189	2
10	9.0	1	0	0	2	3	2	0	2	0	192

cf.pct

10×11 DataFrame

Row	levels	pred_0.0	pred_1.0	pred_2.0	pred_3.0	pred_4.0	pred_5.0	pred_6.0	pred_7.0	pred_8.0	pred_9.0
	String	Float64	Float64	Float64	Float64	Float64	Float64	Float64	Float64	Float64	Float64
1	0.0	99.5	0.0	0.0	0.0	0.0	0.0	0.5	0.0	0.0	0.0
2	1.0	0.0	99.1	0.0	0.9	0.0	0.0	0.0	0.0	0.0	0.0
3	2.0	0.0	0.5	98.6	0.5	0.0	0.0	0.0	0.5	0.0	0.0
4	3.0	0.0	0.0	0.5	97.5	0.0	0.5	0.0	0.0	1.5	0.0
5	4.0	0.5	0.5	0.5	0.0	97.5	0.0	0.0	0.0	0.0	1.0
6	5.0	1.1	0.0	0.0	1.1	0.0	93.3	1.7	0.6	1.7	0.6
7	6.0	0.5	0.0	0.0	0.0	0.5	0.5	98.4	0.0	0.0	0.0
8	7.0	0.5	1.0	2.0	0.0	0.0	0.0	0.0	96.1	0.0	0.5
9	8.0	0.0	0.0	0.0	1.0	0.5	0.0	0.5	0.0	96.9	1.0
10	9.0	0.5	0.0	0.0	1.0	1.5	1.0	0.0	1.0	0.0	95.0

plotconf(cf).f

plotconf(cf; cnt = false).f