using Jchemo, JchemoData
using JLD2, CairoMakie
using FreqTables

Data importation

path_jdat = dirname(dirname(pathof(JchemoData)))
db = joinpath(path_jdat, "data/forages2.jld2") 
@load db dat
@names dat

(:X, :Y)

X = dat.X 
@head X

... (485, 700)

Y = dat.Y
@head Y

... (485, 4)

y = Y.typ   # response variable (class membership)
test = Y.test
tab(y)

OrderedCollections.OrderedDict{String, Int64} with 3 entries:
  "Cereal and grass forages" => 160
  "Forage trees"             => 101
  "Legume forages"           => 224

freqtable(y, test)

3×2 Named Matrix{Int64}
             Dim1 ╲ Dim2 │   0    1
─────────────────────────┼─────────
Cereal and grass forages │ 100   60
Forage trees             │  56   45
Legume forages           │ 167   57

wlst = names(X)
wl = parse.(Int, wlst)
#plotsp(X, wl; xlabel = "Wavelength (nm)", ylabel = "Absorbance").f

700-element Vector{Int64}:
 1100
 1102
 1104
 1106
 1108
 1110
 1112
 1114
 1116
 1118
    ⋮
 2482
 2484
 2486
 2488
 2490
 2492
 2494
 2496
 2498

Note:: X-data are already preprocessed (SNV + Savitsky-Golay 2nd deriv).

Split Tot to Train/Test

The model is fitted on Train, and the generalization error is estimated on Test. In this example, Train is already defined in variable typ of the dataset, and Test is defined by the remaining samples. But Tot could also be split a posteriori, for instance by sampling (random, systematic or any other designs). See for instance functions samprand, sampsys, etc.

s = Bool.(test)
Xtrain = rmrow(X, s)
ytrain = rmrow(y, s)
Xtest = X[s, :]
ytest = y[s]
ntot = nro(X)
ntrain = nro(Xtrain)
ntest = nro(Xtest)
(ntot = ntot, ntrain, ntest)

(ntot = 485, ntrain = 323, ntest = 162)

tab(ytrain)

OrderedCollections.OrderedDict{String, Int64} with 3 entries:
  "Cereal and grass forages" => 100
  "Forage trees"             => 56
  "Legume forages"           => 167

tab(ytest)

OrderedCollections.OrderedDict{String, Int64} with 3 entries:
  "Cereal and grass forages" => 60
  "Forage trees"             => 45
  "Legume forages"           => 57

Replicated K-fold CV

K = 3     # nb. folds (segments)
rep = 25  # nb. replications
segm = segmkf(ntrain, K; rep = rep)

25-element Vector{Vector{Vector{Int64}}}:
 [[6, 7, 10, 11, 25, 26, 27, 32, 35, 36  …  290, 291, 298, 299, 306, 308, 310, 317, 318, 323], [2, 5, 13, 14, 17, 18, 19, 20, 21, 24  …  300, 302, 303, 307, 309, 314, 315, 316, 319, 321], [1, 3, 4, 8, 9, 12, 15, 16, 22, 23  …  295, 297, 301, 304, 305, 311, 312, 313, 320, 322]]
 [[2, 11, 13, 14, 15, 16, 18, 23, 36, 40  …  303, 305, 306, 308, 309, 310, 314, 315, 319, 322], [1, 3, 4, 6, 8, 9, 12, 20, 21, 22  …  296, 297, 299, 301, 302, 304, 311, 312, 317, 321], [5, 7, 10, 17, 19, 25, 30, 31, 33, 37  …  292, 293, 294, 295, 307, 313, 316, 318, 320, 323]]
 [[2, 3, 4, 9, 11, 16, 20, 25, 33, 43  …  301, 305, 307, 308, 311, 313, 317, 318, 321, 323], [5, 6, 10, 14, 15, 22, 23, 24, 26, 27  …  278, 285, 290, 295, 297, 302, 303, 304, 314, 319], [1, 7, 8, 12, 13, 17, 18, 19, 21, 29  …  299, 300, 306, 309, 310, 312, 315, 316, 320, 322]]
 [[1, 6, 9, 14, 17, 18, 19, 21, 22, 35  …  295, 300, 305, 306, 309, 310, 311, 317, 318, 320], [2, 4, 8, 10, 11, 23, 24, 26, 27, 30  …  301, 303, 304, 307, 314, 315, 316, 319, 321, 322], [3, 5, 7, 12, 13, 15, 16, 20, 25, 28  …  292, 293, 296, 298, 299, 302, 308, 312, 313, 323]]
 [[1, 5, 7, 9, 11, 12, 15, 18, 21, 23  …  283, 288, 296, 299, 304, 305, 313, 314, 318, 319], [4, 6, 8, 14, 16, 20, 22, 26, 27, 28  …  295, 300, 302, 303, 306, 309, 310, 312, 316, 320], [2, 3, 10, 13, 17, 19, 24, 25, 31, 34  …  298, 301, 307, 308, 311, 315, 317, 321, 322, 323]]
 [[6, 7, 8, 15, 16, 20, 25, 27, 28, 29  …  285, 288, 293, 299, 300, 304, 305, 312, 314, 319], [1, 3, 5, 9, 13, 14, 19, 21, 23, 24  …  291, 292, 297, 306, 308, 313, 315, 316, 318, 322], [2, 4, 10, 11, 12, 17, 18, 22, 34, 37  …  302, 303, 307, 309, 310, 311, 317, 320, 321, 323]]
 [[13, 16, 17, 21, 23, 27, 29, 34, 35, 37  …  296, 301, 302, 305, 307, 313, 315, 317, 322, 323], [2, 4, 7, 10, 14, 15, 18, 20, 22, 26  …  303, 304, 306, 308, 311, 312, 314, 316, 320, 321], [1, 3, 5, 6, 8, 9, 11, 12, 19, 24  …  273, 274, 281, 285, 287, 292, 309, 310, 318, 319]]
 [[4, 5, 8, 9, 17, 23, 25, 28, 31, 34  …  297, 298, 302, 304, 305, 307, 308, 313, 321, 322], [1, 2, 7, 10, 11, 12, 13, 15, 16, 18  …  303, 306, 310, 311, 314, 316, 318, 319, 320, 323], [3, 6, 14, 19, 20, 22, 30, 33, 35, 39  …  277, 280, 281, 294, 295, 296, 309, 312, 315, 317]]
 [[1, 2, 8, 9, 11, 15, 17, 22, 26, 29  …  294, 296, 301, 302, 303, 306, 313, 316, 318, 321], [6, 12, 14, 16, 18, 24, 27, 28, 30, 31  …  293, 295, 297, 299, 307, 311, 314, 315, 320, 323], [3, 4, 5, 7, 10, 13, 19, 20, 21, 23  …  300, 304, 305, 308, 309, 310, 312, 317, 319, 322]]
 [[11, 12, 14, 17, 21, 22, 23, 25, 29, 31  …  298, 300, 302, 304, 306, 311, 316, 318, 320, 322], [3, 5, 6, 8, 9, 18, 19, 20, 27, 30  …  293, 294, 296, 297, 301, 303, 305, 312, 315, 323], [1, 2, 4, 7, 10, 13, 15, 16, 24, 26  …  299, 307, 308, 309, 310, 313, 314, 317, 319, 321]]
 ⋮
 [[1, 2, 5, 9, 14, 17, 19, 21, 23, 25  …  285, 287, 288, 295, 296, 297, 299, 302, 304, 319], [3, 4, 7, 8, 13, 18, 20, 22, 24, 27  …  309, 310, 311, 312, 313, 314, 315, 316, 321, 322], [6, 10, 11, 12, 15, 16, 26, 28, 30, 32  …  292, 293, 294, 300, 306, 308, 317, 318, 320, 323]]
 [[4, 12, 17, 21, 23, 24, 26, 29, 31, 33  …  300, 302, 306, 312, 313, 317, 318, 319, 320, 322], [1, 2, 3, 5, 7, 9, 10, 13, 14, 15  …  281, 284, 285, 288, 298, 303, 304, 305, 311, 315], [6, 8, 11, 16, 20, 22, 25, 28, 34, 35  …  299, 301, 307, 308, 309, 310, 314, 316, 321, 323]]
 [[1, 3, 7, 8, 15, 16, 18, 19, 22, 23  …  291, 293, 295, 296, 297, 300, 303, 305, 320, 323], [5, 9, 11, 14, 17, 20, 21, 24, 25, 28  …  299, 301, 306, 309, 312, 314, 315, 317, 319, 322], [2, 4, 6, 10, 12, 13, 32, 35, 36, 37  …  302, 304, 307, 308, 310, 311, 313, 316, 318, 321]]
 [[15, 20, 25, 30, 31, 33, 36, 37, 40, 42  …  295, 298, 299, 300, 301, 307, 309, 312, 315, 319], [1, 3, 4, 7, 9, 12, 13, 18, 22, 23  …  297, 302, 303, 308, 311, 314, 320, 321, 322, 323], [2, 5, 6, 8, 10, 11, 14, 16, 17, 19  …  293, 296, 304, 305, 306, 310, 313, 316, 317, 318]]
 [[2, 4, 8, 10, 16, 21, 22, 23, 24, 26  …  290, 292, 295, 296, 298, 300, 307, 308, 317, 318], [1, 6, 11, 14, 15, 17, 18, 20, 25, 27  …  310, 311, 312, 313, 314, 316, 319, 320, 322, 323], [3, 5, 7, 9, 12, 13, 19, 28, 32, 38  …  278, 285, 286, 293, 297, 303, 304, 306, 315, 321]]
 [[3, 5, 15, 18, 22, 23, 25, 27, 28, 29  …  290, 291, 292, 294, 303, 304, 308, 313, 316, 321], [1, 2, 7, 10, 11, 12, 13, 14, 16, 30  …  295, 296, 297, 300, 301, 310, 312, 318, 320, 323], [4, 6, 8, 9, 17, 19, 20, 21, 24, 26  …  305, 306, 307, 309, 311, 314, 315, 317, 319, 322]]
 [[6, 8, 10, 15, 17, 19, 24, 26, 33, 35  …  301, 305, 308, 309, 310, 311, 312, 314, 319, 323], [2, 3, 4, 9, 13, 14, 16, 18, 22, 29  …  295, 302, 303, 306, 307, 313, 315, 316, 317, 321], [1, 5, 7, 11, 12, 20, 21, 23, 25, 27  …  287, 288, 291, 297, 299, 300, 304, 318, 320, 322]]
 [[2, 4, 5, 9, 11, 15, 16, 20, 21, 30  …  293, 298, 299, 306, 311, 313, 314, 315, 316, 322], [1, 3, 6, 7, 8, 10, 13, 17, 24, 27  …  303, 304, 305, 307, 312, 317, 318, 319, 320, 321], [12, 14, 18, 19, 22, 23, 25, 26, 28, 31  …  291, 294, 296, 297, 300, 302, 308, 309, 310, 323]]
 [[2, 4, 11, 12, 20, 21, 24, 25, 27, 29  …  292, 294, 297, 300, 301, 303, 306, 307, 309, 312], [1, 5, 6, 7, 8, 9, 14, 18, 19, 26  …  310, 311, 313, 314, 315, 316, 317, 319, 322, 323], [3, 10, 13, 15, 16, 17, 22, 23, 30, 32  …  293, 295, 296, 298, 299, 302, 304, 318, 320, 321]]

nlv = 0:30
model = plsrda()
rescv = gridcv(model, Xtrain, ytrain; segm, score = errp, nlv)
@names rescv 
res = rescv.res
res_rep = rescv.res_rep

plotgrid(res.nlv, res.y1; step = 2, xlabel = "Nb. LVs", ylabel = "ERRP-CV").f

f, ax = plotgrid(res.nlv, res.y1; step = 2, xlabel = "Nb. LVs", ylabel = "ERRP-CV")
for i = 1:rep, j = 1:K
    zres = res_rep[res_rep.rep .== i .&& res_rep.segm .== j, :]
    lines!(ax, zres.nlv, zres.y1; color = (:grey, .2))
end
lines!(ax, res.nlv, res.y1; color = :red, linewidth = 1)
f

Specifying argument prior

This is recommended if classes are highly unbalanced.

prior = [:unif]  
pars = mpar(prior = prior)
nlv = 0:30
model = plsrda()
res = gridcv(model, Xtrain, ytrain; segm, score = merrp, pars, nlv).res

Selection of the best parameter combination

u = findall(res.y1 .== minimum(res.y1))[1] 
res[u, :]

Final prediction (Test) using the optimal model

model = plsrda(nlv = res.nlv[u], prior = res.prior[u])
fit!(model, Xtrain, ytrain)
pred = predict(model, Xtest).pred

162×1 Matrix{String}:
 "Forage trees"
 "Cereal and grass forages"
 "Cereal and grass forages"
 "Forage trees"
 "Cereal and grass forages"
 "Cereal and grass forages"
 "Legume forages"
 "Forage trees"
 "Forage trees"
 "Forage trees"
 ⋮
 "Cereal and grass forages"
 "Cereal and grass forages"
 "Forage trees"
 "Forage trees"
 "Legume forages"
 "Legume forages"
 "Legume forages"
 "Legume forages"
 "Legume forages"

Generalization error

errp(pred, ytest)

1×1 Matrix{Float64}:
 0.12345679012345678

merrp(pred, ytest)

1×1 Matrix{Float64}:
 0.12300194931773878

cf = conf(pred, ytest)
@names cf

(:cnt, :pct, :A, :Apct, :diagpct, :accpct, :lev)

cf.cnt

cf.pct

---