using Jchemo, JchemoData
using JLD2, DataFrames, CairoMakie

Data importation

path_jdat = dirname(dirname(pathof(JchemoData)))
db = joinpath(path_jdat, "data/tecator.jld2") 
@load db dat
@names dat

(:X, :Y)

X = dat.X
@head X

... (178, 100)

Y = dat.Y
@head Y

... (178, 4)

X data

wlst = names(X) 
wl = parse.(Int, wlst)

100-element Vector{Int64}:
  850
  852
  854
  856
  858
  860
  862
  864
  866
  868
    ⋮
 1032
 1034
 1036
 1038
 1040
 1042
 1044
 1046
 1048

plotsp(X, wl; xlabel = "Wavelength (nm)", ylabel = "Absorbance").f

Preprocessing by SNV and derivation

model1 = snv()
model2 = savgol(npoint = 15, deriv = 2, degree = 3)
model = pip(model1, model2)
fit!(model, X)
Xp = transf(model, X)
@head Xp

3×100 Matrix{Float64}:
 0.000397076  0.000495203  0.000598623  …  0.00827138  0.00917311  0.00946072
 0.00242055   0.00244366   0.00234233      0.00580631  0.00689249  0.00749316
 0.0011927    0.00122721   0.00120098      0.0101019   0.0108142   0.0108444
... (178, 100)

plotsp(Xp, wl; xlabel = "Wavelength (nm)", ylabel = "Absorbance").f

Y data

summ(Y).res

typ = Y.typ
tab(typ)

OrderedCollections.OrderedDict{String, Int64} with 3 entries:
  "test"  => 31
  "train" => 115
  "val"   => 32

Building a new variable, typ2, aggregating categories test and validation in a new category test

## Training/test (0/1) observations
typ2 = ones(Int, nro(typ))
typ2[typ .== "train"] .= 0
tab(typ2)

OrderedCollections.OrderedDict{Int64, Int64} with 2 entries:
  0 => 115
  1 => 63

summ(Y, typ2)

Class: 0
4×7 DataFrame
 Row │ variable  mean    std     min    max    n      nmissing
     │ Symbol    Union…  Union…  Any    Any    Int64  Int64
─────┼─────────────────────────────────────────────────────────
   1 │ water     63.142  9.846   39.3   76.6     115         0
   2 │ fat       18.358  12.605  0.9    49.1     115         0
   3 │ protein   17.597  2.989   11.0   21.8     115         0
   4 │ typ                       train  train    115         0


Class: 1
4×7 DataFrame
 Row │ variable  mean    std     min   max   n      nmissing
     │ Symbol    Union…  Union…  Any   Any   Int64  Int64
─────┼───────────────────────────────────────────────────────
   1 │ water     62.87   9.745   40.7  76.1     63         0
   2 │ fat       18.414  12.774  1.4   47.8     63         0
   3 │ protein   17.702  3.031   11.1  21.6     63         0
   4 │ typ                       test  val      63         0

A given variable

namy = names(Y)[1:3]
j = 2
nam = namy[2]
y = Y[:, nam]
s = typ2 .== 0
ytrain = y[s] 
ytest = rmrow(y, s)

63-element Vector{Float64}:
 29.8
  1.4
  4.6
 11.0
 17.0
 22.4
 27.9
 46.5
  6.1
  2.0
  ⋮
 18.1
 19.4
 24.8
 27.2
 28.4
 31.3
 33.8
 35.5
 42.5

f = Figure(size = (400, 300))
ax = Axis(f[1, 1]; xlabel = uppercase(nam), ylabel = "Nb. observations")
hist!(ax, ytrain; bins = 50, label = "Train")
hist!(ax, ytest; bins = 50, label = "Test")
axislegend(position = :rt)
f

f = Figure(size = (500, 400))
offs = [10; 0]
ax = Axis(f[1, 1]; xlabel = uppercase(nam),  ylabel = "Nb. observations", 
    yticks = (offs, ["Train"; "Test"]))
hist!(ax, ytrain; offset = offs[1], bins = 50)
hist!(ax, ytest; offset = offs[2], bins = 50)
f

f = Figure(size = (400, 300))
ax = Axis(f[1, 1]; xlabel = uppercase(nam), ylabel = "Density")
bdw = .5 
density!(ax, ytrain; bandwidth = bdw, color = :blue, label = "Train")
density!(ax, ytest; bandwidth = bdw, color = (:red, .5), label = "Test")
axislegend(position = :rt)  
f

f = Figure(size = (500, 400))
offs = [.15; 0]
ax = Axis(f[1, 1]; xlabel = uppercase(nam), ylabel = "Density", 
    yticks = (offs, ["Train"; "Test"]))
bdw = .5
density!(ax, ytrain; bandwidth = bdw, offset = offs[1], color = (:slategray, 0.5))
density!(ax, ytest; bandwidth = bdw, offset = offs[2], color = (:slategray, 0.5))
f

f = Figure(size = (400, 300))
ax = Axis(f[1, 1]; xticks = (0:1, ["Train", "Test"]), xlabel = "Group", ylabel = uppercase(nam))
boxplot!(ax, typ2, y; width = .3, show_notch = true)
f

---