diff --git a/docs/src/patterns/classification.md b/docs/src/patterns/classification.md index 2913cea5..47e04043 100644 --- a/docs/src/patterns/classification.md +++ b/docs/src/patterns/classification.md @@ -1,5 +1,6 @@ # Classification -See these examples from tests: +See these examples from the JuliaTestAPI.jl test suite: -- [perceptron classifier](https://github.com/JuliaAI/LearnAPI.jl/blob/dev/test/patterns/gradient_descent.jl) +- [perceptron + classifier](https://github.com/JuliaAI/LearnTestAPI.jl/blob/dev/test/patterns/gradient_descent.jl) diff --git a/docs/src/patterns/density_estimation.md b/docs/src/patterns/density_estimation.md index e9ca083b..9fc0144a 100644 --- a/docs/src/patterns/density_estimation.md +++ b/docs/src/patterns/density_estimation.md @@ -1,5 +1,5 @@ # Density Estimation -See these examples from tests: +See these examples from the JuliaTestAPI.jl test suite: -- [normal distribution estimator](https://github.com/JuliaAI/LearnAPI.jl/blob/dev/test/patterns/incremental_algorithms.jl) +- [normal distribution estimator](https://github.com/JuliaAI/LearnTestAPI.jl/blob/dev/test/patterns/incremental_algorithms.jl) diff --git a/docs/src/patterns/ensembling.md b/docs/src/patterns/ensembling.md index a93ae305..ea5faf88 100644 --- a/docs/src/patterns/ensembling.md +++ b/docs/src/patterns/ensembling.md @@ -1,5 +1,6 @@ # Ensembling -See [this -example](https://github.com/JuliaAI/LearnAPI.jl/blob/dev/test/patterns/ensembling.jl) -from tests. +See these examples from the JuliaTestAPI.jl test suite: + +- [bagged ensembling of a regression model](https://github.com/JuliaAI/LearnTestAPI.jl/blob/dev/test/patterns/ensembling.jl) + diff --git a/docs/src/patterns/feature_engineering.md b/docs/src/patterns/feature_engineering.md index 850dc0e3..6e3c656c 100644 --- a/docs/src/patterns/feature_engineering.md +++ b/docs/src/patterns/feature_engineering.md @@ -1,4 +1,7 @@ # Feature Engineering -For a simple feature selection algorithm (no "learning) see -[these examples](https://github.com/JuliaAI/LearnAPI.jl/blob/dev/test/patterns/static_algorithms.jl) from tests. +See these examples from the JuliaTestAI.jl test suite: + +- [feature + selectors](https://github.com/JuliaAI/LearnTestAPI.jl/blob/dev/test/patterns/static_algorithms.jl) + from tests. diff --git a/docs/src/patterns/gradient_descent.md b/docs/src/patterns/gradient_descent.md index 7fd4a11c..c898b38e 100644 --- a/docs/src/patterns/gradient_descent.md +++ b/docs/src/patterns/gradient_descent.md @@ -1,5 +1,6 @@ # Gradient Descent -See [this -example](https://github.com/JuliaAI/LearnAPI.jl/blob/dev/test/patterns/gradient_descent.jl) -from tests. +See these examples from the JuliaTestAI.jl test suite: + +- [perceptron +classifier](https://github.com/JuliaAI/LearnTestAPI.jl/blob/dev/test/patterns/gradient_descent.jl) diff --git a/docs/src/patterns/incremental_algorithms.md b/docs/src/patterns/incremental_algorithms.md index 89ad8643..d2855a55 100644 --- a/docs/src/patterns/incremental_algorithms.md +++ b/docs/src/patterns/incremental_algorithms.md @@ -1,5 +1,5 @@ # Incremental Algorithms -See these examples from tests: +See these examples from the JuliaTestAI.jl test suite: -- [normal distribution estimator](https://github.com/JuliaAI/LearnAPI.jl/blob/dev/test/patterns/incremental_algorithms.jl) +- [normal distribution estimator](https://github.com/JuliaAI/LearnTestAPI.jl/blob/dev/test/patterns/incremental_algorithms.jl) diff --git a/docs/src/patterns/iterative_algorithms.md b/docs/src/patterns/iterative_algorithms.md index 1cf4ab23..b12c6142 100644 --- a/docs/src/patterns/iterative_algorithms.md +++ b/docs/src/patterns/iterative_algorithms.md @@ -1,7 +1,7 @@ # Iterative Algorithms -See these examples from tests: +See these examples from the JuliaTestAI.jl test suite: -- [bagged ensembling](https://github.com/JuliaAI/LearnAPI.jl/blob/dev/test/patterns/ensembling.jl) +- [bagged ensembling](https://github.com/JuliaAI/LearnTestAPI.jl/blob/dev/test/patterns/ensembling.jl) -- [perceptron classifier](https://github.com/JuliaAI/LearnAPI.jl/blob/dev/test/patterns/gradient_descent.jl) +- [perceptron classifier](https://github.com/JuliaAI/LearnTestAPI.jl/blob/dev/test/patterns/gradient_descent.jl) diff --git a/docs/src/patterns/meta_algorithms.md b/docs/src/patterns/meta_algorithms.md index 17ccad8f..6a9e7300 100644 --- a/docs/src/patterns/meta_algorithms.md +++ b/docs/src/patterns/meta_algorithms.md @@ -1,7 +1,7 @@ # Meta-algorithms -Many meta-algorithms are can be implemented as wrappers. An example is [this bagged +Many meta-algorithms are can be implemented as wrappers. An example is [this bagged ensemble -algorithm](https://github.com/JuliaAI/LearnAPI.jl/blob/dev/test/patterns/ensembling.jl) +algorithm](https://github.com/JuliaAI/LearnTestAPI.jl/blob/dev/test/patterns/ensembling.jl) from tests. diff --git a/docs/src/patterns/regression.md b/docs/src/patterns/regression.md index 7cf3b6d0..1a50aecf 100644 --- a/docs/src/patterns/regression.md +++ b/docs/src/patterns/regression.md @@ -1,5 +1,7 @@ # Regression -See [these -examples](https://github.com/JuliaAI/LearnAPI.jl/blob/dev/test/patterns/regression.jl) -from tests. +See these examples from the JuliaTestAI.jl test suite: + +- [ridge + regression](https://github.com/JuliaAI/LearnTestAPI.jl/blob/dev/test/patterns/regression.jl) + diff --git a/docs/src/patterns/static_algorithms.md b/docs/src/patterns/static_algorithms.md index 21a517dc..4724006f 100644 --- a/docs/src/patterns/static_algorithms.md +++ b/docs/src/patterns/static_algorithms.md @@ -1,7 +1,9 @@ # Static Algorithms -See [these -examples](https://github.com/JuliaAI/LearnAPI.jl/blob/dev/test/patterns/static_algorithms.jl) -from tests. +See these examples from the JuliaTestAI.jl test suite: + +- [feature + selection](https://github.com/JuliaAI/LearnTestAPI.jl/blob/dev/test/patterns/static_algorithms.jl) + diff --git a/test/patterns/ensembling.jl b/test/patterns/ensembling.jl deleted file mode 100644 index 73b864b8..00000000 --- a/test/patterns/ensembling.jl +++ /dev/null @@ -1,215 +0,0 @@ -using LearnAPI -using LinearAlgebra -using Tables -import MLUtils -import DataFrames -using Random -using Statistics -using StableRNGs - -# # ENSEMBLE OF REGRESSORS (A MODEL WRAPPER) - -# We implement a learner that creates an bagged ensemble of regressors, i.e, where each -# atomic model is trained on a random sample of the training observations (same number, -# but sampled with replacement). In particular this learner has an iteration parameter -# `n`, and we implement `update` to execute a warm restarts when `n` increases. - -# no docstring here - that goes with the constructor; some fields left abstract for -# simplicity -# -struct Ensemble - atom # the base regressor being bagged - rng - n::Int -end - -# Since the `atom` hyperparameter is another learner, the user must explicitly set it in -# constructor calls or an error is thrown. We also need to overload the -# `LearnAPI.is_composite` trait (done later). - -""" - Ensemble(atom; rng=Random.default_rng(), n=10) - -Instantiate a bagged ensemble of `n` regressors, with base regressor `atom`, etc - -""" -Ensemble(atom; rng=Random.default_rng(), n=10) = - Ensemble(atom, rng, n) # `LearnAPI.constructor` defined later - -# need a pure keyword argument constructor: -function Ensemble(; atom=nothing, kwargs...) - isnothing(atom) && error("You must specify `atom=...` ") - Ensemble(atom; kwargs...) -end - -struct EnsembleFitted - learner::Ensemble - atom::Ridge - rng # mutated copy of `learner.rng` - models # leaving type abstract for simplicity -end - -LearnAPI.learner(model::EnsembleFitted) = model.learner - -# We add the same data interface that the atomic regressor uses: -LearnAPI.obs(learner::Ensemble, data) = LearnAPI.obs(learner.atom, data) -LearnAPI.obs(model::EnsembleFitted, data) = LearnAPI.obs(first(model.models), data) -LearnAPI.target(learner::Ensemble, data) = LearnAPI.target(learner.atom, data) -LearnAPI.features(learner::Ensemble, data) = LearnAPI.features(learner.atom, data) - -function LearnAPI.fit(learner::Ensemble, data; verbosity=1) - - # unpack hyperparameters: - atom = learner.atom - rng = deepcopy(learner.rng) # to prevent mutation of `learner`! - n = learner.n - - # ensure data can be subsampled using MLUtils.jl, and that we're feeding the atomic - # `fit` data in an efficient (pre-processed) form: - - observations = obs(atom, data) - - # initialize ensemble: - models = [] - - # get number of observations: - N = MLUtils.numobs(observations) - - # train the ensemble: - for _ in 1:n - bag = rand(rng, 1:N, N) - data_subset = MLUtils.getobs(observations, bag) - # step down one verbosity level in atomic fit: - model = fit(atom, data_subset; verbosity=verbosity - 1) - push!(models, model) - end - - # make some noise, if allowed: - verbosity > 0 && @info "Trained $n ridge regression models. " - - return EnsembleFitted(learner, atom, rng, models) - -end - -# Consistent with the documented `update` contract, we implement this behaviour: If `n` is -# increased, `update` adds new regressors to the ensemble, including any new -# hyperparameter updates (e.g, new `atom`) when computing the new atomic -# models. Otherwise, update is equivalent to retraining from scratch, with the provided -# hyperparameter updates. -function LearnAPI.update(model::EnsembleFitted, data; verbosity=1, replacements...) - learner_old = LearnAPI.learner(model) - learner = LearnAPI.clone(learner_old; replacements...) - - :n in keys(replacements) || return fit(learner, data) - - n = learner.n - Δn = n - learner_old.n - n < 0 && return fit(model, learner) - - atom = learner.atom - observations = obs(atom, data) - N = MLUtils.numobs(observations) - - # initialize: - models = model.models - rng = model.rng # as mutated in previous `fit`/`update` calls - - # add new regressors to the ensemble: - for _ in 1:Δn - bag = rand(rng, 1:N, N) - data_subset = MLUtils.getobs(observations, bag) - model = fit(atom, data_subset; verbosity=verbosity-1) - push!(models, model) - end - - # make some noise, if allowed: - verbosity > 0 && @info "Trained $Δn additional ridge regression models. " - - return EnsembleFitted(learner, atom, rng, models) -end - -LearnAPI.predict(model::EnsembleFitted, ::Point, data) = - mean(model.models) do atomic_model - predict(atomic_model, Point(), data) - end - -LearnAPI.strip(model::EnsembleFitted) = EnsembleFitted( - model.learner, - model.atom, - model.rng, - LearnAPI.strip.(Ref(model.atom), models), -) - -# learner traits (note the inclusion of `iteration_parameter`): -@trait( - Ensemble, - constructor = Ensemble, - iteration_parameter = :n, - is_composite = true, - kinds_of_proxy = (Point(),), - tags = ("regression", "ensemble algorithms", "iterative models"), - functions = ( - :(LearnAPI.fit), - :(LearnAPI.learner), - :(LearnAPI.strip), - :(LearnAPI.obs), - :(LearnAPI.features), - :(LearnAPI.target), - :(LearnAPI.update), - :(LearnAPI.predict), - ) -) - -# convenience method: -LearnAPI.fit(learner::Ensemble, X, y, extras...; kwargs...) = - fit(learner, (X, y, extras...); kwargs...) -LearnAPI.update(learner::EnsembleFitted, X, y, extras...; kwargs...) = - update(learner, (X, y, extras...); kwargs...) - - -# synthetic test data: -N = 10 # number of observations -train = 1:6 -test = 7:10 -a, b, c = rand(N), rand(N), rand(N) -X = (; a, b, c) -X = DataFrames.DataFrame(X) -y = 2a - b + 3c + 0.05*rand(N) -data = (X, y) -Xtrain = Tables.subset(X, train) -Xtest = Tables.subset(X, test) - -@testset "test an implementation of bagged ensemble of ridge regressors" begin - rng = StableRNG(123) - atom = Ridge() - learner = Ensemble(atom; n=4, rng) - @test LearnAPI.clone(learner) == learner - @test :(LearnAPI.obs) in LearnAPI.functions(learner) - @test LearnAPI.target(learner, data) == y - @test LearnAPI.features(learner, data) == X - - model = @test_logs( - (:info, r"Trained 4 ridge"), - fit(learner, Xtrain, y[train]; verbosity=1), - ); - - ŷ4 = predict(model, Point(), Xtest) - @test ŷ4 == predict(model, Xtest) - - # add 3 atomic models to the ensemble: - model = update(model, Xtrain, y[train]; verbosity=0, n=7); - ŷ7 = predict(model, Xtest) - - # compare with cold restart: - model_cold = fit(LearnAPI.clone(learner; n=7), Xtrain, y[train]; verbosity=0); - @test ŷ7 ≈ predict(model_cold, Xtest) - - # test that we get a cold restart if another hyperparameter is changed: - model2 = update(model, Xtrain, y[train]; atom=Ridge(0.05)) - learner2 = Ensemble(Ridge(0.05); n=7, rng) - model_cold = fit(learner2, Xtrain, y[train]; verbosity=0) - @test predict(model2, Xtest) ≈ predict(model_cold, Xtest) - -end - -true diff --git a/test/patterns/gradient_descent.jl b/test/patterns/gradient_descent.jl deleted file mode 100644 index 27c9791e..00000000 --- a/test/patterns/gradient_descent.jl +++ /dev/null @@ -1,396 +0,0 @@ -using Pkg -Pkg.activate("perceptron", shared=true) - -using LearnAPI -using Random -using Statistics -using StableRNGs -import Optimisers -import Zygote -import NNlib -import CategoricalDistributions -import CategoricalDistributions: pdf, mode -import ComponentArrays - -# # PERCEPTRON - -# We implement a simple perceptron classifier to illustrate some common patterns for -# gradient descent algorithms. This includes implementation of the following methods: - -# - `update` -# - `update_observations` -# - `iteration_parameter` -# - `training_losses` -# - `obs` for pre-processing (non-tabular) classification training data -# - `predict(learner, ::Distribution, Xnew)` - -# For simplicity, we use single-observation batches for gradient descent updates, and we -# may dodge some optimizations. - -# This is an example of a probability-predicting classifier. - - -# ## Helpers - -""" - brier_loss(probs, hot) - -Return Brier (quadratic) loss. - -- `probs`: predicted probability vector -- `hot`: corresponding ground truth observation, as a one-hot encoded `BitVector` - -""" -function brier_loss(probs, hot) - offset = 1 + sum(probs.^2) - return offset - 2*(sum(probs.*hot)) -end - -""" - corefit(perceptron, optimiser, X, y_hot, epochs, state, verbosity) - -Return updated `perceptron`, `state` and training losses by carrying out gradient descent -for the specified number of `epochs`. - -- `perceptron`: component array with components `weights` and `bias` -- `optimiser`: optimiser from Optimiser.jl -- `X`: feature matrix, of size `(p, n)` -- `y_hot`: one-hot encoded target, of size `(nclasses, n)` -- `epochs`: number of epochs -- `state`: optimiser state - -""" -function corefit(perceptron, X, y_hot, epochs, state, verbosity) - n = size(y_hot) |> last - losses = map(1:epochs) do _ - total_loss = zero(Float32) - for i in 1:n - loss, grad = Zygote.withgradient(perceptron) do p - probs = p.weights*X[:,i] + p.bias |> NNlib.softmax - brier_loss(probs, y_hot[:,i]) - end - ∇loss = only(grad) - state, perceptron = Optimisers.update(state, perceptron, ∇loss) - total_loss += loss - end - # make some noise, if allowed: - verbosity > 0 && @info "Training loss: $total_loss" - total_loss - end - return perceptron, state, losses -end - - -# ## Implementation - -# ### Learner - -# no docstring here - that goes with the constructor; -# SOME FIELDS LEFT ABSTRACT FOR SIMPLICITY -struct PerceptronClassifier - epochs::Int - optimiser # an optmiser from Optimsers.jl - rng -end - -""" - PerceptronClassifier(; epochs=50, optimiser=Optimisers.Adam(), rng=Random.default_rng()) - -Instantiate a perceptron classifier. - -Train an instance, `learner`, by doing `model = fit(learner, X, y)`, where - -- `X is a `Float32` matrix, with observations-as-columns -- `y` (target) is some one-dimensional `CategoricalArray`. - -Get probabilistic predictions with `predict(model, Xnew)` and -point predictions with `predict(model, Point(), Xnew)`. - -# Warm restart options - - update_observations(model, newdata; replacements...) - -Return an updated model, with the weights and bias of the previously learned perceptron -used as the starting state in new gradient descent updates. Adopt any specified -hyperparameter `replacements` (properties of `LearnAPI.learner(model)`). - - update(model, newdata; epochs=n, replacements...) - -If `Δepochs = n - perceptron.epochs` is non-negative, then return an updated model, with -the weights and bias of the previously learned perceptron used as the starting state in -new gradient descent updates for `Δepochs` epochs, and using the provided `newdata` -instead of the previous training data. Any other hyperparaameter `replacements` are also -adopted. If `Δepochs` is negative or not specified, instead return `fit(learner, -newdata)`, where `learner=LearnAPI.clone(learner; epochs=n, replacements....)`. - -""" -PerceptronClassifier(; epochs=50, optimiser=Optimisers.Adam(), rng=Random.default_rng()) = - PerceptronClassifier(epochs, optimiser, rng) - - -# ### Data interface - -# For raw training data: -LearnAPI.target(learner::PerceptronClassifier, data::Tuple) = last(data) - -# For wrapping pre-processed training data (output of `obs(learner, data)`): -struct PerceptronClassifierObs - X::Matrix{Float32} - y_hot::BitMatrix # one-hot encoded target - classes # the (ordered) pool of `y`, as `CategoricalValue`s -end - -# For pre-processing the training data: -function LearnAPI.obs(learner::PerceptronClassifier, data::Tuple) - X, y = data - classes = CategoricalDistributions.classes(y) - y_hot = classes .== permutedims(y) # one-hot encoding - return PerceptronClassifierObs(X, y_hot, classes) -end - -# implement `RadomAccess()` interface for output of `obs`: -Base.length(observations::PerceptronClassifierObs) = length(observations.y) -Base.getindex(observations, I) = PerceptronClassifierObs( - (@view observations.X[:, I]), - (@view observations.y[I]), - observations.classes, -) - -LearnAPI.target( - learner::PerceptronClassifier, - observations::PerceptronClassifierObs, -) = observations.y - -LearnAPI.features( - learner::PerceptronClassifier, - observations::PerceptronClassifierObs, -) = observations.X - -# Note that data consumed by `predict` needs no pre-processing, so no need to overload -# `obs(model, data)`. - - -# ### Fitting and updating - -# For wrapping outcomes of learning: -struct PerceptronClassifierFitted - learner::PerceptronClassifier - perceptron # component array storing weights and bias - state # optimiser state - classes # target classes - losses -end - -LearnAPI.learner(model::PerceptronClassifierFitted) = model.learner - -# `fit` for pre-processed data (output of `obs(learner, data)`): -function LearnAPI.fit( - learner::PerceptronClassifier, - observations::PerceptronClassifierObs; - verbosity=1, - ) - - # unpack hyperparameters: - epochs = learner.epochs - optimiser = learner.optimiser - rng = deepcopy(learner.rng) # to prevent mutation of `learner`! - - # unpack data: - X = observations.X - y_hot = observations.y_hot - classes = observations.classes - nclasses = length(classes) - - # initialize bias and weights: - weights = randn(rng, Float32, nclasses, p) - bias = zeros(Float32, nclasses) - perceptron = (; weights, bias) |> ComponentArrays.ComponentArray - - # initialize optimiser: - state = Optimisers.setup(optimiser, perceptron) - - perceptron, state, losses = corefit(perceptron, X, y_hot, epochs, state, verbosity) - - return PerceptronClassifierFitted(learner, perceptron, state, classes, losses) -end - -# `fit` for unprocessed data: -LearnAPI.fit(learner::PerceptronClassifier, data; kwargs...) = - fit(learner, obs(learner, data); kwargs...) - -# see the `PerceptronClassifier` docstring for `update_observations` logic. -function LearnAPI.update_observations( - model::PerceptronClassifierFitted, - observations_new::PerceptronClassifierObs; - verbosity=1, - replacements..., - ) - - # unpack data: - X = observations_new.X - y_hot = observations_new.y_hot - classes = observations_new.classes - nclasses = length(classes) - - classes == model.classes || error("New training target has incompatible classes.") - - learner_old = LearnAPI.learner(model) - learner = LearnAPI.clone(learner_old; replacements...) - - perceptron = model.perceptron - state = model.state - losses = model.losses - epochs = learner.epochs - - perceptron, state, losses_new = corefit(perceptron, X, y_hot, epochs, state, verbosity) - losses = vcat(losses, losses_new) - - return PerceptronClassifierFitted(learner, perceptron, state, classes, losses) -end -LearnAPI.update_observations(model::PerceptronClassifierFitted, data; kwargs...) = - update_observations(model, obs(LearnAPI.learner(model), data); kwargs...) - -# see the `PerceptronClassifier` docstring for `update` logic. -function LearnAPI.update( - model::PerceptronClassifierFitted, - observations::PerceptronClassifierObs; - verbosity=1, - replacements..., - ) - - # unpack data: - X = observations.X - y_hot = observations.y_hot - classes = observations.classes - nclasses = length(classes) - - classes == model.classes || error("New training target has incompatible classes.") - - learner_old = LearnAPI.learner(model) - learner = LearnAPI.clone(learner_old; replacements...) - :epochs in keys(replacements) || return fit(learner, observations) - - perceptron = model.perceptron - state = model.state - losses = model.losses - - epochs = learner.epochs - Δepochs = epochs - learner_old.epochs - epochs < 0 && return fit(model, learner) - - perceptron, state, losses_new = corefit(perceptron, X, y_hot, Δepochs, state, verbosity) - losses = vcat(losses, losses_new) - - return PerceptronClassifierFitted(learner, perceptron, state, classes, losses) -end -LearnAPI.update(model::PerceptronClassifierFitted, data; kwargs...) = - update(model, obs(LearnAPI.learner(model), data); kwargs...) - - -# ### Predict - -function LearnAPI.predict(model::PerceptronClassifierFitted, ::Distribution, Xnew) - perceptron = model.perceptron - classes = model.classes - probs = perceptron.weights*Xnew .+ perceptron.bias |> NNlib.softmax - return CategoricalDistributions.UnivariateFinite(classes, probs') -end - -LearnAPI.predict(model::PerceptronClassifierFitted, ::Point, Xnew) = - mode.(predict(model, Distribution(), Xnew)) - - -# ### Accessor functions - -LearnAPI.training_losses(model::PerceptronClassifierFitted) = model.losses - - -# ### Traits - -@trait( - PerceptronClassifier, - constructor = PerceptronClassifier, - iteration_parameter = :epochs, - kinds_of_proxy = (Distribution(), Point()), - tags = ("classification", "iterative algorithms", "incremental algorithms"), - functions = ( - :(LearnAPI.fit), - :(LearnAPI.learner), - :(LearnAPI.strip), - :(LearnAPI.obs), - :(LearnAPI.features), - :(LearnAPI.target), - :(LearnAPI.update), - :(LearnAPI.update_observations), - :(LearnAPI.predict), - :(LearnAPI.training_losses), - ) -) - - -# ### Convenience methods - -LearnAPI.fit(learner::PerceptronClassifier, X, y; kwargs...) = - fit(learner, (X, y); kwargs...) -LearnAPI.update_observations(learner::PerceptronClassifier, X, y; kwargs...) = - update_observations(learner, (X, y); kwargs...) -LearnAPI.update(learner::PerceptronClassifier, X, y; kwargs...) = - update(learner, (X, y); kwargs...) - - -# ## Tests - -# synthetic test data: -N = 10 -n = 10N # number of observations -p = 2 # number of features -train = 1:6N -test = (6N+1:10N) -rng = StableRNG(123) -X = randn(rng, Float32, p, n); -coefficients = rand(rng, Float32, p)' -y_continuous = coefficients*X |> vec -η1 = quantile(y_continuous, 1/3) -η2 = quantile(y_continuous, 2/3) -y = map(y_continuous) do η - η < η1 && return "A" - η < η2 && return "B" - "C" -end |> CategoricalDistributions.categorical; -Xtrain = X[:, train]; -Xtest = X[:, test]; -ytrain = y[train]; -ytest = y[test]; - -@testset "PerceptronClassfier" begin - rng = StableRNG(123) - learner = PerceptronClassifier(; optimiser=Optimisers.Adam(0.01), epochs=40, rng) - @test LearnAPI.clone(learner) == learner - @test :(LearnAPI.update) in LearnAPI.functions(learner) - @test LearnAPI.target(learner, (X, y)) == y - @test LearnAPI.features(learner, (X, y)) == X - - model40 = fit(learner, Xtrain, ytrain; verbosity=0) - - # 40 epochs is sufficient for 90% accuracy in this case: - @test sum(predict(model40, Point(), Xtest) .== ytest)/length(ytest) > 0.9 - - # get probabilistic predictions: - ŷ40 = predict(model40, Distribution(), Xtest); - @test predict(model40, Xtest) ≈ ŷ40 - - # add 30 epochs in an `update`: - model70 = update(model40, Xtrain, y[train]; verbosity=0, epochs=70) - ŷ70 = predict(model70, Xtest); - @test !(ŷ70 ≈ ŷ40) - - # compare with cold restart: - model = fit(LearnAPI.clone(learner; epochs=70), Xtrain, y[train]; verbosity=0); - @test ŷ70 ≈ predict(model, Xtest) - - # instead add 30 epochs using `update_observations` instead: - model70b = update_observations(model40, Xtrain, y[train]; verbosity=0, epochs=30) - @test ŷ70 ≈ predict(model70b, Xtest) ≈ predict(model, Xtest) -end - -true diff --git a/test/patterns/incremental_algorithms.jl b/test/patterns/incremental_algorithms.jl deleted file mode 100644 index 20b01779..00000000 --- a/test/patterns/incremental_algorithms.jl +++ /dev/null @@ -1,135 +0,0 @@ -using LearnAPI -using Statistics -using StableRNGs - -import Distributions - -# # NORMAL DENSITY ESTIMATOR - -# An example of density estimation and also of incremental learning -# (`update_observations`). - - -# ## Implementation - -""" - NormalEstimator() - -Instantiate a learner for finding the maximum likelihood normal distribution fitting -some real univariate data `y`. Estimates can be updated with new data. - -```julia -model = fit(NormalEstimator(), y) -d = predict(model) # returns the learned `Normal` distribution -``` - -While the above is equivalent to the single operation `d = -predict(NormalEstimator(), y)`, the above workflow allows for the presentation of -additional observations post facto: The following is equivalent to `d2 = -predict(NormalEstimator(), vcat(y, ynew))`: - -```julia -update_observations(model, ynew) -d2 = predict(model) -``` - -Inspect all learned parameters with `LearnAPI.extras(model)`. Predict a 95% -confidence interval with `predict(model, ConfidenceInterval())` - -""" -struct NormalEstimator end - -struct NormalEstimatorFitted{T} - Σy::T - ȳ::T - ss::T # sum of squared residuals - n::Int -end - -LearnAPI.learner(::NormalEstimatorFitted) = NormalEstimator() - -function LearnAPI.fit(::NormalEstimator, y) - n = length(y) - Σy = sum(y) - ȳ = Σy/n - ss = sum(x->x^2, y) - n*ȳ^2 - return NormalEstimatorFitted(Σy, ȳ, ss, n) -end - -function LearnAPI.update_observations(model::NormalEstimatorFitted, ynew) - m = length(ynew) - n = model.n + m - Σynew = sum(ynew) - Σy = model.Σy + Σynew - ȳ = Σy/n - δ = model.n*((m*model.ȳ - Σynew)/n)^2 - ss = model.ss + δ + sum(x -> (x - ȳ)^2, ynew) - return NormalEstimatorFitted(Σy, ȳ, ss, n) -end - -LearnAPI.features(::NormalEstimator, y) = nothing -LearnAPI.target(::NormalEstimator, y) = y - -LearnAPI.predict(model::NormalEstimatorFitted, ::SingleDistribution) = - Distributions.Normal(model.ȳ, sqrt(model.ss/model.n)) -LearnAPI.predict(model::NormalEstimatorFitted, ::Point) = model.ȳ -function LearnAPI.predict(model::NormalEstimatorFitted, ::ConfidenceInterval) - d = predict(model, SingleDistribution()) - return (quantile(d, 0.025), quantile(d, 0.975)) -end - -# for fit and predict in one line: -LearnAPI.predict(::NormalEstimator, k::LearnAPI.KindOfProxy, y) = - predict(fit(NormalEstimator(), y), k) -LearnAPI.predict(::NormalEstimator, y) = predict(NormalEstimator(), SingleDistribution(), y) - -LearnAPI.extras(model::NormalEstimatorFitted) = (μ=model.ȳ, σ=sqrt(model.ss/model.n)) - -@trait( - NormalEstimator, - constructor = NormalEstimator, - kinds_of_proxy = (SingleDistribution(), Point(), ConfidenceInterval()), - tags = ("density estimation", "incremental algorithms"), - is_pure_julia = true, - human_name = "normal distribution estimator", - functions = ( - :(LearnAPI.fit), - :(LearnAPI.learner), - :(LearnAPI.strip), - :(LearnAPI.obs), - :(LearnAPI.features), - :(LearnAPI.target), - :(LearnAPI.predict), - :(LearnAPI.update_observations), - :(LearnAPI.extras), - ), -) - -# ## Tests - -@testset "NormalEstimator" begin - rng = StableRNG(123) - y = rand(rng, 50); - ynew = rand(rng, 10); - learner = NormalEstimator() - model = fit(learner, y) - d = predict(model) - μ, σ = Distributions.params(d) - @test μ ≈ mean(y) - @test σ ≈ std(y)*sqrt(49/50) # `std` uses Bessel's correction - - # accessor function: - @test LearnAPI.extras(model) == (; μ, σ) - - # one-liner: - @test predict(learner, y) == d - @test predict(learner, Point(), y) ≈ μ - @test predict(learner, ConfidenceInterval(), y)[1] ≈ quantile(d, 0.025) - - # updating: - model = update_observations(model, ynew) - μ2, σ2 = LearnAPI.extras(model) - μ3, σ3 = LearnAPI.extras(fit(learner, vcat(y, ynew))) # training ab initio - @test μ2 ≈ μ3 - @test σ2 ≈ σ3 -end diff --git a/test/patterns/regression.jl b/test/patterns/regression.jl deleted file mode 100644 index f7d8d073..00000000 --- a/test/patterns/regression.jl +++ /dev/null @@ -1,290 +0,0 @@ -using LearnAPI -using LinearAlgebra -using Tables -import MLUtils -import DataFrames - - -# # NAIVE RIDGE REGRESSION WITH NO INTERCEPTS - -# We overload `obs` to expose internal representation of data. See later for a simpler -# variation using the `obs` fallback. - - -# ## Implementation - -# no docstring here - that goes with the constructor -struct Ridge - lambda::Float64 -end - -""" - Ridge(; lambda=0.1) - -Instantiate a ridge regression learner, with regularization of `lambda`. - -""" -Ridge(; lambda=0.1) = Ridge(lambda) # LearnAPI.constructor defined later - -struct RidgeFitObs{T,M<:AbstractMatrix{T}} - A::M # p x n - names::Vector{Symbol} - y::Vector{T} -end - -struct RidgeFitted{T,F} - learner::Ridge - coefficients::Vector{T} - feature_importances::F -end - -LearnAPI.learner(model::RidgeFitted) = model.learner - -Base.getindex(data::RidgeFitObs, I) = - RidgeFitObs(data.A[:,I], data.names, data.y[I]) -Base.length(data::RidgeFitObs) = length(data.y) - -# observations for consumption by `fit`: -function LearnAPI.obs(::Ridge, data) - X, y = data - table = Tables.columntable(X) - names = Tables.columnnames(table) |> collect - RidgeFitObs(Tables.matrix(table)', names, y) -end - -# for observations: -function LearnAPI.fit(learner::Ridge, observations::RidgeFitObs; verbosity=1) - - # unpack hyperparameters and data: - lambda = learner.lambda - A = observations.A - names = observations.names - y = observations.y - - # apply core learner: - coefficients = (A*A' + learner.lambda*I)\(A*y) # 1 x p matrix - - # determine crude feature importances: - feature_importances = - [names[j] => abs(coefficients[j]) for j in eachindex(names)] - sort!(feature_importances, by=last) |> reverse! - - # make some noise, if allowed: - verbosity > 0 && - @info "Features in order of importance: $(first.(feature_importances))" - - return RidgeFitted(learner, coefficients, feature_importances) - -end - -# for unprocessed `data = (X, y)`: -LearnAPI.fit(learner::Ridge, data; kwargs...) = - fit(learner, obs(learner, data); kwargs...) - -# extracting stuff from training data: -LearnAPI.target(::Ridge, data) = last(data) -LearnAPI.target(::Ridge, observations::RidgeFitObs) = observations.y -LearnAPI.features(::Ridge, observations::RidgeFitObs) = observations.A - -# observations for consumption by `predict`: -LearnAPI.obs(::RidgeFitted, X) = Tables.matrix(X)' - -# matrix input: -LearnAPI.predict(model::RidgeFitted, ::Point, observations::AbstractMatrix) = - observations'*model.coefficients - -# tabular input: -LearnAPI.predict(model::RidgeFitted, ::Point, Xnew) = - predict(model, Point(), obs(model, Xnew)) - -# accessor function: -LearnAPI.feature_importances(model::RidgeFitted) = model.feature_importances - -LearnAPI.strip(model::RidgeFitted) = - RidgeFitted(model.learner, model.coefficients, nothing) - -@trait( - Ridge, - constructor = Ridge, - kinds_of_proxy = (Point(),), - tags = ("regression",), - functions = ( - :(LearnAPI.fit), - :(LearnAPI.learner), - :(LearnAPI.strip), - :(LearnAPI.obs), - :(LearnAPI.features), - :(LearnAPI.target), - :(LearnAPI.predict), - :(LearnAPI.feature_importances), - ) -) - -# convenience method: -LearnAPI.fit(learner::Ridge, X, y; kwargs...) = - fit(learner, (X, y); kwargs...) - - -# ## Tests - -# synthetic test data: -n = 30 # number of observations -train = 1:6 -test = 7:10 -a, b, c = rand(n), rand(n), rand(n) -X = (; a, b, c) -X = DataFrames.DataFrame(X) -y = 2a - b + 3c + 0.05*rand(n) -data = (X, y) - -@testset "test an implementation of ridge regression" begin - learner = Ridge(lambda=0.5) - @test :(LearnAPI.obs) in LearnAPI.functions(learner) - - @test LearnAPI.target(learner, data) == y - @test LearnAPI.features(learner, data) == X - - # verbose fitting: - @test_logs( - (:info, r"Feature"), - fit( - learner, - Tables.subset(X, train), - y[train]; - verbosity=1, - ), - ) - - # quiet fitting: - model = @test_logs( - fit( - learner, - Tables.subset(X, train), - y[train]; - verbosity=0, - ), - ) - - ŷ = predict(model, Point(), Tables.subset(X, test)) - @test ŷ isa Vector{Float64} - @test predict(model, Tables.subset(X, test)) == ŷ - - fitobs = LearnAPI.obs(learner, data) - predictobs = LearnAPI.obs(model, X) - model = fit(learner, MLUtils.getobs(fitobs, train); verbosity=0) - @test LearnAPI.target(learner, fitobs) == y - @test predict(model, Point(), MLUtils.getobs(predictobs, test)) ≈ ŷ - @test predict(model, LearnAPI.features(learner, fitobs)) ≈ predict(model, X) - - @test LearnAPI.feature_importances(model) isa Vector{<:Pair{Symbol}} - - filename = tempname() - using Serialization - small_model = LearnAPI.strip(model) - serialize(filename, small_model) - - recovered_model = deserialize(filename) - @test LearnAPI.learner(recovered_model) == learner - @test predict( - recovered_model, - Point(), - MLUtils.getobs(predictobs, test) - ) ≈ ŷ - -end - -# # VARIATION OF RIDGE REGRESSION THAT USES FALLBACK OF LearnAPI.obs - -# no docstring here - that goes with the constructor -struct BabyRidge - lambda::Float64 -end - - -# ## Implementation - -""" - BabyRidge(; lambda=0.1) - -Instantiate a ridge regression learner, with regularization of `lambda`. - -""" -BabyRidge(; lambda=0.1) = BabyRidge(lambda) # LearnAPI.constructor defined later - -struct BabyRidgeFitted{T,F} - learner::BabyRidge - coefficients::Vector{T} - feature_importances::F -end - -function LearnAPI.fit(learner::BabyRidge, data; verbosity=1) - - X, y = data - - lambda = learner.lambda - table = Tables.columntable(X) - names = Tables.columnnames(table) |> collect - A = Tables.matrix(table)' - - # apply core learner: - coefficients = (A*A' + learner.lambda*I)\(A*y) # vector - - feature_importances = nothing - - return BabyRidgeFitted(learner, coefficients, feature_importances) - -end - -# extracting stuff from training data: -LearnAPI.target(::BabyRidge, data) = last(data) - -LearnAPI.learner(model::BabyRidgeFitted) = model.learner - -LearnAPI.predict(model::BabyRidgeFitted, ::Point, Xnew) = - Tables.matrix(Xnew)*model.coefficients - -LearnAPI.strip(model::BabyRidgeFitted) = - BabyRidgeFitted(model.learner, model.coefficients, nothing) - -@trait( - BabyRidge, - constructor = BabyRidge, - kinds_of_proxy = (Point(),), - tags = ("regression",), - functions = ( - :(LearnAPI.fit), - :(LearnAPI.learner), - :(LearnAPI.strip), - :(LearnAPI.obs), - :(LearnAPI.features), - :(LearnAPI.target), - :(LearnAPI.predict), - :(LearnAPI.feature_importances), - ) -) - -# convenience method: -LearnAPI.fit(learner::BabyRidge, X, y; kwargs...) = - fit(learner, (X, y); kwargs...) - - -# ## Tests - -@testset "test a variation which does not overload LearnAPI.obs" begin - learner = BabyRidge(lambda=0.5) - - model = fit(learner, Tables.subset(X, train), y[train]; verbosity=0) - ŷ = predict(model, Point(), Tables.subset(X, test)) - @test ŷ isa Vector{Float64} - - fitobs = obs(learner, data) - predictobs = LearnAPI.obs(model, X) - model = fit(learner, MLUtils.getobs(fitobs, train); verbosity=0) - @test predict(model, Point(), MLUtils.getobs(predictobs, test)) == ŷ == - predict(model, MLUtils.getobs(predictobs, test)) - @test LearnAPI.target(learner, data) == y - @test LearnAPI.predict(model, X) ≈ - LearnAPI.predict(model, LearnAPI.features(learner, data)) -end - -true diff --git a/test/patterns/static_algorithms.jl b/test/patterns/static_algorithms.jl deleted file mode 100644 index 243cab44..00000000 --- a/test/patterns/static_algorithms.jl +++ /dev/null @@ -1,147 +0,0 @@ -using LearnAPI -using Tables -import MLUtils -import DataFrames - - -# # TRANSFORMER TO SELECT SOME FEATURES (COLUMNS) OF A TABLE - -# See later for a variation that stores the names of rejected features in the model -# object, for inspection by an accessor function. - -struct Selector - names::Vector{Symbol} -end -Selector(; names=Symbol[]) = Selector(names) # LearnAPI.constructor defined later - -# `fit` consumes no observational data, does no "learning", and just returns a thinly -# wrapped `learner` (to distinguish it from the learner in dispatch): -LearnAPI.fit(learner::Selector; verbosity=1) = Ref(learner) -LearnAPI.learner(model) = model[] - -function LearnAPI.transform(model::Base.RefValue{Selector}, X) - learner = LearnAPI.learner(model) - table = Tables.columntable(X) - names = Tables.columnnames(table) - filtered_names = filter(in(learner.names), names) - filtered_columns = (Tables.getcolumn(table, name) for name in filtered_names) - filtered_table = NamedTuple{filtered_names}((filtered_columns...,)) - return Tables.materializer(X)(filtered_table) -end - -# fit and transform in one go: -function LearnAPI.transform(learner::Selector, X) - model = fit(learner) - transform(model, X) -end - -# note the necessity of overloading `is_static` (`fit` consumes no data): -@trait( - Selector, - constructor = Selector, - tags = ("feature engineering",), - is_static = true, - functions = ( - :(LearnAPI.fit), - :(LearnAPI.learner), - :(LearnAPI.strip), - :(LearnAPI.obs), - :(LearnAPI.transform), - ), -) - -@testset "test a static transformer" begin - learner = Selector(names=[:x, :w]) - X = DataFrames.DataFrame(rand(3, 4), [:x, :y, :z, :w]) - model = fit(learner) # no data arguments! - # if provided, data is ignored: - @test LearnAPI.learner(model) == learner - W = transform(model, X) - @test W == DataFrames.DataFrame(Tables.matrix(X)[:,[1,4]], [:x, :w]) - @test W == transform(learner, X) -end - - -# # FEATURE SELECTOR THAT REPORTS BYPRODUCTS OF SELECTION PROCESS - -# This a variation of `Selector` above that stores the names of rejected features in the -# output of `fit`, for inspection by an accessor function called `rejected`. - -struct FancySelector - names::Vector{Symbol} -end - -""" - FancySelector(; names=Symbol[]) - -Instantiate a feature selector that exposes the names of rejected features. - -```julia -learner = FancySelector(names=[:x, :w]) -X = DataFrames.DataFrame(rand(3, 4), [:x, :y, :z, :w]) -model = fit(learner) # no data arguments! -transform(model, X) # mutates `model` -@assert rejected(model) == [:y, :z] -``` - -""" -FancySelector(; names=Symbol[]) = FancySelector(names) - -mutable struct FancySelectorFitted - learner::FancySelector - rejected::Vector{Symbol} - FancySelectorFitted(learner) = new(learner) -end -LearnAPI.learner(model::FancySelectorFitted) = model.learner -rejected(model::FancySelectorFitted) = model.rejected - -# Here we are wrapping `learner` with a place-holder for the `rejected` feature names. -LearnAPI.fit(learner::FancySelector; verbosity=1) = FancySelectorFitted(learner) - -# output the filtered table and add `rejected` field to model (mutatated!) -function LearnAPI.transform(model::FancySelectorFitted, X) - table = Tables.columntable(X) - names = Tables.columnnames(table) - keep = LearnAPI.learner(model).names - filtered_names = filter(in(keep), names) - model.rejected = setdiff(names, filtered_names) - filtered_columns = (Tables.getcolumn(table, name) for name in filtered_names) - filtered_table = NamedTuple{filtered_names}((filtered_columns...,)) - return Tables.materializer(X)(filtered_table) -end - -# fit and transform in one step: -function LearnAPI.transform(learner::FancySelector, X) - model = fit(learner) - transform(model, X) -end - -# note the necessity of overloading `is_static` (`fit` consumes no data): -@trait( - FancySelector, - constructor = FancySelector, - is_static = true, - tags = ("feature engineering",), - functions = ( - :(LearnAPI.fit), - :(LearnAPI.learner), - :(LearnAPI.strip), - :(LearnAPI.obs), - :(LearnAPI.transform), - :(MyPkg.rejected), # accessor function not owned by LearnAPI.jl, - ) -) - -@testset "test a variation that reports byproducts" begin - learner = FancySelector(names=[:x, :w]) - X = DataFrames.DataFrame(rand(3, 4), [:x, :y, :z, :w]) - model = fit(learner) # no data arguments! - @test !isdefined(model, :reject) - @test LearnAPI.learner(model) == learner - filtered = DataFrames.DataFrame(Tables.matrix(X)[:,[1,4]], [:x, :w]) - @test transform(model, X) == filtered - @test transform(learner, X) == filtered - @test rejected(model) == [:y, :z] -end - -true