diff --git a/.github/codecov.yml b/.github/codecov.yml new file mode 100644 index 00000000..c62bedf5 --- /dev/null +++ b/.github/codecov.yml @@ -0,0 +1,9 @@ +coverage: + status: + project: + default: + threshold: 0.5% + removed_code_behavior: fully_covered_patch + patch: + default: + target: 80 diff --git a/.github/workflows/SpellCheck.yml b/.github/workflows/SpellCheck.yml new file mode 100644 index 00000000..3d62423c --- /dev/null +++ b/.github/workflows/SpellCheck.yml @@ -0,0 +1,13 @@ +name: Spell Check + +on: [pull_request] + +jobs: + typos-check: + name: Spell Check with Typos + runs-on: ubuntu-latest + steps: + - name: Checkout Actions Repository + uses: actions/checkout@v4 + - name: Check spelling + uses: crate-ci/typos@master \ No newline at end of file diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 37ef5474..d71082e6 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -17,7 +17,7 @@ jobs: fail-fast: false matrix: version: - - '1.6' + - '1.10' # LTS release - '1' # automatically expands to the latest stable 1.x release of Julia. os: - ubuntu-latest @@ -44,9 +44,11 @@ jobs: env: JULIA_NUM_THREADS: 2 - uses: julia-actions/julia-processcoverage@v1 - - uses: codecov/codecov-action@v1 + - uses: codecov/codecov-action@v4 with: - file: lcov.info + token: ${{ secrets.CODECOV_TOKEN }} + fail_ci_if_error: false + verbose: true docs: name: Documentation runs-on: ubuntu-latest @@ -65,4 +67,4 @@ jobs: using Documenter: DocMeta, doctest using LearnAPI DocMeta.setdocmeta!(LearnAPI, :DocTestSetup, :(using LearnAPI); recursive=true) - doctest(LearnAPI)' \ No newline at end of file + doctest(LearnAPI)' diff --git a/LICENSE b/LICENSE index 7609ebe2..4690371a 100644 --- a/LICENSE +++ b/LICENSE @@ -1,6 +1,6 @@ MIT License -MIT License Copyright (c) 2021 - JuliaAI +MIT License Copyright (c) 2024 - Anthony Blaom Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/Project.toml b/Project.toml index 61029ea6..2172f4e7 100644 --- a/Project.toml +++ b/Project.toml @@ -3,16 +3,11 @@ uuid = "92ad9a40-7767-427a-9ee6-6e577f1266cb" authors = ["Anthony D. Blaom "] version = "0.1.0" -[deps] -InteractiveUtils = "b77e0a4c-d291-57a0-90e8-8db25a27a240" -Statistics = "10745b16-79ce-11e8-11f9-7d13ad32a3b2" - [compat] -julia = "1.6" +julia = "1.10" [extras] -SparseArrays = "2f01184e-e22b-5df5-ae63-d93ebab69eaf" Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" [targets] -test = ["SparseArrays", "Test"] +test = ["Test",] diff --git a/README.md b/README.md index af565313..7ef89a62 100644 --- a/README.md +++ b/README.md @@ -2,27 +2,53 @@ A base Julia interface for machine learning and statistics +[![Lifecycle:Maturing](https://img.shields.io/badge/Lifecycle-Maturing-007EC6)](ROADMAP.md) +[![Build Status](https://github.com/JuliaAI/LearnAPI.jl/workflows/CI/badge.svg)](https://github.com/JuliaAI/LearnAPI.jl/actions) +[![codecov](https://codecov.io/gh/JuliaAI/LearnAPI.jl/graph/badge.svg?token=9IWT9KYINZ)](https://codecov.io/gh/JuliaAI/LearnAPI.jl?branch=dev) +[![Docs](https://img.shields.io/badge/docs-dev-blue.svg)](https://juliaai.github.io/LearnAPI.jl/dev/) -**Devlopement Status:** +Comprehensive documentation is [here](https://juliaai.github.io/LearnAPI.jl/dev/). -- [X] Detailed proposal stage ([this - documentation](https://juliaai.github.io/LearnAPI.jl/dev/)). -- [ ] Initial feedback stage (opened mid-January, 2023). General feedback can be provided at [this Julia Discourse thread](https://discourse.julialang.org/t/ann-learnapi-jl-proposal-for-a-basement-level-machine-learning-api/93048/20). -- [ ] Implement feedback and finish "To do" list (below) -- [ ] Proof of concept implementation -- [ ] Polish -- [ ] **Register 0.2.0** +New contributions welcome. See the [road map](ROADMAP.md). -You can join a discussion on the LearnAPI proposal at [this](https://discourse.julialang.org/t/ann-learnapi-jl-proposal-for-a-basement-level-machine-learning-api/93048) Julia Discourse thread. +## Code snippet -To do: +Configure a machine learning algorithm: -- [ ] Add methods to create/save persistent representation of learned parameters -- [ ] Add more repo tests -- [ ] Add methods to test an implementation -- [ ] Add user guide ("Common Implementation Patterns" section of manual) +```julia +julia> ridge = Ridge(lambda=0.1) +``` -[![Build Status](https://github.com/JuliaAI/LearnAPI.jl/workflows/CI/badge.svg)](https://github.com/JuliaAI/LearnAPI.jl/actions) -[![Coverage](https://codecov.io/gh/JuliaAI/LearnAPI.jl/branch/master/graph/badge.svg)](https://codecov.io/github/JuliaAI/LearnAPI.jl?branch=master) -[![Docs](https://img.shields.io/badge/docs-dev-blue.svg)](https://juliaai.github.io/LearnAPI.jl/dev/) +Inspect available functionality: + +``` +julia> @functions ridge +(fit, LearnAPI.learner, LearnAPI.strip, obs, LearnAPI.features, LearnAPI.target, predict, LearnAPI.coefficients) +``` + +Train: + +```julia +julia> model = fit(ridge, data) +``` + +Predict: + +```julia +julia> predict(model, data)[1] +"virginica" +``` + +Predict a probability distribution ([proxy](https://juliaai.github.io/LearnAPI.jl/dev/kinds_of_target_proxy/#proxy_types) for the target): + +```julia +julia> predict(model, Distribution(), data)[1] +UnivariateFinite{Multiclass{3}}(setosa=>0.0, versicolor=>0.25, virginica=>0.75) +``` + +## Credits + +Created by Anthony Blaom, in cooperation with Cameron Bieganek and other [members of the +Julia +community](https://discourse.julialang.org/t/ann-learnapi-jl-proposal-for-a-basement-level-machine-learning-api/93048). diff --git a/ROADMAP.md b/ROADMAP.md new file mode 100644 index 00000000..7da578ae --- /dev/null +++ b/ROADMAP.md @@ -0,0 +1,47 @@ +# Road map + +- [ ] Mock up a challenging `update` use-case: controlling an iterative algorithm that + wants, for efficiency, to internally compute the out-of-sample predictions that will + be for used to *externally* determined early stopping cc: @jeremiedb + +- [ ] Get code coverage to 100% (see next item) + +- [ ] Add to this repo or a utility repo methods to test a valid implementation of + LearnAPI.jl + +- [ ] Flush out "Common Implementation Patterns". The current plan is to mock up example + implementations, and add them as LearnAPI.jl tests, with links to the test file from + "Common Implementation Patterns". As real-world implementations roll out, we could + increasingly point to those instead, to conserve effort + - [x] regression + - [ ] classification + - [ ] clustering + - [x] gradient descent + - [x] iterative algorithms + - [ ] incremental algorithms + - [ ] dimension reduction + - [x] feature engineering + - [x] static algorithms + - [ ] missing value imputation + - [ ] transformers + - [x] ensemble algorithms + - [ ] time series forecasting + - [ ] time series classification + - [ ] survival analysis + - [ ] density estimation + - [ ] Bayesian algorithms + - [ ] outlier detection + - [ ] collaborative filtering + - [ ] text analysis + - [ ] audio analysis + - [ ] natural language processing + - [ ] image processing + - [ ] meta-algorithms + +- [ ] In a utility package provide: + - [ ] Methods to facilitate common-use case data interfaces: support simultaneously + `fit` data of the form `data = (X, y)` where `X` is table *or* matrix, and `data` a + table with target specified by hyperparameter; here `obs` will return a thin wrapping + of the matrix of `X`, the target `y`, and the names of all fields. We can have + options to make `X` a concrete array or an adjoint, depending on what is more + efficient for the algorithm. diff --git a/docs/Project.toml b/docs/Project.toml index e08cc4f7..4d8ef094 100644 --- a/docs/Project.toml +++ b/docs/Project.toml @@ -1,8 +1,11 @@ [deps] Documenter = "e30172f5-a6a5-5a46-863b-614d45cd2de4" +DocumenterInterLinks = "d12716ef-a0f6-4df4-a9f1-a5a34e75c656" +LearnAPI = "92ad9a40-7767-427a-9ee6-6e577f1266cb" +MLUtils = "f1d291b0-491e-4a28-83b9-f70985020b54" ScientificTypesBase = "30f210dd-8aff-4c5f-94ba-8e64358c1161" Tables = "bd369af6-aec1-5ad0-b16a-f7cc5008161c" [compat] -Documenter = "^0.27" -julia = "1" +Documenter = "1" +julia = "1.10" diff --git a/docs/make.jl b/docs/make.jl index cfd9356c..158117cd 100644 --- a/docs/make.jl +++ b/docs/make.jl @@ -1,31 +1,39 @@ using Documenter using LearnAPI using ScientificTypesBase +using DocumenterInterLinks -const REPO="github.com/JuliaAI/LearnAPI.jl" +const REPO = Remotes.GitHub("JuliaAI", "LearnAPI.jl") -makedocs(; +makedocs( modules=[LearnAPI,], - format=Documenter.HTML(prettyurls = get(ENV, "CI", nothing) == "true"), + format=Documenter.HTML( + prettyurls = true,#get(ENV, "CI", nothing) == "true", + collapselevel = 1, + ), pages=[ - "Overview" => "index.md", - "Goals and Approach" => "goals_and_approach.md", + "Home" => "index.md", "Anatomy of an Implementation" => "anatomy_of_an_implementation.md", - "Reference" => "reference.md", - "Fit, update and ingest" => "fit_update_and_ingest.md", - "Predict and other operations" => "operations.md", - "Accessor Functions" => "accessor_functions.md", - "Optional Data Interface" => "optional_data_interface.md", - "Algorithm Traits" => "algorithm_traits.md", + "Reference" => [ + "Overview" => "reference.md", + "fit/update" => "fit_update.md", + "predict/transform" => "predict_transform.md", + "Kinds of Target Proxy" => "kinds_of_target_proxy.md", + "obs and Data Interfaces" => "obs.md", + "target/weights/features" => "target_weights_features.md", + "Accessor Functions" => "accessor_functions.md", + "Learner Traits" => "traits.md", + ], "Common Implementation Patterns" => "common_implementation_patterns.md", "Testing an Implementation" => "testing_an_implementation.md", ], - repo="https://$REPO/blob/{commit}{path}#L{line}", - sitename="LearnAPI.jl" + sitename="LearnAPI.jl", + warnonly = [:cross_references, :missing_docs], + repo = Remotes.GitHub("JuliaAI", "LearnAPI.jl"), ) deploydocs( - ; repo=REPO, devbranch="dev", push_preview=false, + repo="github.com/JuliaAI/LearnAPI.jl.git", ) diff --git a/docs/src/accessor_functions.md b/docs/src/accessor_functions.md index 13203c32..11ac67e0 100644 --- a/docs/src/accessor_functions.md +++ b/docs/src/accessor_functions.md @@ -1,16 +1,53 @@ -# Accessor Functions +# [Accessor Functions](@id accessor_functions) -> **Summary.** While byproducts of training are ordinarily recorded in the `report` -> component of the output of `fit`/`update!`/`ingest!`, some families of algorithms report an -> item that is likely shared by multiple algorithm types, and it is useful to have common -> interface for accessing these directly. Training losses and feature importances are two -> examples. +The sole argument of an accessor function is the output, `model`, of +[`fit`](@ref). Learners are free to implement any number of these, or none of them. Only +`LearnAPI.strip` has a fallback, namely the identity. + +- [`LearnAPI.learner(model)`](@ref) +- [`LearnAPI.extras(model)`](@ref) +- [`LearnAPI.strip(model)`](@ref) +- [`LearnAPI.coefficients(model)`](@ref) +- [`LearnAPI.intercept(model)`](@ref) +- [`LearnAPI.tree(model)`](@ref) +- [`LearnAPI.trees(model)`](@ref) +- [`LearnAPI.feature_names(model)`](@ref) +- [`LearnAPI.feature_importances(model)`](@ref) +- [`LearnAPI.training_losses(model)`](@ref) +- [`LearnAPI.out_of_sample_losses(model)`](@ref) +- [`LearnAPI.predictions(model)`](@ref) +- [`LearnAPI.out_of_sample_indices(model)`](@ref) +- [`LearnAPI.training_scores(model)`](@ref) +- [`LearnAPI.components(model)`](@ref) + +Learner-specific accessor functions may also be implemented. The names of all accessor +functions are included in the list returned by [`LearnAPI.functions(learner)`](@ref). + +## Implementation guide + +All new implementations must implement [`LearnAPI.learner`](@ref). While, all others are +optional, any implemented accessor functions must be added to the list returned by +[`LearnAPI.functions`](@ref). + + +## Reference ```@docs +LearnAPI.learner +LearnAPI.extras +LearnAPI.strip +LearnAPI.coefficients +LearnAPI.intercept +LearnAPI.tree +LearnAPI.trees +LearnAPI.feature_names LearnAPI.feature_importances LearnAPI.training_losses +LearnAPI.out_of_sample_losses +LearnAPI.predictions +LearnAPI.out_of_sample_indices LearnAPI.training_scores -LearnAPI.training_labels +LearnAPI.components ``` diff --git a/docs/src/algorithm_traits.md b/docs/src/algorithm_traits.md deleted file mode 100644 index e2ccdb0f..00000000 --- a/docs/src/algorithm_traits.md +++ /dev/null @@ -1,139 +0,0 @@ -# Algorithm Traits - -> **Summary.** Traits allow one to promise particular behaviour for an algorithm, such as: -> *This algorithm supports per-observation weights, which must appear as the third -> argument of `fit`*, or *This algorithm's `transform` method predicts `Real` vectors*. - -Algorithm traits are functions whose first (and usually only) argument is an algorithm. In -a new implementation, a single-argument trait is declared following this pattern: - -```julia -LearnAPI.is_pure_julia(algorithm::MyAlgorithmType) = true -``` - -!!! important - - The value of a trait must be the same for all algorithms of the same type, - even if the types differ only in type parameters. There are exceptions for - some traits, if - `is_wrapper(algorithm) = true` for all instances `algorithm` of some type - (composite algorithms). This requirement occasionally requires that - an existing algorithm implementation be split into separate LearnAPI - implementations (e.g., one for regression and another for classification). - -The declaration above has the shorthand - -```julia -@trait MyAlgorithmType is_pure_julia=true -``` - -Multiple traits can be declared like this: - - -```julia -@trait( - MyAlgorithmType, - is_pure_julia = true, - pkg_name = "MyPackage", -) -``` - -### Special two-argument traits - -The two-argument version of [`LearnAPI.predict_output_scitype`](@ref) and -[`LearnAPI.predict_output_scitype`](@ref) are the only overloadable traits with more than -one argument. They cannot be declared using the `@trait` macro. - -## Trait summary - -**Overloadable traits** are available for overloading by any new LearnAPI -implementation. **Derived traits** are not, and should not be called by performance -critical code - -### Overloadable traits - -In the examples column of the table below, `Table`, `Continuous`, `Sampleable` are names owned by the -package [ScientificTypesBase.jl](https://github.com/JuliaAI/ScientificTypesBase.jl/). - -| trait | fallback value | return value | example | -|:-------------------------------------------------|:----------------------|:--------------|:--------| -| [`LearnAPI.functions`](@ref)`(algorithm)` | `()` | implemented LearnAPI functions (traits excluded) | `(:fit, :predict)` | -| [`LearnAPI.preferred_kind_of_proxy`](@ref)`(algorithm)` | `LearnAPI.None()` | an instance `tp` of `KindOfProxy` for which an implementation of `LearnAPI.predict(algorithm, tp, ...)` is guaranteed. | `LearnAPI.Distribution()` | -| [`LearnAPI.position_of_target`](@ref)`(algorithm)` | `0` | ¹ the positional index of the **target** in `data` in `fit(..., data...; metadata)` calls | 2 | -| [`LearnAPI.position_of_weights`](@ref)`(algorithm)` | `0` | ¹ the positional index of **per-observation weights** in `data` in `fit(..., data...; metadata)` | 3 | -| [`LearnAPI.descriptors`](@ref)`(algorithm)` | `()` | lists one or more suggestive algorithm descriptors from `LearnAPI.descriptors()` | (:classifier, :probabilistic) | -| [`LearnAPI.is_pure_julia`](@ref)`(algorithm)` | `false` | is `true` if implementation is 100% Julia code | `true` | -| [`LearnAPI.pkg_name`](@ref)`(algorithm)` | `"unknown"` | name of package providing core code (may be different from package providing LearnAPI.jl implementation) | `"DecisionTree"` | -| [`LearnAPI.pkg_license`](@ref)`(algorithm)` | `"unknown"` | name of license of package providing core code | `"MIT"` | -| [`LearnAPI.doc_url`](@ref)`(algorithm)` | `"unknown"` | url providing documentation of the core code | `"https://en.wikipedia.org/wiki/Decision_tree_learning"` | -| [`LearnAPI.load_path`](@ref)`(algorithm)` | `"unknown"` | a string indicating where the struct for `typeof(algorithm)` is defined, beginning with name of package providing implementation | `FastTrees.LearnAPI.DecisionTreeClassifier` | -| [`LearnAPI.is_wrapper`](@ref)`(algorithm)` | `false` | is `true` if one or more properties (fields) of `algorithm` may be an algorithm | `true` | -| [`LearnAPI.human_name`](@ref)`(algorithm)` | type name with spaces | human name for the algorithm; should be a noun | "elastic net regressor" | -| [`LearnAPI.iteration_parameter`](@ref)`(algorithm)` | `nothing` | symbolic name of an iteration parameter | :epochs | -| [`LearnAPI.fit_keywords`](@ref)`(algorithm)` | `()` | tuple of symbols for keyword arguments accepted by `fit` (corresponding to metadata) | `(:class_weights,)` | -| [`LearnAPI.fit_scitype`](@ref)`(algorithm)` | `Union{}` | upper bound on `scitype(data)` in `fit(algorithm, verbosity, data...)`² | `Tuple{Table(Continuous), AbstractVector{Continuous}}` | -| [`LearnAPI.fit_observation_scitype`](@ref)`(algorithm)` | `Union{}`| upper bound on `scitype(observation)` for `observation` in `data` and `data` in `fit(algorithm, verbosity, data...)`² | `Tuple{AbstractVector{Continuous}, Continuous}` | -| [`LearnAPI.fit_type`](@ref)`(algorithm)` | `Union{}` | upper bound on `type(data)` in `fit(algorithm, verbosity, data...)`² | `Tuple{AbstractMatrix{<:Real}, AbstractVector{<:Real}}` | -| [`LearnAPI.fit_observation_type`](@ref)`(algorithm)` | `Union{}`| upper bound on `type(observation)` for `observation` in `data` and `data` in `fit(algorithm, verbosity, data...)`* | `Tuple{AbstractVector{<:Real}, Real}` | -| [`LearnAPI.predict_input_scitype`](@ref)`(algorithm)` | `Union{}` | upper bound on `scitype(data)` in `predict(algorithm, fitted_params, data...)`² | `Table(Continuous)` | -| [`LearnAPI.predict_output_scitype`](@ref)`(algorithm, kind_of_proxy)` | `Any` | upper bound on `scitype(first(predict(algorithm, kind_of_proxy, ...)))` | `AbstractVector{Continuous}` | -| [`LearnAPI.predict_input_type`](@ref)`(algorithm)` | `Union{}` | upper bound on `typeof(data)` in `predict(algorithm, fitted_params, data...)`² | `AbstractMatrix{<:Real}` | -| [`LearnAPI.predict_output_type`](@ref)`(algorithm, kind_of_proxy)` | `Any` | upper bound on `typeof(first(predict(algorithm, kind_of_proxy, ...)))` | `AbstractVector{<:Real}` | -| [`LearnAPI.transform_input_scitype`](@ref)`(algorithm)` | `Union{}` | upper bound on `scitype(data)` in `transform(algorithm, fitted_params, data...)`² | `Table(Continuous)` | -| [`LearnAPI.transform_output_scitype`](@ref)`(algorithm)` | `Any` | upper bound on `scitype(first(transform(algorithm, ...)))` | `Table(Continuous)` | -| [`LearnAPI.transform_input_type`](@ref)`(algorithm)` | `Union{}` | upper bound on `typeof(data)` in `transform(algorithm, fitted_params, data...)`² | `AbstractMatrix{<:Real}}` | -| [`LearnAPI.transform_output_type`](@ref)`(algorithm)` | `Any` | upper bound on `typeof(first(transform(algorithm, ...)))` | `AbstractMatrix{<:Real}` | - -¹ If the value is `0`, then the variable in boldface type is not supported and not -expected to appear in `data`. If `length(data)` is less than the trait value, then `data` -is understood to exclude the variable, but note that `fit` can have multiple signatures of -varying lengths, as in `fit(algorithm, verbosity, X, y)` and `fit(algorithm, verbosity, X, y, -w)`. A non-zero value is a promise that `fit` includes a signature of sufficient length to -include the variable. - -² Assuming no [optional data interface](@ref data_interface) is implemented. See docstring -for the general case. - - -### Derived Traits - -The following convenience methods are provided but intended for overloading: - -| trait | return value | example | -|:-------------------------------------|:------------------------------------------|:-----------| -| `LearnAPI.name(algorithm)` | algorithm type name as string | "PCA" | -| `LearnAPI.is_algorithm(algorithm)` | `true` if `functions(algorithm)` is not empty | `true` | -| [`LearnAPI.predict_output_scitype`](@ref)(algorithm) | dictionary of upper bounds on the scitype of predictions, keyed on subtypes of [`LearnAPI.KindOfProxy`](@ref) | -| [`LearnAPI.predict_output_type`](@ref)(algorithm) | dictionary of upper bounds on the type of predictions, keyed on subtypes of [`LearnAPI.KindOfProxy`](@ref) | - - -## Reference - -```@docs -LearnAPI.functions -LearnAPI.preferred_kind_of_proxy -LearnAPI.position_of_target -LearnAPI.position_of_weights -LearnAPI.descriptors -LearnAPI.is_pure_julia -LearnAPI.pkg_name -LearnAPI.pkg_license -LearnAPI.doc_url -LearnAPI.load_path -LearnAPI.is_wrapper -LearnAPI.fit_keywords -LearnAPI.human_name -LearnAPI.iteration_parameter -LearnAPI.fit_scitype -LearnAPI.fit_type -LearnAPI.fit_observation_scitype -LearnAPI.fit_observation_type -LearnAPI.predict_input_scitype -LearnAPI.predict_output_scitype -LearnAPI.predict_input_type -LearnAPI.predict_output_type -LearnAPI.transform_input_scitype -LearnAPI.transform_output_scitype -LearnAPI.transform_input_type -LearnAPI.transform_output_type -``` diff --git a/docs/src/anatomy_of_an_implementation.md b/docs/src/anatomy_of_an_implementation.md index d0180f35..c977cf50 100644 --- a/docs/src/anatomy_of_an_implementation.md +++ b/docs/src/anatomy_of_an_implementation.md @@ -1,25 +1,53 @@ # Anatomy of an Implementation -> **Summary.** Formally, an **algorithm** is a container for the hyperparameters of some -> ML/statistics algorithm. A basic implementation of the ridge regressor requires -> implementing `fit` and `predict` methods dispatched on the algorithm type; `predict` is -> an example of an **operation**, the others are `transform` and `inverse_transform`. In -> this example we also implement an **accessor function**, called `feature_importance`, -> returning the absolute values of the linear coefficients. The ridge regressor has a -> target variable and outputs literal predictions of the target (rather than, say, -> probabilistic predictions); accordingly the overloaded `predict` method is dispatched on -> the `LiteralTarget` subtype of `KindOfProxy`. An **algorithm trait** declares this type -> as the preferred kind of target proxy. Other traits articulate the algorithm's training -> data type requirements and the input/output type of `predict`. +The core LearnAPI.jl pattern looks like this: -We begin by describing an implementation of LearnAPI.jl for basic ridge regression -(without intercept) to introduce the main actors in any implementation. +```julia +model = fit(learner, data) +predict(model, newdata) +``` + +Here `learner` specifies hyperparameters, while `model` stores learned parameters and any byproducts of algorithm execution. + +[Transformers](@ref) ordinarily implement `transform` instead of `predict`. For more on +`predict` versus `transform`, see [Predict or transform?](@ref) + +["Static" algorithms](@ref static_algorithms) have a `fit` that consumes no `data` +(instead `predict` or `transform` does the heavy lifting). In [density +estimation](@ref density_estimation), `predict` consumes no data. + +These are the basic possibilities. + +Elaborating on the core pattern above, we detail in this tutorial an implementation of the +LearnAPI.jl for naive [ridge regression](https://en.wikipedia.org/wiki/Ridge_regression) +with no intercept. The kind of workflow we want to enable has been previewed in [Sample +workflow](@ref). Readers can also refer to the [demonstration](@ref workflow) of the +implementation given later. + +!!! note + + New implementations of `fit`, `predict`, etc, + always have a *single* `data` argument as above. + For convenience, a signature such as `fit(learner, X, y)`, calling + `fit(learner, (X, y))`, can be added, but the LearnAPI.jl specification is + silent on the meaning or existence of signatures with extra arguments. +!!! note -## Defining an algorithm type + If the `data` object consumed by `fit`, `predict`, or `transform` is not + not a suitable table¹, array³, tuple of tables and arrays, or some + other object implementing + the [MLUtils.jl](https://juliaml.github.io/MLUtils.jl/dev/) + `getobs`/`numobs` interface, + then an implementation must: (i) overload [`obs`](@ref) to articulate how + provided data can be transformed into a form that does support + this interface, as illustrated below under + [Providing a separate data front end](@ref); or (ii) overload the trait + [`LearnAPI.data_interface`](@ref) to specify a more relaxed data + API. The first line below imports the lightweight package LearnAPI.jl whose methods we will be -extending, the second, libraries needed for the core algorithm. +extending. The second imports libraries needed for the core algorithm. ```@example anatomy using LearnAPI @@ -27,293 +55,531 @@ using LinearAlgebra, Tables nothing # hide ``` -Next, we define a struct to store the single hyperparameter `lambda` of this algorithm: +## Defining learners + +Here's a new type whose instances specify the single ridge regression hyperparameter: ```@example anatomy -struct MyRidge <: LearnAPI.Algorithm - lambda::Float64 +struct Ridge{T<:Real} + lambda::T end nothing # hide ``` -The subtyping `MyRidge <: LearnAPI.Algorithm` is optional but recommended where it is not -otherwise disruptive. - -Instances of `MyRidge` are called **algorithms** and `MyRidge` is an **algorithm type**. +Instances of `Ridge` are *[learners](@ref learners)*, in LearnAPI.jl parlance. -A keyword argument constructor providing defaults for all hyperparameters should be -provided: +Associated with each new type of LearnAPI.jl learner will be a keyword +argument constructor, providing default values for all properties (typically, struct +fields) that are not other learners, and we must implement +[`LearnAPI.constructor(learner)`](@ref), for recovering the constructor from an instance: ```@example anatomy +""" + Ridge(; lambda=0.1) + +Instantiate a ridge regression learner, with regularization of `lambda`. +""" +Ridge(; lambda=0.1) = Ridge(lambda) +LearnAPI.constructor(::Ridge) = Ridge nothing # hide -MyRidge(; lambda=0.1) = MyRidge(lambda) +``` + +For example, in this case, if `learner = Ridge(0.2)`, then +`LearnAPI.constructor(learner)(lambda=0.2) == learner` is true. Note that we attach +the docstring to the *constructor*, not the struct. + + +## Implementing `fit` + +A ridge regressor requires two types of data for training: input features `X`, which here +we suppose are tabular¹, and a [target](@ref proxy) `y`, which we suppose is a vector.⁴ + +It is convenient to define a new type for the `fit` output, which will include +coefficients labelled by feature name for inspection after training: + +```@example anatomy +struct RidgeFitted{T,F} + learner::Ridge + coefficients::Vector{T} + named_coefficients::F +end nothing # hide ``` -## Implementing training (fit) +Note that we also include `learner` in the struct, for it must be possible to recover +`learner` from the output of `fit`; see [Accessor functions](@ref) below. -A ridge regressor requires two types of data for training: **input features** `X` and a -[**target**](@ref scope) `y`. Training is implemented by overloading `fit`. Here `verbosity` is an integer -(`0` should train silently, unless warnings are needed): +The implementation of `fit` looks like this: ```@example anatomy -function LearnAPI.fit(algorithm::MyRidge, verbosity, X, y) +function LearnAPI.fit(learner::Ridge, data; verbosity=LearnAPI.default_verbosity()) - # process input: - x = Tables.matrix(X) # convert table to matrix - s = Tables.schema(X) - features = s.names + X, y = data - # core solver: - coefficients = (x'x + algorithm.lambda*I)\(x'y) + # data preprocessing: + table = Tables.columntable(X) + names = Tables.columnnames(table) |> collect + A = Tables.matrix(table, transpose=true) - # prepare output - learned parameters: - fitted_params = (; coefficients) + lambda = learner.lambda - # prepare output - algorithm state: - state = nothing # not relevant here + # apply core algorithm: + coefficients = (A*A' + learner.lambda*I)\(A*y) # vector - # prepare output - byproducts of training: - feature_importances = - [features[j] => abs(coefficients[j]) for j in eachindex(features)] - sort!(feature_importances, by=last) |> reverse! - verbosity > 0 && @info "Features in order of importance: $(first.(feature_importances))" - report = (; feature_importances) + # determine named coefficients: + named_coefficients = [names[j] => coefficients[j] for j in eachindex(names)] - return fitted_params, state, report + # make some noise, if allowed: + verbosity > 0 && @info "Coefficients: $named_coefficients" + + return RidgeFitted(learner, coefficients, named_coefficients) end -nothing # hide ``` -Regarding the return value of `fit`: +## Implementing `predict` + +One way users will be able to call `predict` is like this: + +```julia +predict(model, Point(), Xnew) +``` -- The `fitted_params` variable is for the algorithm's learned parameters, for passing to - `predict` (see below). +where `Xnew` is a table (of the same form as `X` above). The argument `Point()` +signals that literal predictions of the target variable are sought, as opposed to some +proxy for the target, such as probability density functions. `Point` is an +example of a [`LearnAPI.KindOfProxy`](@ref proxy_types) type. Targets and target proxies +are discussed [here](@ref proxy). -- The `state` variable is only relevant when additionally implementing a [`LearnAPI.update!`](@ref) - or [`LearnAPI.ingest!`](@ref) method (see [Fit, update! and ingest!](@ref)). +We provide this implementation for our ridge regressor: -- The `report` is for other byproducts of training, apart from the learned parameters (the - ones we'll need to provide `predict` below). +```@example anatomy +LearnAPI.predict(model::RidgeFitted, ::Point, Xnew) = + Tables.matrix(Xnew)*model.coefficients +``` -Our `fit` method assumes that `X` is a table (satisfies the [Tables.jl -spec](https://github.com/JuliaData/Tables.jl)) whose rows are the observations; and it -will need need `y` to be an `AbstractFloat` vector. An algorithm implementation is free to -dictate the representation of data that `fit` accepts but articulates its requirements -using appropriate traits; see [Training data types](@ref) below. We recommend against data -type checks internal to `fit`; this would ordinarily be the responsibility of a higher -level API, using those traits. +If the kind of proxy is omitted, as in `predict(model, Xnew)`, then a fallback grabs the +first element of the tuple returned by [`LearnAPI.kinds_of_proxy(learner)`](@ref), which +we overload appropriately below. -## Operations +## Extracting the target from training data -Now we need a method for predicting the target on new input features: +The `fit` method consumes data which includes a [target variable](@ref proxy), i.e., the +learner is a supervised learner. We must therefore declare how the target variable can be extracted +from training data, by implementing [`LearnAPI.target`](@ref): ```@example anatomy -function LearnAPI.predict(::MyRidge, ::LearnAPI.LiteralTarget, fitted_params, Xnew) - Xmatrix = Tables.matrix(Xnew) - report = nothing - return Xmatrix*fitted_params.coefficients, report -end -nothing # hide +LearnAPI.target(learner, data) = last(data) ``` -The second argument of `predict` is always an instance of `KindOfProxy`, and will always -be `LiteralTarget()` in this case, as only literal values of the target (rather than, say -probabilistic predictions) are being supported. +There is a similar method, [`LearnAPI.features`](@ref) for declaring how training features +can be extracted (something that can be passed to `predict`) but this method has a +fallback which suffices here: it returns `first(data)` if `data` is a tuple, and `data` +otherwise. -In some algorithms `predict` computes something of interest in addition to the target -prediction, and this `report` item is returned as the second component of the return -value. When there's nothing to report, we must return `nothing`, as here. -Our `predict` method is an example of an **operation**. Other operations include -`transform` and `inverse_transform` and an algorithm can implement more than one. For -example, a K-means clustering algorithm might implement `transform` for dimension -reduction, and `predict` to return cluster labels. +## Accessor functions + +An [accessor function](@ref accessor_functions) has the output of [`fit`](@ref) as it's +sole argument. Every new implementation must implement the accessor function +[`LearnAPI.learner`](@ref) for recovering a learner from a fitted object: -The `predict` method is reserved for predictions of a [target variable](@ref proxy), and -only `predict` has the extra `::KindOfProxy` argument. +```@example anatomy +LearnAPI.learner(model::RidgeFitted) = model.learner +``` +Other accessor functions extract learned parameters or some standard byproducts of +training, such as feature importances or training losses.² Here we implement an accessor +function to extract the linear coefficients: -## Accessor functions +```@example anatomy +LearnAPI.coefficients(model::RidgeFitted) = model.named_coefficients +nothing #hide +``` -The arguments of an operation are always `(algorithm, fitted_params, data...)`. The -interface also provides **accessor functions** for extracting information, from the -`fitted_params` and/or fit `report`, that is shared by several algorithm types. There is -one for feature importances that we can implement for `MyRidge`: +The [`LearnAPI.strip(model)`](@ref) accessor function is for returning a version of +`model` suitable for serialization (typically smaller and data anonymized). It has a +fallback that just returns `model` but for the sake of illustration, we overload it to +dump the named version of the coefficients: ```@example anatomy -LearnAPI.feature_importances(::MyRidge, fitted_params, report) = - report.feature_importances -nothing # hide +LearnAPI.strip(model::RidgeFitted) = + RidgeFitted(model.learner, model.coefficients, nothing) ``` -Another example of an accessor function is [`LearnAPI.training_losses`](@ref). +Crucially, we can still use `LearnAPI.strip(model)` in place of `model` to make new +predictions. -## [Algorithm traits](@id traits) +## Learner traits -We have implemented `predict`, and it is possible to implement `predict` methods for -multiple `KindOfProxy` types (see See [Target proxies](@ref) for a complete -list). Accordingly, we are required to declare a preferred target proxy, which we do using -[`LearnAPI.preferred_kind_of_proxy`](@ref): +Learner [traits](@ref traits) record extra generic information about a learner, or +make specific promises of behavior. They are methods that have a learner as the sole +argument, and so we regard [`LearnAPI.constructor`](@ref) defined above as a trait. -```@example anatomy -LearnAPI.preferred_kind_of_proxy(::MyRidge) = LearnAPI.LiteralTarget() -nothing # hide +Because we have implemented `predict`, we are required to overload the +[`LearnAPI.kinds_of_proxy`](@ref) trait. Because we can only make point predictions of the +target, we make this definition: + +```julia +LearnAPI.kinds_of_proxy(::Ridge) = (Point(),) ``` -Or, you can use the shorthand + +A macro provides a shortcut, convenient when multiple traits are to be defined: ```@example anatomy -@trait MyRidge preferred_kind_of_proxy=LearnAPI.LiteralTarget() +@trait( + Ridge, + constructor = Ridge, + kinds_of_proxy=(Point(),), + tags = (:regression,), + functions = ( + :(LearnAPI.fit), + :(LearnAPI.learner), + :(LearnAPI.clone), + :(LearnAPI.strip), + :(LearnAPI.obs), + :(LearnAPI.features), + :(LearnAPI.target), + :(LearnAPI.predict), + :(LearnAPI.coefficients), + ) +) nothing # hide ``` -[`LearnAPI.preferred_kind_of_proxy`](@ref) is an example of a **algorithm trait**. A -complete list of traits and the contracts they imply is given in [Algorithm Traits](@ref). +The last trait, `functions`, returns a list of all LearnAPI.jl methods that can be +meaningfully applied to the learner or associated model. You always include the first five +you see here: `fit`, `learner`, `clone` ,`strip`, `obs`. Here [`clone`](@ref) is a utility +function provided by LearnAPI that you never overload; overloading [`obs`](@ref) is +optional (see [Providing a separate data front end](@ref)) but it is always included +because it has a fallback. See [`LearnAPI.functions`](@ref) for a checklist. + +[`LearnAPI.functions`](@ref) and [`LearnAPI.constructor`](@ref), are the only universally +compulsory traits. However, it is worthwhile studying the [list of all traits](@ref +traits_list) to see which might apply to a new implementation, to enable maximum buy into +functionality provided by third party packages, and to assist third party algorithms that +match machine learning algorithms to user-defined tasks. -We also need to indicate that a target variable appears in training (this is a supervised -algorithm). We do this by declaring *where* in the list of training data arguments (in this -case `(X, y)`) the target variable (in this case `y`) appears: +Note that we know `Ridge` instances are supervised learners because `:(LearnAPI.target) +in LearnAPI.functions(learner)`, for every instance `learner`. With [some +exceptions](@ref trait_contract), the value of a trait should depend only on the *type* of +the argument. + +## Signatures added for convenience + +We add one `fit` signature for user-convenience only. The LearnAPI.jl specification has +nothing to say about `fit` signatures with more than two positional arguments. ```@example anatomy -@trait MyRidge position_of_target=2 -nothing # hide +LearnAPI.fit(learner::Ridge, X, y; kwargs...) = fit(learner, (X, y); kwargs...) ``` -As explained in the introduction, LearnAPI.jl does not attempt to define strict algorithm -categories, such as "regression" or "clustering". However, we can optionally specify suggestive -descriptors, as in +## [Demonstration](@id workflow) + +We now illustrate how to interact directly with `Ridge` instances using the methods +just implemented. ```@example anatomy -@trait MyRidge descriptors=(:regression,) +# synthesize some data: +n = 10 # number of observations +train = 1:6 +test = 7:10 +a, b, c = rand(n), rand(n), rand(n) +X = (; a, b, c) +y = 2a - b + 3c + 0.05*rand(n) nothing # hide ``` -This declaration actually promises nothing, but can help in generating documentation. Do -`LearnAPI.descriptors()` to get a list of available descriptors. +```@example anatomy +learner = Ridge(lambda=0.5) +@functions learner +``` -Finally, we are required to declare what methods (excluding traits) we have explicitly -overloaded for our type: +Training and predicting: ```@example anatomy -@trait MyRidge methods=( - :fit, - :predict, - :feature_importances, -) -nothing # hide +Xtrain = Tables.subset(X, train) +ytrain = y[train] +model = fit(learner, (Xtrain, ytrain)) # `fit(learner, Xtrain, ytrain)` will also work +ŷ = predict(model, Tables.subset(X, test)) ``` -## Training data types +Extracting coefficients: -Since LearnAPI.jl is a basement level API, one is discouraged from including explicit type -checks in an implementation of `fit`. Instead one uses traits to make promises about the -acceptable type of `data` consumed by `fit`. In general, this can be a promise regarding -the ordinary type of `data` or the [scientific -type](https://github.com/JuliaAI/ScientificTypes.jl) of `data` (but not -both). Alternatively, one may only promise a bound on the type/scitype of *observations* -in the data . See [Algorithm Traits](@ref) for further details. In this case we'll be -happy to restrict the scitype of the data: +```@example anatomy +LearnAPI.coefficients(model) +``` + +Serialization/deserialization: ```@example anatomy -import ScientificTypesBase: scitype, Table, Continuous -@trait MyRidge fit_scitype = Tuple{Table(Continuous), AbstractVector{Continuous}} -nothing # hide +using Serialization +small_model = LearnAPI.strip(model) +filename = tempname() +serialize(filename, small_model) ``` -This is a contract that `data` is acceptable in the call `fit(algorithm, verbosity, data...)` -whenever +```julia +recovered_model = deserialize(filename) +@assert LearnAPI.learner(recovered_model) == learner +@assert predict(recovered_model, X) == predict(model, X) +``` + +## Providing a separate data front end + +```@setup anatomy2 +using LearnAPI +using LinearAlgebra, Tables + +struct Ridge{T<:Real} + lambda::T +end + +Ridge(; lambda=0.1) = Ridge(lambda) + +struct RidgeFitted{T,F} + learner::Ridge + coefficients::Vector{T} + named_coefficients::F +end + +LearnAPI.learner(model::RidgeFitted) = model.learner +LearnAPI.coefficients(model::RidgeFitted) = model.named_coefficients +LearnAPI.strip(model::RidgeFitted) = + RidgeFitted(model.learner, model.coefficients, nothing) + +@trait( + Ridge, + constructor = Ridge, + kinds_of_proxy=(Point(),), + tags = (:regression,), + functions = ( + :(LearnAPI.fit), + :(LearnAPI.learner), + :(LearnAPI.clone), + :(LearnAPI.strip), + :(LearnAPI.obs), + :(LearnAPI.features), + :(LearnAPI.target), + :(LearnAPI.predict), + :(LearnAPI.coefficients), + ) +) + +n = 10 # number of observations +train = 1:6 +test = 7:10 +a, b, c = rand(n), rand(n), rand(n) +X = (; a, b, c) +y = 2a - b + 3c + 0.05*rand(n) +``` + +An implementation may optionally implement [`obs`](@ref), to expose to the user (or some +meta-algorithm like cross-validation) the representation of input data internal to `fit` +or `predict`, such as the matrix version `A` of `X` in the ridge example. That is, we may +factor out of `fit` (and also `predict`) a data pre-processing step, `obs`, to expose +its outcomes. These outcomes become alternative user inputs to `fit`/`predict`. + +In the default case, the alternative data representations will implement the MLUtils.jl +`getobs/numobs` interface for observation subsampling, which is generally all a user or +meta-algorithm will need, before passing the data on to `fit`/`predict` as you would the +original data. + +So, instead of the pattern ```julia -scitype(data) <: Tuple{Table(Continuous), AbstractVector{Continuous}} +model = fit(learner, data) +predict(model, newdata) ``` -Or, in other words: +one enables the following alternative (which in any case will still work, because of a +no-op `obs` fallback provided by LearnAPI.jl): + +```julia +observations = obs(learner, data) # pre-processed training data -- `X` in `fit(algorithm, verbosity, X, y)` is acceptable, provided `scitype(X) <: - Table(Continuous)` - meaning that `X` `Tables.istable(X) == true` (see - [Tables.jl](https://github.com/JuliaData/Tables.jl)) and each column has some - `<:AbstractFloat` element type. +# optional subsampling: +observations = MLUtils.getobs(observations, train_indices) -- `y` in `fit(algorithm, verbosity, X, y)` is acceptable if `scitype(y) <: - AbstractVector{Continuous}` - meaning that it is an abstract vector with `<:AbstractFloat` - elements. +model = fit(learner, observations) -## Input types for operations +newobservations = obs(model, newdata) -An optional promise about what `data` is guaranteed to work in a call like -`predict(algorithm, fitted_params, data...)` is articulated this way: +# optional subsampling: +newobservations = MLUtils.getobs(observations, test_indices) -```@example anatomy -@trait MyRidge predict_input_scitype = Tuple{AbstractVector{<:Continuous}} +predict(model, newobservations) ``` -Note that `data` is always a `Tuple`, even if it has only one component (the typical -case), which explains the `Tuple` on the right-hand side. +See also the demonstration [below](@ref advanced_demo). + +Here we specifically wrap all the pre-processed data into single object, for which we +introduce a new type: + +```@example anatomy2 +struct RidgeFitObs{T,M<:AbstractMatrix{T}} + A::M # `p` x `n` matrix + names::Vector{Symbol} # features + y::Vector{T} # target +end +``` -Optionally, we may express our promise using regular types, using the -[`LearnAPI.predict_input_type`](@ref) trait. +Now we overload `obs` to carry out the data pre-processing previously in `fit`, like this: -One can optionally make promises about the outut of an operation. See [Algorithm -Traits](@ref) for details. +```@example anatomy2 +function LearnAPI.obs(::Ridge, data) + X, y = data + table = Tables.columntable(X) + names = Tables.columnnames(table) |> collect + return RidgeFitObs(Tables.matrix(table)', names, y) +end +``` +We informally refer to the output of `obs` as "observations" (see [The `obs` +contract](@ref) below). The previous core `fit` signature is now replaced with two +methods - one to handle "regular" input, and one to handle the pre-processed data +(observations) which appears first below: -## [Illustrative fit/predict workflow](@id workflow) +```@example anatomy2 +function LearnAPI.fit(learner::Ridge, observations::RidgeFitObs; verbosity=LearnAPI.default_verbosity()) -We now illustrate how to interact directly with `MyRidge` instances using the methods we -have implemented: + lambda = learner.lambda -Here's some toy data for supervised learning: + A = observations.A + names = observations.names + y = observations.y -```@example anatomy -using Tables + # apply core learner: + coefficients = (A*A' + learner.lambda*I)\(A*y) # 1 x p matrix -n = 10 # number of training observations -train = 1:6 -test = 7:10 + # determine named coefficients: + named_coefficients = [names[j] => coefficients[j] for j in eachindex(names)] -a, b, c = rand(n), rand(n), rand(n) -X = (; a, b, c) |> Tables.rowtable -y = 2a - b + 3c + 0.05*rand(n) -nothing # hide + # make some noise, if allowed: + verbosity > 0 && @info "Coefficients: $named_coefficients" + + return RidgeFitted(learner, coefficients, named_coefficients) + +end + +LearnAPI.fit(learner::Ridge, data; kwargs...) = + fit(learner, obs(learner, data); kwargs...) ``` -Instantiate an algorithm with relevant hyperparameters (which is all the object stores): -```@example anatomy -algorithm = MyRidge(lambda=0.5) +### The `obs` contract + +Providing `fit` signatures matching the output of [`obs`](@ref), is the first part of the +`obs` contract. Since `obs(learner, data)` should evidently support all `data` that +`fit(learner, data)` supports, we must be able to apply `obs(learner, _)` to it's own +output (`observations` below). This leads to the additional "no-op" declaration + +```@example anatomy2 +LearnAPI.obs(::Ridge, observations::RidgeFitObs) = observations ``` -Train the algorithm (the `0` means do so silently): +In other words, we ensure that `obs(learner, _)` is +[involutive](https://en.wikipedia.org/wiki/Involution_(mathematics)). -```@example anatomy -import LearnAPI: fit, predict, feature_importances +The second part of the `obs` contract is this: *The output of `obs` must implement the +interface specified by the trait* [`LearnAPI.data_interface(learner)`](@ref). Assuming +this is [`LearnAPI.RandomAccess()`](@ref) (the default) it usually suffices to overload +`Base.getindex` and `Base.length`: -fitted_params, state, fit_report = fit(algorithm, 0, X[train], y[train]) +```@example anatomy2 +Base.getindex(data::RidgeFitObs, I) = + RidgeFitObs(data.A[:,I], data.names, y[I]) +Base.length(data::RidgeFitObs) = length(data.y) ``` -Inspect the learned parameters and report: +We do something similar for `predict`, but there's no need for a new type in this case: -```@example anatomy -@info "training outcomes" fitted_params fit_report +```@example anatomy2 +LearnAPI.obs(::RidgeFitted, Xnew) = Tables.matrix(Xnew)' +LearnAPI.obs(::RidgeFitted, observations::AbstractArray) = observations # involutivity + +LearnAPI.predict(model::RidgeFitted, ::Point, observations::AbstractMatrix) = + observations'*model.coefficients + +LearnAPI.predict(model::RidgeFitted, ::Point, Xnew) = + predict(model, Point(), obs(model, Xnew)) ``` -Inspect feature importances: +### `target` and `features` methods -```@example anatomy -feature_importances(algorithm, fitted_params, fit_report) +In the general case, we only need to implement [`LearnAPI.target`](@ref) and +[`LearnAPI.features`](@ref) to handle all possible output of `obs(learner, data)`, and now +the fallback for `LearnAPI.features` mentioned before is inadequate. + +```@example anatomy2 +LearnAPI.target(::Ridge, observations::RidgeFitObs) = observations.y +LearnAPI.features(::Ridge, observations::RidgeFitObs) = observations.A ``` -Make a prediction using new data: +### Important notes: -```@example anatomy -yhat, predict_report = predict(algorithm, LearnAPI.LiteralTarget(), fitted_params, X[test]) +- The observations to be consumed by `fit` are returned by `obs(learner::Ridge, ...)`, + while those consumed by `predict` are returned by `obs(model::RidgeFitted, ...)`. We + need the different signatures because the form of data consumed by `fit` and `predict` + are generally different. + +- We need the adjoint operator, `'`, because the last dimension in arrays is the + observation dimension, according to the MLUtils.jl convention. Remember, `Xnew` is a + table here. + +Since LearnAPI.jl provides fallbacks for `obs` that simply return the unadulterated data +argument, overloading `obs` is optional. This is provided data in publicized +`fit`/`predict` signatures consists only of objects implement the +[`LearnAPI.RandomAccess`](@ref) interface (most tables¹, arrays³, and tuples thereof). + +To opt out of supporting the MLUtils.jl interface altogether, an implementation must +overload the trait, [`LearnAPI.data_interface(learner)`](@ref). See [Data +interfaces](@ref data_interfaces) for details. + + +### Addition of signatures for user convenience + +As above, we add a signature for convenience, which the LearnAPI.jl specification +neither requires nor forbids: + +```@example anatomy2 +LearnAPI.fit(learner::Ridge, X, y; kwargs...) = fit(learner, (X, y); kwargs...) ``` -Compare predictions with ground truth +## [Demonstration of an advanced `obs` workflow](@id advanced_demo) -```@example anatomy -deviations = yhat - y[test] -loss = deviations .^2 |> sum -@info "Sum of squares loss" loss +We now can train and predict using internal data representations, resampled using the +generic MLUtils.jl interface: + +```@example anatomy2 +import MLUtils +learner = Ridge() +observations_for_fit = obs(learner, (X, y)) +model = fit(learner, MLUtils.getobs(observations_for_fit, train)) +observations_for_predict = obs(model, X) +ẑ = predict(model, MLUtils.getobs(observations_for_predict, test)) +``` + +```julia +@assert ẑ == ŷ ``` + +For an application of [`obs`](@ref) to efficient cross-validation, see [here](@ref +obs_workflows). + +--- + +¹ In LearnAPI.jl a *table* is any object `X` implementing the +[Tables.jl](https://tables.juliadata.org/dev/) interface, additionally satisfying +`Tables.istable(X) == true` and implementing `DataAPI.nrow` (and whence +`MLUtils.numobs`). Tables that are also (unnamed) tuples are disallowed. + +² An implementation can provide further accessor functions, if necessary, but +like the native ones, they must be included in the [`LearnAPI.functions`](@ref) +declaration. + +³ The last index must be the observation index. + +⁴ The `data = (X, y)` pattern implemented here is not the only supported pattern. For, +example, `data` might be `(T, formula)` where `T` is a table and `formula` is an R-style +formula. diff --git a/docs/src/common_implementation_patterns.md b/docs/src/common_implementation_patterns.md index 26c221fd..85ebe507 100644 --- a/docs/src/common_implementation_patterns.md +++ b/docs/src/common_implementation_patterns.md @@ -1,46 +1,65 @@ -# Common Implementation Patterns +# [Common Implementation Patterns](@id patterns) -!!! warning +!!! important This section is only an implementation guide. The definitive specification of the - Learn API is given in [Reference](@ref reference). + LearnAPI is given in [Reference](@ref reference). This guide is intended to be consulted after reading [Anatomy of an Implementation](@ref), which introduces the main interface objects and terminology. -Although an implementation is defined purely by the methods and traits it implements, most +Although an implementation is defined purely by the methods and traits it implements, many implementations fall into one (or more) of the following informally understood patterns or "tasks": -- [Classifiers](@ref): Supervised learners for categorical targets +- [Regression](@ref): Supervised learners for continuous targets -- [Regressors](@ref): Supervised learners for continuous targets +- [Classification](@ref): Supervised learners for categorical targets + +- Clusterering: Algorithms that group data into clusters for classification and + possibly dimension reduction. May be true learners (generalize to new data) or static. + +- [Gradient Descent](@ref): Including neural networks. - [Iterative Algorithms](@ref) -- [Incremental Algorithms](@ref) +- [Incremental Algorithms](@ref): Algorithms that can be updated with new observations. -- [Static Transformers](@ref): Transformations that do not learn but which have - hyperparameters and/or deliver ancillary information about the transformation +- [Feature Engineering](@ref): Algorithms for selecting or combining features - [Dimension Reduction](@ref): Transformers that learn to reduce feature space dimension -- [Missing Value Imputation](@ref): Transformers that replace missing values. +- Missing Value Imputation -- [Clusterering](@ref): Algorithms that group data into clusters for classification and - possibly dimension reduction. May be true learners (generalize to new data) or static. +- [Transformers](@ref transformers): Other transformers, such as standardizers, and + categorical encoders. + +- [Static Algorithms](@ref): Algorithms that do not learn, in the sense they must be + re-executed for each new data set (do not generalize), but which have hyperparameters + and/or deliver ancillary information about the computation. + +- [Ensembling](@ref): Algorithms that blend predictions of multiple algorithms + +- Time Series Forecasting + +- Time Series Classification + +- Survival Analysis + +- [Density Estimation](@ref): Algorithms that learn a probability distribution + +- Bayesian Algorithms -- [Outlier Detection](@ref): Supervised, unsupervised, or semi-supervised learners for +- Outlier Detection: Supervised, unsupervised, or semi-supervised learners for anomaly detection. -- [Learning a Probability Distribution](@ref): Algorithms that fit a distribution or - distribution-like object to data +- Text Analysis -- [Time Series Forecasting](@ref) +- Audio Analysis -- [Time Series Classification](@ref) +- Natural Language Processing -- [Supervised Bayesian Algorithms](@ref) +- Image Processing -- [Survival Analysis](@ref) +- [Meta-algorithms](@ref) diff --git a/docs/src/fit_update.md b/docs/src/fit_update.md new file mode 100644 index 00000000..c649e6dd --- /dev/null +++ b/docs/src/fit_update.md @@ -0,0 +1,128 @@ +# [`fit`, `update`, `update_observations`, and `update_features`](@id fit_docs) + +### Training + +```julia +fit(learner, data; verbosity=LearnAPI.default_verbosity()) -> model +fit(learner; verbosity=LearnAPI.default_verbosity()) -> static_model +``` + +A "static" algorithm is one that does not generalize to new observations (e.g., some +clustering algorithms); there is no training data and the algorithm is executed by +`predict` or `transform` which receive the data. See example below. + + +### Updating + +``` +update(model, data; verbosity=..., param1=new_value1, param2=new_value2, ...) -> updated_model +update_observations(model, new_data; verbosity=..., param1=new_value1, ...) -> updated_model +update_features(model, new_data; verbosity=..., param1=new_value1, ...) -> updated_model +``` + +## Typical workflows + +### Supervised models + +Supposing `Learner` is some supervised classifier type, with an iteration parameter `n`: + +```julia +learner = Learner(n=100) +model = fit(learner, (X, y)) + +# Predict probability distributions: +ŷ = predict(model, Distribution(), Xnew) + +# Inspect some byproducts of training: +LearnAPI.feature_importances(model) + +# Add 50 iterations and predict again: +model = update(model; n=150) +predict(model, Distribution(), X) +``` + +See also [Classification](@ref) and [Regression](@ref). + +### Transformers + +A dimension-reducing transformer, `learner`, might be used in this way: + +```julia +model = fit(learner, X) +transform(model, X) # or `transform(model, Xnew)` +``` + +or, if implemented, using a single call: + +```julia +transform(learner, X) # `fit` implied +``` + +### [Static algorithms (no "learning")](@id static_algorithms) + +Suppose `learner` is some clustering algorithm that cannot be generalized to new data +(e.g. DBSCAN): + +```julia +model = fit(learner) # no training data +labels = predict(model, X) # may mutate `model` + +# Or, in one line: +labels = predict(learner, X) + +# But two-line version exposes byproducts of the clustering algorithm (e.g., outliers): +LearnAPI.extras(model) +``` + +See also [Static Algorithms](@ref) + +### [Density estimation](@id density_estimation) + +In density estimation, `fit` consumes no features, only a target variable; `predict`, +which consumes no data, returns the learned density: + +```julia +model = fit(learner, y) # no features +predict(model) # shortcut for `predict(model, SingleDistribution())`, or similar +``` + +A one-liner will typically be implemented as well: + +```julia +predict(learner, y) +``` + +See also [Density Estimation](@ref). + + +## Implementation guide + +### Training + +Exactly one of the following must be implemented: + +| method | fallback | +|:-----------------------------------------------------------------------|:---------| +| [`fit`](@ref)`(learner, data; verbosity=LearnAPI.default_verbosity())` | none | +| [`fit`](@ref)`(learner; verbosity=LearnAPI.default_verbosity())` | none | + +### Updating + +| method | fallback | compulsory? | +|:-------------------------------------------------------------------------------------|:---------|-------------| +| [`update`](@ref)`(model, data; verbosity=..., hyperparameter_updates...)` | none | no | +| [`update_observations`](@ref)`(model, new_data; verbosity=..., hyperparameter_updates...)` | none | no | +| [`update_features`](@ref)`(model, new_data; verbosity=..., hyperparameter_updates...)` | none | no | + +There are some contracts governing the behaviour of the update methods, as they relate to +a previous `fit` call. Consult the document strings for details. + +## Reference + +```@docs +fit +update +update_observations +update_features +LearnAPI.default_verbosity +``` diff --git a/docs/src/fit_update_and_ingest.md b/docs/src/fit_update_and_ingest.md deleted file mode 100644 index db935a2a..00000000 --- a/docs/src/fit_update_and_ingest.md +++ /dev/null @@ -1,46 +0,0 @@ -# Fit, update! and ingest! - -> **Summary.** Algorithms that learn, i.e., generalize to new data, must overload `fit`; the -> fallback performs no operation and returns all `nothing`. Implement `update!` if certain -> hyperparameter changes do not necessitate retraining from scratch (e.g., increasing an -> iteration parameter). Implement `ingest!` to implement incremental learning. All -> training methods implemented must be named in the return value of the -> `functions` trait. - -| method | fallback | compulsory? | requires | -|:---------------------------|:---------------------------------------------------|-------------|-------------------| -| [`LearnAPI.fit`](@ref) | does nothing, returns `(nothing, nothing, nothing)`| no | | -| [`LearnAPI.update!`](@ref) | calls `fit` | no | [`LearnAPI.fit`](@ref) | -| [`LearnAPI.ingest!`](@ref) | none | no | [`LearnAPI.fit`](@ref) | - -All three methods above return a triple `(fitted_params, state, report)` whose components -are explained under [`LearnAPI.fit`](@ref) below. Items that might be returned in -`report` include: feature rankings/importances, SVM support vectors, clustering centers, -methods for visualizing training outcomes, methods for saving learned parameters in a -custom format, degrees of freedom, deviances. Precisely what `report` includes might be -controlled by hyperparameters (algorithm properties) especially if there is a performance -cost to it's inclusion. - -Implement `fit` unless all [operations](@ref operations), such as `predict` and -`transform`, ignore their `fitted_params` argument (which will be `nothing`). This is the -case for many algorithms that have hyperparameters, but do not generalize to new data, such -as a basic DBSCAN clustering algorithm. - -The `update!` method is intended for all subsequent calls to train an algorithm *using the same -observations*, but with possibly altered hyperparameters (`algorithm` argument). A fallback -implementation simply calls `fit`. The main use cases for implementing `update` are: - -- warm-restarting iterative algorithms - -- "smart" training of composite algorithms, such as linear pipelines; here "smart" means that - hyperparameter changes only trigger the retraining of downstream components. - -The `ingest!` method supports incremental learning (same hyperparameters, but new training -observations). Like `update!`, it depends on the output a preceding `fit` or `ingest!` -call. - -```@docs -LearnAPI.fit -LearnAPI.update! -LearnAPI.ingest! -``` diff --git a/docs/src/goals_and_approach.md b/docs/src/goals_and_approach.md deleted file mode 100644 index 467b600c..00000000 --- a/docs/src/goals_and_approach.md +++ /dev/null @@ -1,49 +0,0 @@ -# Goals and Approach - -## Goals - -- Ease of implementation for existing ML/statistics algorithms - -- Breadth of applicability - -- Flexibility in extending functionality - -- Provision of clear interface points for algorithm-generic tooling, such as performance - evaluation through resampling, hyperparameter optimization, and iterative algorithm - control. - -- Should make minimal assumptions about data containers - -- Should be documented in detail - -In particular, the first three goals are to take precedence over user convenience, which -is addressed with a separate, [User Interface](@ref). - - -## Approach - -ML/Statistics algorithms have a complicated taxonomy. Grouping algorithms, or modelling -tasks, into a relatively small number of categories, such as "classification" and -"clusterering", and then imposing uniform behavior within each group, is challenging. In -our experience developing the [MLJ -ecosystem](https://github.com/alan-turing-institute/MLJ.jl), this either leads to -limitations on the algorithms that can be included in a general interface, or additional -complexity needed to cope with exceptional cases. Even if a complete data science -framework might benefit from such groupings, a basement-level API should, in our view, -avoid them. - -In addition to basic methods, like `fit` and `predict`, LearnAPI provides a number of -optional algorithm -[traits](https://ahsmart.com/pub/holy-traits-design-patterns-and-best-practice-book/), -each promising a specific kind of behavior, such as "This algorithm supports class -weights". There is no abstract type hierarchy for ML/statistics algorithms. - -LearnAPI.jl intentionally focuses on the notion of [target variables and target -proxies](@ref proxy), which can exist in both the superised and unsupervised setting, -rather than on the supervised/unsupervised dichotomy. In this view a supervised model is -simply one which has a target variable *and* whose target variable appears in training. - -LearnAPI is a basement-level interface and not a general ML/statistics toolbox. Algorithms -can be supervised or not supervised, can generalize to new data observations (i.e., -"learn") or not generalize (e.g., "one-shot" clusterers). - diff --git a/docs/src/index.md b/docs/src/index.md index f6bede45..55c18898 100644 --- a/docs/src/index.md +++ b/docs/src/index.md @@ -1,168 +1,110 @@ ```@raw html + +
+ Tutorial  |  + Reference  |  + Patterns +
+ LearnAPI.jl
A base Julia interface for machine learning and statistics -

+
+
``` -## Accelerated overview - -LearnAPI.jl provides a collection methods stubs, such as `fit` and `predict`, to be -implemented by algorithms from machine learning and statistics. Through such -implementations, such algorithms buy into algorithm-generic functionality, such as -hyperparameter optimization, as provided by ML/statistics toolboxes and other -packages. LearnAPI.jl also provides a number of Julia traits for making specific promises -of behavior. - -It is designed to be powerful, from the point of view of adding algorithm-generic -functionality, while minimizing the burden on developers implementing the API for a -specific algorithm. - -- To see how to **DEVELOPERS INTERACT** with algorithms implementing LearnAPI, see [Basic fit/predict - workflow](@ref workflow). - -- To see how **USERS INTERACT** with LearnAPI algorithms, see [User - Interface](@ref).[under construction] - -- For developers wanting to **IMPLEMENT** LearnAPI, see [Anatomy of - an Implementation](@ref). - -For more on package goals and philosophy, see [Goals and Approach](@ref). - - -## Methods - -In LearnAPI an *algorithm* is a Julia object storing the hyperparameters of some -ML/statistics algorithm. - -The following methods, dispatched on algorithm type, are provided: - -- `fit`, overloaded if an algorithm involves a learning step, as in classical supervised - learning; the principal output of `fit` is learned parameters - -- `update!`, for adding iterations to an algorithm, or responding efficiently to other - post-`fit`changes in hyperparameters +LearnAPI.jl is a lightweight, functional-style interface, providing a collection of +[methods](@ref Methods), such as `fit` and `predict`, to be implemented by algorithms from +machine learning and statistics, some examples of which are listed [here](@ref +patterns). A careful design ensures algorithms implementing LearnAPI.jl can buy into +functionality, such as external performance estimates, hyperparameter optimization and +model composition, provided by ML/statistics toolboxes and other packages. LearnAPI.jl +includes a number of Julia [traits](@ref traits) for promising specific behavior. -- `ingest!`, for incremental learning (training further using *new* data, without - re-initializing learned parameters) +LearnAPI.jl's has no package dependencies. -- *operations*, which apply the algorithm to data, typically not seen in - training, if there is any: - - - `predict`, for predicting values of a target variable or a proxy for the target, such as probability distributions; see below - - - `transform`, for other kinds transformations - - - `inverse_transform`, for reconstructing data from a transformed representation - -- common *accessor functions*, such as `feature_importances` and `training_losses`, for - extracting, from training outcomes, information common to a number of different - algorithms - -- *algorithm traits*, such as `predict_output_type(algorithm)`, for promising specific behavior - -Since this is a functional-style interface, `fit` returns algorithm `state`, in addition to -learned parameters, for passing to the optional `update!` and `ingest!` methods. These -training methods also return a `report` component, for exposing byproducts of training -different from learned parameters. Similarly, all operations also return a `report` -component (important for algorithms that do not generalize to new data). - - -## [Informal concepts](@id scope) - -LearnAPI.jl is predicated on a few basic, informally defined notions, in *italics* -below, which some higher-level interface might decide to formalize. - -- An object which generates ordered sequences of individual *observations* is called - *data*. For example a `DataFrame` instance, from - [DataFrames.jl](https://dataframes.juliadata.org/stable/), is considered data, the - observations being the rows. A matrix can be considered data, but whether the - observations are rows or columns is ambiguous and not fixed by LearnAPI. - -- Each machine learning algorithm's behavior is governed by a number of user-specified - *hyperparameters*. The regularization parameter in ridge regression is an - example. Hyperparameters are data-independent. For example, the number of target classes - is not a hyperparameter. - -- Information needed for training that is not a hyperparameter and not data is called - *metadata*. Examples, include target *class* weights and group lasso feature - groupings. Further examples include feature names, and the pool of target classes, when - these are not embedded in the data representation. +```@raw html +🚧 +``` +!!! warning -### [Targets and target proxies](@id proxy) + The API described here is under active development and not ready for adoption. + Join an ongoing design discussion at + [this](https://discourse.julialang.org/t/ann-learnapi-jl-proposal-for-a-basement-level-machine-learning-api/93048) + Julia Discourse thread. -After training, a supervised classifier predicts labels on some input which are then -compared with ground truth labels using some accuracy measure, to assesses the performance -of the classifier. Alternatively, the classifier predicts class probabilities, which are -instead paired with ground truth labels using a proper scoring rule, say. In outlier -detection, "outlier"/"inlier" predictions, or probability-like scores, are similarly -compared with ground truth labels. In clustering, integer labels assigned to observations -by the clustering algorithm can can be paired with human labels using, say, the Rand -index. In survival analysis, predicted survival functions or probability distributions are -compared with censored ground truth survival times. -More generally, whenever we have a predicted variable (e.g., a class label) paired with -itself or some proxy (such as a class probability) we call the variable a *target* -variable, and the predicted output a *target proxy*. It is immaterial whether or not the -target appears in training (is supervised) or whether the model generalizes to new -observations (learns) or not. +## Sample workflow -The target and the kind of predicted proxy are crucial features of ML/statistics -performance measures (not provided by this package) and LearnAPI.jl provides a detailed -list of proxy dispatch types (see [Target proxies](@ref)), as well as algorithm traits to -articulate target type /scitype. +Suppose `forest` is some object encapsulating the hyperparameters of the [random forest +algorithm](https://en.wikipedia.org/wiki/Random_forest) (the number of trees, etc.). Then, +a LearnAPI.jl interface can be implemented, for objects with the type of `forest`, to +enable the basic workflow below. In this case data is presented following the +"scikit-learn" `X, y` pattern, although LearnAPI.jl supports other patterns as well. +```julia +# `X` is some training features +# `y` is some training target +# `Xnew` is some test or production features -## Optional data interface +# List LearnaAPI functions implemented for `forest`: +@functions forest -It can be useful to distinguish between data that exists at some high level, convenient -for the general user - such as a table (dataframe) or the path to a directory containing -image files - and a performant, algorithm-specific representation of that data, such as a -matrix or image "data loader". When retraining using the same data with new -hyperparameters, one wants to avoid recreating the algorithm-specific representation, and, -accordingly, a higher level interface may want to cache such representations. Furthermore, -in resampling (e.g., cross-validation), a higher level interface wants to directly -resample the algorithm-specific representation, so it needs to know how to do that. To -meet these two ends, LearnAPI provides two additional *data methods* dispatched on -algorithm type: +# Train: +model = fit(forest, X, y) -- `reformat(algorithm, ...)`, for converting from a user data representation to a - performant algorithm-specific representation, whose output is for use in `fit`, - `predict`, etc. above +# Generate point predictions: +ŷ = predict(model, Xnew) # or `predict(model, Point(), Xnew)` -- `getobs(algorithm, ...)`, for extracting a subsample of observations of the - algorithm-specific representation +# Predict probability distributions: +predict(model, Distribution(), Xnew) -It should be emphasized that LearnAPI is itself agnostic to particular representations of -data or the particular methods of accessing observations within them. By overloading these -methods, each `algorithm` is free to choose its own data interface. +# Apply an "accessor function" to inspect byproducts of training: +LearnAPI.feature_importances(model) -See [Optional data Interface](@ref data_interface) for more details. +# Slim down and otherwise prepare model for serialization: +small_model = LearnAPI.strip(model) +serialize("my_random_forest.jls", small_model) +``` -## Contents +`Distribution` and `Point` are singleton types owned by LearnAPI.jl. They allow +dispatch based on the [kind of target proxy](@ref proxy), a key LearnAPI.jl concept. +LearnAPI.jl places more emphasis on the notion of target variables and target proxies than +on the usual supervised/unsupervised learning dichotomy. From this point of view, a +supervised learner is simply one in which a target variable exists, and happens to +appear as an input to training but not to prediction. -It is useful to have a guide to the interface, linked below, organized around common -*informally defined* patterns or "tasks". However, the definitive specification of the -interface is the [Reference](@ref reference) section. +## Data interfaces -- Overview: [Anatomy of an Implementation](@ref) +Algorithms are free to consume data in any format. However, a method called [`obs`](@ref +data_interface) (read as "observations") gives users and meta-algorithms access to an +algorithm-specific representation of input data, which is also guaranteed to implement a +standard interface for accessing individual observations, unless the algorithm explicitly +opts out. Moreover, the `fit` and `predict` methods will also be able to consume these +alternative data representations, for performance benefits in some situations. -- Official Specification: [Reference](@ref reference) +The fallback data interface is the [MLUtils.jl](https://github.com/JuliaML/MLUtils.jl) +`getobs/numobs` interface, here tagged as [`LearnAPI.RandomAccess()`](@ref), and if the +input consumed by the algorithm already implements that interface (tables, arrays, etc.) +then overloading `obs` is completely optional. Plain iteration interfaces, with or without +knowledge of the number of observations, can also be specified, to support, e.g., data +loaders reading images from disk. -- User guide: [Common Implementation Patterns](@ref) [under construction] +## Learning more -- [Testing an Implementation](@ref) [under construction] +- [Anatomy of an Implementation](@ref): informal introduction to the main actors in a new + LearnAPI.jl implementation -!!! info +- [Reference](@ref reference): official specification - It is recommended developers read [Anatomy of an Implementation](@ref) before - consulting the guide or reference sections. +- [Common Implementation Patterns](@ref patterns): implementation suggestions for common, + informally defined, algorithm types -*Note.* In the future, LearnAPI.jl may become the new foundation for the -[MLJ](https://alan-turing-institute.github.io/MLJ.jl/dev/) toolbox. However, LearnAPI.jl -is meant as a general purpose, stand-alone, lightweight, low level API (and has no -reference to the "machines" used in MLJ). +- [Testing an Implementation](@ref) diff --git a/docs/src/kinds_of_target_proxy.md b/docs/src/kinds_of_target_proxy.md new file mode 100644 index 00000000..ff9d3f4b --- /dev/null +++ b/docs/src/kinds_of_target_proxy.md @@ -0,0 +1,27 @@ +# [Kinds of Target Proxy](@id proxy_types) + +The available kinds of [target proxy](@ref proxy) (used for `predict` dispatch) are +classified by subtypes of `LearnAPI.KindOfProxy`. These types are intended for dispatch +only and have no fields. + +```@docs +LearnAPI.KindOfProxy +``` + +## Simple target proxies + +```@docs +LearnAPI.IID +``` + +## Proxies for density estimation algorithms + +```@docs +LearnAPI.Single +``` + +## Joint probability distributions + +```@docs +LearnAPI.Joint +``` diff --git a/docs/src/obs.md b/docs/src/obs.md new file mode 100644 index 00000000..a583f27d --- /dev/null +++ b/docs/src/obs.md @@ -0,0 +1,112 @@ +# [`obs` and Data Interfaces](@id data_interface) + +The `obs` method takes data intended as input to `fit`, `predict` or `transform`, and +transforms it to a learner-specific form guaranteed to implement a form of observation +access designated by the learner. The transformed data can then passed on to the relevant +method in place of the original input (after first resampling it, if the learner supports +this). Using `obs` may provide performance advantages over naive workflows in some cases +(e.g., cross-validation). + +```julia +obs(learner, data) # can be passed to `fit` instead of `data` +obs(model, data) # can be passed to `predict` or `transform` instead of `data` +``` + +## [Typical workflows](@id obs_workflows) + +LearnAPI.jl makes no universal assumptions about the form of `data` in a call +like `fit(learner, data)`. However, if we define + +```julia +observations = obs(learner, data) +``` + +then, assuming the typical case that `LearnAPI.data_interface(learner) == +LearnAPI.RandomAccess()`, `observations` implements the +[MLUtils.jl](https://juliaml.github.io/MLUtils.jl/dev/) `getobs`/`numobs` interface, for +grabbing and counting observations. Moreover, we can pass `observations` to `fit` in place +of the original data, or first resample it using `MLUtils.getobs`: + +```julia +# equivalent to `model = fit(learner, data)` +model = fit(learner, observations) + +# with resampling: +resampled_observations = MLUtils.getobs(observations, 1:10) +model = fit(learner, resampled_observations) +``` + +In some implementations, the alternative pattern above can be used to avoid repeating +unnecessary internal data preprocessing, or inefficient resampling. For example, here's +how a user might call `obs` and `MLUtils.getobs` to perform efficient cross-validation: + +```julia +using LearnAPI +import MLUtils + +learner = + +data = + +train_test_folds = map([1:10, 11:20, 21:30]) do test + (setdiff(1:30, test), test) +end + +fitobs = obs(learner, data) +never_trained = true + +scores = map(train_test_folds) do (train, test) + + # train using model-specific representation of data: + fitobs_subset = MLUtils.getobs(fitobs, train) + model = fit(learner, fitobs_subset) + + # predict on the fold complement: + if never_trained + X = LearnAPI.features(learner, data) + global predictobs = obs(model, X) + global never_trained = false + end + predictobs_subset = MLUtils.getobs(predictobs, test) + ŷ = predict(model, Point(), predictobs_subset) + + y = LearnAPI.target(learner, data) + return + +end +``` + +## Implementation guide + +| method | comment | compulsory? | fallback | +|:-------------------------------|:------------------------------------|:-------------:|:---------------| +| [`obs(learner, data)`](@ref) | here `data` is `fit`-consumable | not typically | returns `data` | +| [`obs(model, data)`](@ref) | here `data` is `predict`-consumable | not typically | returns `data` | + + +A sample implementation is given in [Providing a separate data front end](@ref). + + +## Reference + +```@docs +obs +``` + +### [Data interfaces](@id data_interfaces) + +New implementations must overload [`LearnAPI.data_interface(learner)`](@ref) if the +output of [`obs`](@ref) does not implement [`LearnAPI.RandomAccess()`](@ref). Arrays, most +tables, and all tuples thereof, implement `RandomAccess()`. + +- [`LearnAPI.RandomAccess`](@ref) (default) +- [`LearnAPI.FiniteIterable`](@ref) +- [`LearnAPI.Iterable`](@ref) + + +```@docs +LearnAPI.RandomAccess +LearnAPI.FiniteIterable +LearnAPI.Iterable +``` + diff --git a/docs/src/operations.md b/docs/src/operations.md deleted file mode 100644 index 1fc0f103..00000000 --- a/docs/src/operations.md +++ /dev/null @@ -1,114 +0,0 @@ -# [Predict and Other Operations](@id operations) - -> **Summary** An method delivering output for some algorithm which has finished learning, -> applied to (new) data, is called an **operation**. The output depends on the fitted -> parameters associated with the algorithm, which is `nothing` for non-generalizing -> algorithms. Implement the `predict` operation when the output is predictions of a target -> variable or, more generally a proxy for the target, such as probability distributions. -> Otherwise implement `transform` and, optionally `inverse_transform`. - -The methods `predict`, `transform` and `inverse_transform` are called *operations*. They -are all dispatched on an algorithm, fitted parameters and data. The `predict` operation -additionally includes a `::ProxyType` argument in position two. If [`LearnAPI.fit`](@ref) -is not implemented, then the fitted parameters will always be `nothing`. - -Here's a snippet of code with a `LearnAPI.predict` call: - -```julia -fitted_params, state, fit_report = LearnAPI.fit(some_algorithm, 1, X, y) -ŷ, predict_report = - LearnAPI.predict(some_algorithm, LearnAPI.LiteralTarget(), fitted_params, Xnew) -``` - -| method | compulsory? | fallback | requires | -|:-----------------------------------|:-----------:|:--------:|:-----------:| -[`LearnAPI.predict`](@ref) | no | none | | -[`LearnAPI.transform`](@ref) | no | none | | -[`LearnAPI.inverse_transform`](@ref) | no | none | `transform` | - - -## General requirements - -- Operations always return a tuple `(output, report)` where `output` is the usual output - (e.g., the target predictions if the operation is `predict`) and `report` - includes byproducts of the computation, typically `nothing` unless the algorithm does not - generalize to new data (does not implement `fit`). - -- If implementing a `predict` method, you must also make a - [`LearnAPI.preferred_kind_of_proxy`](@ref) declaration. - -- The name of each operation explicitly overloaded must be included in the return value - of the [`LearnAPI.functions`](@ref) trait. - -## Predict or transform? - -If the algorithm has a notion of [target variable](@ref proxy), then implement a `predict` -method for each supported kind of target proxy (`LiteralTarget()`, `Distribution()`, -etc). See [Target proxies](@ref) below. - -If an operation is to have an inverse operation, then it cannot be `predict` - use -`transform`, and (optionally) `inverse_transform`, for inversion, broadly understood. See -[`LearnAPI.inverse_transform`](@ref) below. - - -## Target proxies - -The concept of **target proxy** is defined under [Targets and target proxies](@ref -proxy). The available kinds of target proxy are classified by subtypes of -`LearnAPI.KindOfProxy`. These types are intended for dispatch only and have no fields. - -```@docs -LearnAPI.KindOfProxy -``` -```@docs -LearnAPI.IID -``` - -| type | form of an observation | -|:-------------------------------:|:----------------------------------------------------| -| `LearnAPI.None` | has no declared relationship with a target variable | -| `LearnAPI.LiteralTarget` | same as target observations | -| `LearnAPI.Sampleable` | object that can be sampled to obtain object of the same form as target observation | -| `LearnAPI.Distribution` | explicit probability density/mass function whose sample space is all possible target observations | -| `LearnAPI.LogDistribution` | explicit log-probability density/mass function whose sample space is possible target observations | -| † `LearnAPI.Probability` | raw numerical probability or probability vector | -| † `LearnAPI.LogProbability` | log-probability or log-probability vector | -| † `LearnAPI.Parametric` | a list of parameters (e.g., mean and variance) describing some distribution | -| `LearnAPI.LabelAmbiguous` | collections of labels (in case of multi-class target) but without a known correspondence to the original target labels (and of possibly different number) as in, e.g., clustering | -| `LearnAPI.LabelAmbiguousSampleable` | sampleable version of `LabelAmbiguous`; see `Sampleable` above | -| `LearnAPI.LabelAmbiguousDistribution`| pdf/pmf version of `LabelAmbiguous`; see `Distribution` above | -| `LearnAPI.ConfidenceInterval` | confidence interval (possible requirement: observation `isa Tuple{Real,Real}`) | -| `LearnAPI.Set` | finite but possibly varying number of target observations | -| `LearnAPI.ProbabilisticSet` | as for `Set` but labeled with probabilities (not necessarily summing to one) | -| `LearnAPI.SurvivalFunction` | survival function (possible requirement: observation is single-argument function mapping `Real` to `Real`) | -| `LearnAPI.SurvivalDistribution` | probability distribution for survival time | -| `LearnAPI.OutlierScore` | numerical score reflecting degree of outlierness (not necessarily normalized) | -| `LearnAPI.Continuous` | real-valued approximation/interpolation of a discrete-valued target, such as a count (e.g., number of phone calls) | - -† Provided for completeness but discouraged to avoid [ambiguities in -representation](https://github.com/alan-turing-institute/MLJ.jl/blob/dev/paper/paper.md#a-unified-approach-to-probabilistic-predictions-and-their-evaluation). - -> Table of concrete subtypes of `LearnAPI.IID <: LearnAPI.KindOfProxy`. - -In the following table of subtypes `T <: LearnAPI.KindOfProxy` not falling under the `IID` -umbrella, the first return value of `predict(algorithm, ::T, fitted_params, data...)` is -not divided into individual observations, but represents a *single* probability -distribution for the sample space ``Y^n``, where ``Y`` is the space the target variable -takes its values, and `n` is the number of observations in `data`. - -| type `T` | form of output of `predict(algorithm, ::T, fitted_params, data...)` | -|:-------------------------------:|:--------------------------------------------------------------------------| -| `LearnAPI.JointSampleable` | object that can be sampled to obtain a *vector* whose elements have the form of target observations; the vector length matches the number of observations in `data`. | -| `LearnAPI.JointDistribution` | explicit probability density/mass function whose sample space is vectors of target observations; the vector length matches the number of observations in `data` | -| `LearnAPI.JointLogDistribution` | explicit log-probability density/mass function whose sample space is vectors of target observations; the vector length matches the number of observations in `data` | - -> Table of `LearnAPI.KindOfProxy` subtypes not subtyping `LearnAPI.IID` - - -## Reference - -```@docs -LearnAPI.predict -LearnAPI.transform -LearnAPI.inverse_transform -``` diff --git a/docs/src/optional_data_interface.md b/docs/src/optional_data_interface.md deleted file mode 100644 index beccda9c..00000000 --- a/docs/src/optional_data_interface.md +++ /dev/null @@ -1,34 +0,0 @@ -# [Optional Data Interface](@id data_interface) - -> **Summary.** Implement `getobs` to articulate how to generate individual observations -> from data consumed by a LearnAPI algorithm. Implement `reformat` to provide a higher level -> interface the means to avoid repeating transformations from user representations of data -> (such as a dataframe) and algorithm-specific representations (such as a matrix). - -## Resampling - -To aid in programmatic resampling, such as cross-validation, it is helpful if each machine -learning algorithm articulates how the data it consumes can be subsampled - that is, how a -subset of observations can be extracted from that data. Another advantage of doing so is -to mitigate some of the ambiguities around structuring observations within the container: -Are the observations in a matrix the rows or the columns? - -In LearnAPI, an implementation can articulate a subsampling method by implementing -`LearnAPI.getobs(algorithm, func, I, data...)` for each function `func` consuming data, such -as `fit` and `predict`. Examples are given below. - -```@docs -LearnAPI.getobs -``` -## Preprocessing - -So that a higher level interface can avoid unnecessarily repeating calls to convert -user-supplied data (e.g., a dataframe) into some performant, algorithm-specific -representation, an algorithm can move such data conversions out of `fit`, `predict`, etc., and -into an implementation of `LearnAPI.reformat` created for each signature of such methods -that are implemented. Examples are given below. - -```@docs -LearnAPI.reformat -``` - diff --git a/docs/src/patterns/classification.md b/docs/src/patterns/classification.md new file mode 100644 index 00000000..fd278478 --- /dev/null +++ b/docs/src/patterns/classification.md @@ -0,0 +1,5 @@ +# Classification + +See these examples from the JuliaTestAPI.jl test suite: + +- [perceptron classifier](https://github.com/JuliaAI/LearnTestAPI.jl/blob/dev/test/patterns/gradient_descent.jl) diff --git a/docs/src/patterns/classifiers.md b/docs/src/patterns/classifiers.md deleted file mode 100644 index 3571bc78..00000000 --- a/docs/src/patterns/classifiers.md +++ /dev/null @@ -1 +0,0 @@ -# Classifiers diff --git a/docs/src/patterns/density_estimation.md b/docs/src/patterns/density_estimation.md new file mode 100644 index 00000000..9fc0144a --- /dev/null +++ b/docs/src/patterns/density_estimation.md @@ -0,0 +1,5 @@ +# Density Estimation + +See these examples from the JuliaTestAPI.jl test suite: + +- [normal distribution estimator](https://github.com/JuliaAI/LearnTestAPI.jl/blob/dev/test/patterns/incremental_algorithms.jl) diff --git a/docs/src/patterns/dimension_reduction.md b/docs/src/patterns/dimension_reduction.md index 3174adb8..e886dd15 100644 --- a/docs/src/patterns/dimension_reduction.md +++ b/docs/src/patterns/dimension_reduction.md @@ -1 +1,6 @@ # Dimension Reduction + +See these examples from the JuliaTestAPI.jl test suite: + +- [Truncated SVD](https://github.com/JuliaAI/LearnTestAPI.jl/blob/dev/test/patterns/dimension_reduction.jl) + diff --git a/docs/src/patterns/ensembling.md b/docs/src/patterns/ensembling.md new file mode 100644 index 00000000..8d774f5e --- /dev/null +++ b/docs/src/patterns/ensembling.md @@ -0,0 +1,7 @@ +# Ensembling + +See these examples from the JuliaTestAPI.jl test suite: + +- [bagged ensembling of a regression model](https://github.com/JuliaAI/LearnTestAPI.jl/blob/dev/test/patterns/ensembling.jl) + +- [extremely randomized ensemble of decision stumps (regression)](https://github.com/JuliaAI/LearnTestAPI.jl/blob/dev/test/patterns/ensembling.jl) diff --git a/docs/src/patterns/feature_engineering.md b/docs/src/patterns/feature_engineering.md new file mode 100644 index 00000000..6e3c656c --- /dev/null +++ b/docs/src/patterns/feature_engineering.md @@ -0,0 +1,7 @@ +# Feature Engineering + +See these examples from the JuliaTestAI.jl test suite: + +- [feature + selectors](https://github.com/JuliaAI/LearnTestAPI.jl/blob/dev/test/patterns/static_algorithms.jl) + from tests. diff --git a/docs/src/patterns/gradient_descent.md b/docs/src/patterns/gradient_descent.md new file mode 100644 index 00000000..9dc5401a --- /dev/null +++ b/docs/src/patterns/gradient_descent.md @@ -0,0 +1,5 @@ +# Gradient Descent + +See these examples from the JuliaTestAI.jl test suite: + +- [perceptron classifier](https://github.com/JuliaAI/LearnTestAPI.jl/blob/dev/test/patterns/gradient_descent.jl) diff --git a/docs/src/patterns/incremental_algorithms.md b/docs/src/patterns/incremental_algorithms.md index 54095fa5..d2855a55 100644 --- a/docs/src/patterns/incremental_algorithms.md +++ b/docs/src/patterns/incremental_algorithms.md @@ -1 +1,5 @@ -# Incremental Models +# Incremental Algorithms + +See these examples from the JuliaTestAI.jl test suite: + +- [normal distribution estimator](https://github.com/JuliaAI/LearnTestAPI.jl/blob/dev/test/patterns/incremental_algorithms.jl) diff --git a/docs/src/patterns/incremental_models.md b/docs/src/patterns/incremental_models.md deleted file mode 100644 index 2876c57f..00000000 --- a/docs/src/patterns/incremental_models.md +++ /dev/null @@ -1 +0,0 @@ -# Incremental Algorithms diff --git a/docs/src/patterns/iterative_algorithms.md b/docs/src/patterns/iterative_algorithms.md index fafd1b7e..265dddf7 100644 --- a/docs/src/patterns/iterative_algorithms.md +++ b/docs/src/patterns/iterative_algorithms.md @@ -1 +1,9 @@ # Iterative Algorithms + +See these examples from the JuliaTestAI.jl test suite: + +- [bagged ensembling](https://github.com/JuliaAI/LearnTestAPI.jl/blob/dev/test/patterns/ensembling.jl) + +- [perceptron classifier](https://github.com/JuliaAI/LearnTestAPI.jl/blob/dev/test/patterns/gradient_descent.jl) + +- [extremely randomized ensemble of decision stumps (regression)](https://github.com/JuliaAI/LearnTestAPI.jl/blob/dev/test/patterns/ensembling.jl) diff --git a/docs/src/patterns/learning_a_probability_distribution.md b/docs/src/patterns/learning_a_probability_distribution.md deleted file mode 100644 index 19a53b86..00000000 --- a/docs/src/patterns/learning_a_probability_distribution.md +++ /dev/null @@ -1 +0,0 @@ -# Learning a Probability Distribution diff --git a/docs/src/patterns/meta_algorithms.md b/docs/src/patterns/meta_algorithms.md new file mode 100644 index 00000000..6a9e7300 --- /dev/null +++ b/docs/src/patterns/meta_algorithms.md @@ -0,0 +1,7 @@ +# Meta-algorithms + +Many meta-algorithms are can be implemented as wrappers. An example is [this bagged +ensemble +algorithm](https://github.com/JuliaAI/LearnTestAPI.jl/blob/dev/test/patterns/ensembling.jl) +from tests. + diff --git a/docs/src/patterns/regression.md b/docs/src/patterns/regression.md new file mode 100644 index 00000000..a6de5b10 --- /dev/null +++ b/docs/src/patterns/regression.md @@ -0,0 +1,7 @@ +# Regression + +See these examples from the JuliaTestAPI.jl test suite: + +- [ridge regression](https://github.com/JuliaAI/LearnTestAPI.jl/blob/dev/test/patterns/regression.jl) + +- [extremely randomized ensemble of decision stumps](https://github.com/JuliaAI/LearnTestAPI.jl/blob/dev/test/patterns/ensembling.jl) diff --git a/docs/src/patterns/regressors.md b/docs/src/patterns/regressors.md deleted file mode 100644 index 0b6163f6..00000000 --- a/docs/src/patterns/regressors.md +++ /dev/null @@ -1 +0,0 @@ -# Regressors diff --git a/docs/src/patterns/static_algorithms.md b/docs/src/patterns/static_algorithms.md new file mode 100644 index 00000000..4724006f --- /dev/null +++ b/docs/src/patterns/static_algorithms.md @@ -0,0 +1,9 @@ +# Static Algorithms + +See these examples from the JuliaTestAI.jl test suite: + +- [feature + selection](https://github.com/JuliaAI/LearnTestAPI.jl/blob/dev/test/patterns/static_algorithms.jl) + + + diff --git a/docs/src/patterns/static_transformers.md b/docs/src/patterns/static_transformers.md deleted file mode 100644 index f413050b..00000000 --- a/docs/src/patterns/static_transformers.md +++ /dev/null @@ -1 +0,0 @@ -# Static Transformers diff --git a/docs/src/patterns/transformers.md b/docs/src/patterns/transformers.md new file mode 100644 index 00000000..f085f928 --- /dev/null +++ b/docs/src/patterns/transformers.md @@ -0,0 +1,7 @@ +# [Transformers](@id transformers) + +Check out the following examples: + +- [Truncated + SVD]((https://github.com/JuliaAI/LearnTestAPI.jl/blob/dev/test/patterns/dimension_reduction.jl + (from the TestLearnAPI.jl test suite) diff --git a/docs/src/predict_transform.md b/docs/src/predict_transform.md new file mode 100644 index 00000000..d6ab8f25 --- /dev/null +++ b/docs/src/predict_transform.md @@ -0,0 +1,116 @@ +# [`predict`, `transform` and `inverse_transform`](@id operations) + +```julia +predict(model, kind_of_proxy, data) +transform(model, data) +inverse_transform(model, data) +``` + +Versions without the `data` argument may apply, for example in [density +estimation](@ref density_estimation). + +## [Typical worklows](@id predict_workflow) + +Train some supervised `learner`: + +```julia +model = fit(learner, (X, y)) +``` + +Predict probability distributions: + +```julia +ŷ = predict(model, Distribution(), Xnew) +``` + +Generate point predictions: + +```julia +ŷ = predict(model, Point(), Xnew) +``` + +Train a dimension-reducing `learner`: + +```julia +model = fit(learner, X) +Xnew_reduced = transform(model, Xnew) +``` + +Apply an approximate right inverse: + +```julia +inverse_transform(model, Xnew_reduced) +``` + +Fit and transform in one line: + +```julia +transform(learner, data) # `fit` implied +``` + +### An advanced workflow + +```julia +fitobs = obs(learner, (X, y)) # learner-specific repr. of data +model = fit(learner, MLUtils.getobs(fitobs, 1:100)) +predictobs = obs(model, MLUtils.getobs(X, 101:150)) +ŷ = predict(model, Point(), predictobs) +``` + + +## [Implementation guide](@id predict_guide) + +| method | compulsory? | fallback | +|:----------------------------|:-----------:|:--------:| +| [`predict`](@ref) | no | none | +| [`transform`](@ref) | no | none | +| [`inverse_transform`](@ref) | no | none | + +### Predict or transform? + +If the learner has a notion of [target variable](@ref proxy), then use +[`predict`](@ref) to output each supported [kind of target proxy](@ref +proxy_types) (`Point()`, `Distribution()`, etc). + +For output not associated with a target variable, implement [`transform`](@ref) +instead, which does not dispatch on [`LearnAPI.KindOfProxy`](@ref), but can be optionally +paired with an implementation of [`inverse_transform`](@ref), for returning (approximate) +right or left inverses to `transform`. + +Of course, the one learner can implement both a `predict` and `transform` method. For +example a K-means clustering algorithm can `predict` labels and `transform` to reduce +dimension using distances from the cluster centres. + + +### [One-liners combining fit and transform/predict](@id one_liners) + +Learners may additionally overload `transform` to apply `fit` first, using the supplied +data if required, and then immediately `transform` the same data. In that case the first +argument of `transform` is a *learner* instead of the output of `fit`: + +```julia +transform(learner, data) # `fit` implied +``` + +This will be shorthand for + +```julia +model = fit(learner, X) # or `fit(learner)` in the static case +transform(model, X) +``` + +The same remarks apply to `predict`, as in + +```julia +predict(learner, kind_of_proxy, data) # `fit` implied +``` + +LearnAPI.jl does not, however, guarantee the provision of these one-liners. + +## [Reference](@id predict_ref) + +```@docs +predict +transform +inverse_transform +``` diff --git a/docs/src/reference.md b/docs/src/reference.md index 1298073f..f068afc7 100644 --- a/docs/src/reference.md +++ b/docs/src/reference.md @@ -1,104 +1,216 @@ # [Reference](@id reference) -> **Summary** In LearnAPI.jl an **algorithm** is a container for hyperparameters of some -> ML/Statistics algorithm (which may or may not "learn"). Functionality is created by -> overloading **methods** provided by the interface, which are divided into training -> methods (e.g., `fit`), operations (e.g.,. `predict` and `transform`) and accessor -> functions (e.g., `feature_importances`). Promises of particular behavior are articulated -> by **algorithm traits**. +Here we give the definitive specification of the LearnAPI.jl interface. For informal +guides see [Anatomy of an Implementation](@ref) and [Common Implementation +Patterns](@ref patterns). -Here we give the definitive specification of the interface provided by LearnAPI.jl. For a -more informal guide see [Anatomy of an Implementation](@ref) and [Common Implementation Patterns](@ref). -!!! important +## [Important terms and concepts](@id scope) - The reader is assumed to be familiar with the LearnAPI-specific meanings of the following terms, as outlined in - [Scope and undefined notions](@ref scope): **data**, **metadata**, - **hyperparameter**, **observation**, **target**, **target proxy**. - -## Algorithms +The LearnAPI.jl specification is predicated on a few basic, informally defined notions: -In LearnAPI.jl an **algorithm** is some julia object `alg` storing the hyperparameters of -some MLJ/statistics algorithm that transforms data in some way. Typically the algorithm -"learns" from data in a training event, but this is not essential; "static" data -processing, with parameters, is included. -The type of `alg` will have a name reflecting that of the algorithm, such as -`DecisionTreeRegressor`. +### Data and observations -Additionally, for `alg::Alg` to be a LearnAPI algorithm, we require: +ML/statistical algorithms are typically applied in conjunction with resampling of +*observations*, as in +[cross-validation](https://en.wikipedia.org/wiki/Cross-validation_(statistics)). In this +document *data* will always refer to objects encapsulating an ordered sequence of +individual observations. -- `Base.propertynames(alg)` returns the hyperparameters of `alg`. +A `DataFrame` instance, from [DataFrames.jl](https://dataframes.juliadata.org/stable/), is +an example of data, the observations being the rows. Typically, data provided to +LearnAPI.jl algorithms, will implement the +[MLUtils.jl](https://juliaml.github.io/MLUtils.jl/stable) `getobs/numobs` interface for +accessing individual observations, but implementations can opt out of this requirement; +see [`obs`](@ref) and [`LearnAPI.data_interface`](@ref) for details. -- If `alg` is an algorithm, then so are all instances of the same type. +!!! note -- If `_alg` is another algorithm, then `alg == _alg` if and only if `typeof(alg) == typeof(_alg)` and - corresponding properties are `==`. This includes properties that are random number - generators (which should be copied in training to avoid mutation). + In the MLUtils.jl + convention, observations in tables are the rows but observations in a matrix are the + columns. -- If an algorithm has other algorithms as hyperparameters, then [`LearnAPI.is_wrapper`](@ref)`(alg)` - must be `true`. +### [Hyperparameters](@id hyperparameters) -- A keyword constructor for `Alg` exists, providing default values for *all* non-algorithm - hyperparameters. +Besides the data it consumes, a machine learning algorithm's behavior is governed by a +number of user-specified *hyperparameters*, such as the number of trees in a random +forest. In LearnAPI.jl, one is allowed to have hyperparameters that are not data-generic. +For example, a class weight dictionary, which will only make sense for a target taking +values in the set of dictionary keys, can be specified as a hyperparameter. -Whenever any LearnAPI method (excluding traits) is overloaded for some type `Alg` (e.g., -`predict`, `transform`, `fit`) then that is a promise that all instances of `Alg` are -algorithms (and the trait [`LearnAPI.functions`](@ref)`(Alg)` will be non-empty). -It is supposed that making copies of algorithm objects is a cheap operation. Consequently, -*learned* parameters, such as weights in a neural network (the `fitted_params` described -in [Fit, update! and ingest!](@ref)) should not be stored in the algorithm object. Storing -learned parameters in an algorithm is not explicitly ruled out, but doing so might lead to -performance issues in packages adopting LearnAPI.jl. +### [Targets and target proxies](@id proxy) +#### Context -### Example +After training, a supervised classifier predicts labels on some input which are then +compared with ground truth labels using some accuracy measure, to assess the performance +of the classifier. Alternatively, the classifier predicts class probabilities, which are +instead paired with ground truth labels using a proper scoring rule, say. In outlier +detection, "outlier"/"inlier" predictions, or probability-like scores, are similarly +compared with ground truth labels. In clustering, integer labels assigned to observations +by the clustering algorithm can can be paired with human labels using, say, the Rand +index. In survival analysis, predicted survival functions or probability distributions are +compared with censored ground truth survival times. And so on ... -Any instance of `GradientRidgeRegressor` defined below is a valid LearnAPI.jl algorithm: +#### Definitions -```julia -struct GradientRidgeRegressor{T<:Real} <: LearnAPI.Algorithm - learning_rate::T - epochs::Int - l2_regularization::T -end -``` +More generally, whenever we have a variable (e.g., a class label) that can, at least in +principle, be paired with a predicted value, or some predicted "proxy" for that variable +(such as a class probability), then we call the variable a *target* variable, and the +predicted output a *target proxy*. In this definition, it is immaterial whether or not the +target appears in training (the algorithm is supervised) or whether or not predictions +generalize to new input observations (the algorithm "learns"). -The same is true if we omit the subtyping `<: LearnAPI.Algorithm`, but not if we also make -this a `mutable struct`. In that case we will need to overload `Base.==` for -`GradientRidgeRegressor`. +LearnAPI.jl provides singleton [target proxy types](@ref proxy_types) for prediction +dispatch. These are also used to distinguish performance metrics provided by the package +[StatisticalMeasures.jl](https://juliaai.github.io/StatisticalMeasures.jl/dev/). -```@docs -LearnAPI.Algorithm -``` -## Methods +### [Learners](@id learners) + +An object implementing the LearnAPI.jl interface is called a *learner*, although it is +more accurately "the configuration of some machine learning or statistical algorithm".¹ A +learner encapsulates a particular set of user-specified [hyperparameters](@ref) as the +object's *properties* (which conceivably differ from its fields). It does not store +learned parameters. + +Informally, we will sometimes use the word "model" to refer to the output of +`fit(learner, ...)` (see below), something which typically *does* store learned +parameters. -None of the methods described in the linked sections below are compulsory, but any -implemented or overloaded method that is not an algorithm trait must be added to the return -value of [`LearnAPI.functions`](@ref), as in +For every `learner`, [`LearnAPI.constructor(learner)`](@ref) must return a keyword +constructor enabling recovery of `learner` from its properties: ```julia -LearnAPI.functions(::Type{ +LearnAPI.weights(learner, observations) -> +LearnAPI.features(learner, observations) -> +``` + +Here `data` is something supported in a call of the form `fit(learner, data)`. + +# Typical workflow + +Not typically appearing in a general user's workflow but useful in meta-alagorithms, such +as cross-validation (see the example in [`obs` and Data Interfaces](@ref data_interface)). + +Supposing `learner` is a supervised classifier predicting a one-dimensional vector +target: + +```julia +observations = obs(learner, data) +model = fit(learner, observations) +X = LearnAPI.features(learner, data) +y = LearnAPI.target(learner, data) +ŷ = predict(model, Point(), X) +training_loss = sum(ŷ .!= y) +``` + +# Implementation guide + +| method | fallback | compulsory? | +|:----------------------------|:-----------------:|--------------------------| +| [`LearnAPI.target`](@ref) | returns `nothing` | no | +| [`LearnAPI.weights`](@ref) | returns `nothing` | no | +| [`LearnAPI.features`](@ref) | see docstring | if fallback insufficient | + + +# Reference + +```@docs +LearnAPI.target +LearnAPI.weights +LearnAPI.features +``` diff --git a/docs/src/testing_an_implementation.md b/docs/src/testing_an_implementation.md index 61da2512..449a031c 100644 --- a/docs/src/testing_an_implementation.md +++ b/docs/src/testing_an_implementation.md @@ -1 +1,9 @@ # Testing an Implementation + +```@raw html +🚧 +``` + +!!! warning + + Under construction diff --git a/docs/src/traits.md b/docs/src/traits.md new file mode 100644 index 00000000..a95404bf --- /dev/null +++ b/docs/src/traits.md @@ -0,0 +1,115 @@ +# [Learner Traits](@id traits) + +Learner traits are simply functions whose sole argument is a learner. + +Traits promise specific learner behavior, such as: *This learner can make point or +probabilistic predictions* or *This learner is supervised* (sees a target in +training). They may also record more mundane information, such as a package license. + +## [Trait summary](@id trait_summary) + +### [Overloadable traits](@id traits_list) + +In the examples column of the table below, `Continuous` is a name owned the package +[ScientificTypesBase.jl](https://github.com/JuliaAI/ScientificTypesBase.jl/). + +| trait | return value | fallback value | example | +|:---------------------------------------------------------|:-----------------------------------------------------------------------------------------------------------------------|:------------------------------------------------------|:---------------------------------------------------------------| +| [`LearnAPI.constructor`](@ref)`(learner)` | constructor for generating new or modified versions of `learner` | (no fallback) | `RidgeRegressor` | +| [`LearnAPI.functions`](@ref)`(learner)` | functions you can apply to `learner` or associated model (traits excluded) | `()` | `(:fit, :predict, :LearnAPI.strip, :(LearnAPI.learner), :obs)` | +| [`LearnAPI.kinds_of_proxy`](@ref)`(learner)` | instances `kind` of `KindOfProxy` for which an implementation of `LearnAPI.predict(learner, kind, ...)` is guaranteed. | `()` | `(Distribution(), Interval())` | +| [`LearnAPI.tags`](@ref)`(learner)` | lists one or more suggestive learner tags from `LearnAPI.tags()` | `()` | (:regression, :probabilistic) | +| [`LearnAPI.is_pure_julia`](@ref)`(learner)` | `true` if implementation is 100% Julia code | `false` | `true` | +| [`LearnAPI.pkg_name`](@ref)`(learner)` | name of package providing core code (may be different from package providing LearnAPI.jl implementation) | `"unknown"` | `"DecisionTree"` | +| [`LearnAPI.pkg_license`](@ref)`(learner)` | name of license of package providing core code | `"unknown"` | `"MIT"` | +| [`LearnAPI.doc_url`](@ref)`(learner)` | url providing documentation of the core code | `"unknown"` | `"https://en.wikipedia.org/wiki/Decision_tree_learning"` | +| [`LearnAPI.load_path`](@ref)`(learner)` | string locating name returned by `LearnAPI.constructor(learner)`, beginning with a package name | `"unknown"` | `FastTrees.LearnAPI.DecisionTreeClassifier` | +| [`LearnAPI.nonlearners`](@ref)`(learner)` | properties *not* corresponding to other learners | all properties | `(:K, :leafsize, :metric,)` | +| [`LearnAPI.human_name`](@ref)`(learner)` | human name for the learner; should be a noun | type name with spaces | "elastic net regressor" | +| [`LearnAPI.iteration_parameter`](@ref)`(learner)` | symbolic name of an iteration parameter | `nothing` | :epochs | +| [`LearnAPI.data_interface`](@ref)`(learner)` | Interface implemented by objects returned by [`obs`](@ref) | `Base.HasLength()` (supports `MLUtils.getobs/numobs`) | `Base.SizeUnknown()` (supports `iterate`) | +| [`LearnAPI.fit_observation_scitype`](@ref)`(learner)` | upper bound on `scitype(observation)` for `observation` in `data` ensuring `fit(learner, data)` works | `Union{}` | `Tuple{AbstractVector{Continuous}, Continuous}` | +| [`LearnAPI.target_observation_scitype`](@ref)`(learner)` | upper bound on the scitype of each observation of the targget | `Any` | `Continuous` | +| [`LearnAPI.is_static`](@ref)`(learner)` | `true` if `fit` consumes no data | `false` | `true` | + +### Derived Traits + +The following are provided for convenience but should not be overloaded by new learners: + +| trait | return value | example | +|:-------------------------------|:-------------------------------------------------------------------------|:--------------| +| `LearnAPI.name(learner)` | learner type name as string | "PCA" | +| `LearnAPI.learners(learner)` | properties with learner values | `(:atom, )` | +| `LearnAPI.is_learner(learner)` | `true` if `learner` is LearnAPI.jl-compliant | `true` | +| `LearnAPI.target(learner)` | `true` if `fit` sees a target variable; see [`LearnAPI.target`](@ref) | `false` | +| `LearnAPI.weights(learner)` | `true` if `fit` supports per-observation; see [`LearnAPI.weights`](@ref) | `false` | + +## Implementation guide + +Only `LearnAPI.constructor` and `LearnAPI.functions` are universally compulsory. + +A single-argument trait is declared following this pattern: + +```julia +LearnAPI.is_pure_julia(learner::MyLearnerType) = true +``` + +A macro [`@trait`](@ref) provides a short-cut: + +```julia +@trait MyLearnerType is_pure_julia=true +``` + +Multiple traits can be declared like this: + + +```julia +@trait( + MyLearnerType, + is_pure_julia = true, + pkg_name = "MyPackage", +) +``` + +### [The global trait contract](@id trait_contract) + +To ensure that trait metadata can be stored in an external learner registry, LearnAPI.jl +requires: + +1. *Finiteness:* The value of a trait is the same for all `learner`s with same value of + [`LearnAPI.constructor(learner)`](@ref). This typically means trait values do not + depend on type parameters! For composite models (`LearnAPI.learners(learner)` + non-empty) this requirement is dropped. + +2. *Low level deserializability:* It should be possible to evaluate the trait *value* when + `LearnAPI` is the only imported module. + +Because of 1, combining a lot of functionality into one learner (e.g. the learner can +perform both classification or regression) can mean traits are necessarily less +informative (as in `LearnAPI.target_observation_scitype(learner) = Any`). + + +## Reference + +```@docs +LearnAPI.constructor +LearnAPI.functions +LearnAPI.kinds_of_proxy +LearnAPI.tags +LearnAPI.is_pure_julia +LearnAPI.pkg_name +LearnAPI.pkg_license +LearnAPI.doc_url +LearnAPI.load_path +LearnAPI.nonlearners +LearnAPI.human_name +LearnAPI.data_interface +LearnAPI.iteration_parameter +LearnAPI.fit_observation_scitype +LearnAPI.target_observation_scitype +LearnAPI.is_static +``` + +```@docs +LearnAPI.learners +``` diff --git a/src/LearnAPI.jl b/src/LearnAPI.jl index 7b2dd1de..9687c2e9 100644 --- a/src/LearnAPI.jl +++ b/src/LearnAPI.jl @@ -1,16 +1,22 @@ module LearnAPI -using Statistics -import InteractiveUtils.subtypes - +include("types.jl") +include("verbosity.jl") include("tools.jl") -include("algorithms.jl") -include("operations.jl") -include("fit_update_ingest.jl") +include("predict_transform.jl") +include("fit_update.jl") +include("target_weights_features.jl") +include("obs.jl") include("accessor_functions.jl") -include("data_interface.jl") -include("algorithm_traits.jl") +include("traits.jl") +include("clone.jl") -export @trait +export @trait, @functions, clone +export fit, update, update_observations, update_features +export predict, transform, inverse_transform, obs +for name in CONCRETE_TARGET_PROXY_SYMBOLS + @eval export $name end + +end # module diff --git a/src/accessor_functions.jl b/src/accessor_functions.jl index 6aaf9531..b5f487b4 100644 --- a/src/accessor_functions.jl +++ b/src/accessor_functions.jl @@ -1,83 +1,435 @@ -const ACCESSOR_FUNCTIONS = ( - :features_importances, - :training_labels, - :training_losses, - :training_scores, -) +# # NOTE ON ADDING NEW ACCESSOR FUNCTIONS + +# - Add new accessor function to ACCESSOR_FUNCTIONS_WITHOUT_EXTRAS, +# defined near the end of this file. + +# - Update the documentation page /docs/src/accesssor_functions.md + + +const DOC_STATIC = + """ + + For "static" learners (those without training `data`) it may be necessary to first + call `transform` or `predict` on `model`. + + """ + +""" + LearnAPI.learner(model) + LearnAPI.learner(stripped_model) + +Recover the learner used to train `model` or the output, `stripped_model`, of +[`LearnAPI.strip(model)`](@ref). + +In other words, if `model = fit(learner, data...)`, for some `learner` and `data`, +then + +```julia +LearnAPI.learner(model) == learner == LearnAPI.learner(LearnAPI.strip(model)) +``` +is `true`. + +# New implementations + +Implementation is compulsory for new learner types. The behaviour described above is the +only contract. You must include `:(LearnAPI.learner)` in the return value of +[`LearnAPI.functions(learner)`](@ref). + +""" +function learner end + +""" + LearnAPI.strip(model; options...) + +Return a version of `model` that will generally have a smaller memory allocation than +`model`, suitable for serialization. Here `model` is any object returned by +[`fit`](@ref). Accessor functions that can be called on `model` may not work on +`LearnAPI.strip(model)`, but [`predict`](@ref), [`transform`](@ref) and +[`inverse_transform`](@ref) will work, if implemented. Check +`LearnAPI.functions(LearnAPI.learner(model))` to view see what the original `model` +implements. + +Implementations may provide learner-specific keyword `options` to control how much of the +original functionality is preserved by `LearnAPI.strip`. + +# Typical workflow + +```julia +model = fit(learner, (X, y)) # or `fit(learner, X, y)` +ŷ = predict(model, Point(), Xnew) + +small_model = LearnAPI.strip(model) +serialize("my_model.jls", small_model) + +recovered_model = deserialize("my_random_forest.jls") +@assert predict(recovered_model, Point(), Xnew) == ŷ +``` + +# Extended help + +# New implementations + +Overloading `LearnAPI.strip` for new learners is optional. The fallback is the +identity. + +New implementations must enforce the following identities, whenever the right-hand side is +defined: + +```julia +predict(LearnAPI.strip(model; options...), args...; kwargs...) == + predict(model, args...; kwargs...) +transform(LearnAPI.strip(model; options...), args...; kwargs...) == + transform(model, args...; kwargs...) +inverse_transform(LearnAPI.strip(model; options), args...; kwargs...) == + inverse_transform(model, args...; kwargs...) +``` + +Additionally: + +```julia +LearnAPI.strip(LearnAPI.strip(model)) == LearnAPI.strip(model) +``` + +""" +LearnAPI.strip(model) = model + +""" + LearnAPI.feature_names(model) + +Where supported, return the names of features encountered when fitting or updating some +`learner` to obtain `model`. + +The value returned value is a vector of symbols. + +This method is implemented if `:(LearnAPI.feature_names) in LearnAPI.functions(learner)`. + +See also [`fit`](@ref). + +# New implementations + +$(DOC_IMPLEMENTED_METHODS(":(LearnAPI.feature_names)")). """ - LearnAPI.feature_importances(algorithm, fitted_params, report) +function feature_names end -Return the algorithm-specific feature importances of `algorithm`, given `fitted_params` and -`report`, as returned by [`LearnAPI.fit`](@ref), [`LearnAPI.update!`](@ref) or -[`LearnAPI.ingest!`](@ref). The value returned has the form of an abstract vector of -`feature::Symbol => importance::Real` pairs (e.g `[:gender => 0.23, :height => 0.7, :weight -=> 0.1]`). +""" + LearnAPI.feature_importances(model) -The `algorithm` supports feature importances if `:feature_importance in -LearnAPI.functions(algorithm)`. +Where supported, return the learner-specific feature importances of a `model` output by +[`fit`](@ref)`(learner, ...)` for some `learner`. The value returned has the form of an +abstract vector of `feature::Symbol => importance::Real` pairs (e.g `[:gender => 0.23, +:height => 0.7, :weight => 0.1]`). -If an algorithm is sometimes unable to report feature importances then -`feature_importances` will return all importances as 0.0, as in `[:gender => 0.0, :height -=> 0.0, :weight => 0.0]`. +The `learner` supports feature importances if `:(LearnAPI.feature_importances) in +LearnAPI.functions(learner)`. + +If a learner is sometimes unable to report feature importances then +`LearnAPI.feature_importances` will return all importances as 0.0, as in `[:gender => 0.0, +:height => 0.0, :weight => 0.0]`. # New implementations -`LearnAPI.feature_importances(algorithm::SomeAlgorithmType, fitted_params, report)` may be -overloaded for any type `SomeAlgorithmType` whose instances are algorithms in the LearnAPI -sense. If an algorithm can report multiple feature importance types, then the specific type to -be reported should be controlled by a hyperparameter (i.e., by some property of `algorithm`). +Implementation is optional. -$(DOC_IMPLEMENTED_METHODS(:feature_importances)). +$(DOC_IMPLEMENTED_METHODS(":(LearnAPI.feature_importances)")). """ function feature_importances end """ - training_losses(algorithm, fitted_params, report) + LearnAPI.coefficients(model) + +For a linear model, return the learned coefficients. The value returned has the form of +an abstract vector of `feature_or_class::Symbol => coefficient::Real` pairs (e.g `[:gender +=> 0.23, :height => 0.7, :weight => 0.1]`) or, in the case of multi-targets, +`feature::Symbol => coefficients::AbstractVector{<:Real}` pairs. -Return the training losses for `algorithm`, given `fitted_params` and -`report`, as returned by [`LearnAPI.fit`](@ref), [`LearnAPI.update!`](@ref) or -[`LearnAPI.ingest!`](@ref). +The `model` reports coefficients if `:(LearnAPI.coefficients) in +LearnAPI.functions(Learn.learner(model))`. + +See also [`LearnAPI.intercept`](@ref). # New implementations -Implement for iterative algorithms that compute and record training losses as part of training -(e.g. neural networks). +Implementation is optional. + +$(DOC_IMPLEMENTED_METHODS(":(LearnAPI.coefficients)")). + +""" +function coefficients end + +""" + LearnAPI.intercept(model) + +For a linear model, return the learned intercept. The value returned is `Real` (single +target) or an `AbstractVector{<:Real}` (multi-target). + +The `model` reports intercept if `:(LearnAPI.intercept) in +LearnAPI.functions(Learn.learner(model))`. + +See also [`LearnAPI.coefficients`](@ref). -$(DOC_IMPLEMENTED_METHODS(:training_losses)). +# New implementations + +Implementation is optional. + +$(DOC_IMPLEMENTED_METHODS(":(LearnAPI.intercept)")). + +""" +function intercept end + +""" + LearnAPI.tree(model) + +Return a user-friendly `tree`, implementing the AbstractTrees.jl interface. In particular, +such a tree can be visualized using `AbstractTrees.print_tree(tree)` or using the +TreeRecipe.jl package. + +See also [`LearnAPI.trees`](@ref). + +# New implementations + +Implementation is optional. The returned object should implement the following interface +defined in AbstractTrees.jl: + +- `tree` subtypes `AbstractTrees.AbstractNode{T}` + +- `AbstractTrees.children(tree)` + +- `AbstractTrees.printnode(tree)` should be human-readable + +$(DOC_IMPLEMENTED_METHODS(":(LearnAPI.tree)")). + +""" +function tree end + +""" + LearnAPI.trees(model) + +For tree ensemble model, return a vector of trees, each implementing the AbstractTrees.jl +interface. + +See also [`LearnAPI.tree`](@ref). + +# New implementations + +Implementation is optional. See [`LearnAPI.tree`](@ref) for the interface each tree in the +ensemble should implement. + +$(DOC_IMPLEMENTED_METHODS(":(LearnAPI.trees)")). + +""" +function trees end + +""" + LearnAPI.training_losses(model) + +Return internally computed training losses obtained when running `model = fit(learner, +...)` for some `learner`, one for each iteration of the algorithm. This will be a +numerical vector. The metric used to compute the loss is generally learner-specific, but +may be a user-specifiable learner hyperparameter. Generally, the smaller the loss, the +better the performance. + +See also [`fit`](@ref). + +# New implementations + +Implement for iterative algorithms that compute measures of training performance as part +of training (e.g. neural networks). Return one value per iteration, in chronological +order, with an optional pre-training initial value. If scores are being computed rather +than losses, ensure values are multiplied by -1. + +$(DOC_IMPLEMENTED_METHODS(":(LearnAPI.training_losses)")). """ function training_losses end """ - training_scores(algorithm, fitted_params, report) + LearnAPI.out_of_sample_losses(model) + +Where supported, return internally computed out-of-sample losses obtained when running +`model = fit(learner, ...)` for some `learner`, one for each iteration of the +algorithm. This will be a numeric vector. The metric used to compute the loss is generally +learner-specific, but may be a user-specifiable learner hyperparameter. Generally, the +smaller the loss, the better the performance. -Return the training scores for `algorithm`, given `fitted_params` and -`report`, as returned by [`LearnAPI.fit`](@ref), [`LearnAPI.update!`](@ref) or -[`LearnAPI.ingest!`](@ref). +If the learner is not setting aside a separate validation set, then the losses are all +`Inf`. + +See also [`fit`](@ref). + +# New implementations + +Only implement this method for learners that specifically allow for the supplied training +data to be internally split into separate "train" and "validation" subsets, and which +additionally compute an out-of-sample loss. Return one value per iteration, in +chronological order, with an optional pre-training initial value. If scores are being +computed rather than losses, ensure values are multiplied by -1. + +$(DOC_IMPLEMENTED_METHODS(":(LearnAPI.out_of_sample_losses)")). + +""" +function out_of_sample_losses end + +""" + LearnAPI.predictions(model) + +Where supported, return internally computed predictions on the training `data` after +running `model = fit(learner, data)` for some `learner`. Semantically equivalent to calling +`LearnAPI.predict(model, X)`, where `X = LearnAPI.features(obs(learner, data))` but +generally cheaper. + +See also [`fit`](@ref). # New implementations -Implement for algorithms, such as outlier detection algorithms, which associate a score with each -observation during training, where these scores are of interest in later processes (e.g, in -defining normalized scores on new data). +Implement for algorithms that internally compute predictions for the training +data. Predictions for the complete test data must be returned, even if only a subset is +internally used for training. Cannot be implemented for static algorithms (algorithms for +which `fit` consumes no data). Here are some possible use cases: + +- Clustering algorithms that generalize to new data, but by first learning labels for the + training data (e.g., K-means); use `predictions(model)` to expose these labels + to the user so they can avoid the expense of a separate `predict` call. + +- Iterative learners such as neural networks, that need to make in-sample predictions + to estimate to estimate an in-sample loss; use `predictions(model)` + to expose these predictions to the user so they can avoid a separate `predict` call. + +- Ensemble learners, such as gradient tree boosting algorithms, may split the training + data into internal train and validation subsets and can efficiently build up predictions + on both with an update for each new ensemble member; expose these predictions to the + user (for external iteration control, for example) using `predictions(model)` and + articulate the actual split used using [`LearnAPI.out_of_sample_indices(model)`](@ref). -$(DOC_IMPLEMENTED_METHODS(:training_scores)). +$(DOC_IMPLEMENTED_METHODS(":(LearnAPI.predictions)")). + +""" +function predictions end + +""" + LearnAPI.out_of_sample_indices(model) + +For a learner also implementing [`LearnAPI.predictions`](@ref), return a vector of +observation indices identifying which part, if any, of `yhat = +LearnAPI.predictions(model)`, is actually out-of-sample predictions. If the learner +trained on all data this will be an empty vector. + +Here's a sample workflow for some such `learner`, with training data, `(X, y)`, where `y` +is the training target, here assumed to be a vector. + +```julia +import MLUtils.getobs +model = fit(learner, (X, y)) +yhat = LearnAPI.predictions(model) +test_indices = LearnAPI.out_of_sample_indices(model) +out_of_sample_loss = yhat[test_indices] .!= y[test_indices] |> mean +``` + +# New implementations + +Implement for algorithms that internally split training data into "train" and +"validate" subsets. Assumes +[`LearnAPI.data_interface(learner)`](@ref)`==LearnAPI.RandomAccess()`. + +$(DOC_IMPLEMENTED_METHODS(":(LearnAPI.out_of_sample_indices)")). +""" +function out_of_sample_indices end + +""" + LearnAPI.training_scores(model) + +Where supported, return the training scores obtained when running `model = fit(learner, +...)` for some `learner`. This will be a numerical vector whose length coincides with the +number of training observations, and whose interpretation depends on the learner. + +See also [`fit`](@ref). + +# New implementations + +Implement for learners, such as outlier detection algorithms, which associate a numerical +score with each observation during training, when these scores are of interest in +workflows (e.g, to normalize the scores for new observations). + +$(DOC_IMPLEMENTED_METHODS(":(LearnAPI.training_scores)")). """ function training_scores end """ - training_labels(algorithm, fitted_params, report) + LearnAPI.components(model) + +For a composite `model`, return the component models (`fit` outputs). These will be in the +form of a vector of named pairs, `sublearner::Symbol => component_model(s)`, one for each +`sublearner` in [`LearnAPI.learners(learner)`](@ref), where `learner = +LearnAPI.learner(model)`. Here `component_model(s)` will be the `fit` output (or vector of +`fit` outputs) generated internally for the the corresponding sublearner. + +The `model` is composite if [`LearnAPI.learners(learner)`](@ref) is non-empty. + +See also [`LearnAPI.learners`](@ref). + +# New implementations + +Implementent if and only if `model` is a composite model. + +$(DOC_IMPLEMENTED_METHODS(":(LearnAPI.components)")). + +""" +function components end + +# :extras intentionally excluded: +const ACCESSOR_FUNCTIONS_WITHOUT_EXTRAS = ( + :(LearnAPI.learner), + :(LearnAPI.coefficients), + :(LearnAPI.intercept), + :(LearnAPI.tree), + :(LearnAPI.trees), + :(LearnAPI.feature_names), + :(LearnAPI.feature_importances), + :(LearnAPI.training_losses), + :(LearnAPI.out_of_sample_losses), + :(LearnAPI.predictions), + :(LearnAPI.out_of_sample_indices), + :(LearnAPI.training_scores), + :(LearnAPI.components), +) -Return the training labels for `algorithm`, given `fitted_params` and -`report`, as returned by [`LearnAPI.fit`](@ref), [`LearnAPI.update!`](@ref) or -[`LearnAPI.ingest!`](@ref). +const ACCESSOR_FUNCTIONS_WITHOUT_EXTRAS_LIST = join( + map(ACCESSOR_FUNCTIONS_WITHOUT_EXTRAS) do f + "[`$f`](@ref)" + end, + ", ", + " and ", +) + + """ + LearnAPI.extras(model) + +Return miscellaneous byproducts of a learning algorithm's execution, from the +object `model` returned by a call of the form `fit(learner, data)`. + +$DOC_STATIC + +See also [`fit`](@ref). # New implementations -$(DOC_IMPLEMENTED_METHODS(:training_labels)). +Implementation is discouraged for byproducts already covered by other LearnAPI.jl accessor +functions: $ACCESSOR_FUNCTIONS_WITHOUT_EXTRAS_LIST. + +$(DOC_IMPLEMENTED_METHODS(":(LearnAPI.extras)")). """ -function training_labels end +function extras end + +const ACCESSOR_FUNCTIONS = + (:(LearnAPI.extras), ACCESSOR_FUNCTIONS_WITHOUT_EXTRAS...) + +const ACCESSOR_FUNCTIONS_LIST = join( + map(ACCESSOR_FUNCTIONS) do f + "[`$f`](@ref)" + end, + ", ", + " and ", +) diff --git a/src/algorithm_traits.jl b/src/algorithm_traits.jl deleted file mode 100644 index d9d5ba86..00000000 --- a/src/algorithm_traits.jl +++ /dev/null @@ -1,653 +0,0 @@ -# There are two types of traits - ordinary traits that an implementation overloads to make -# promises of algorithm behavior, and derived traits, which are never overloaded. - -const DOC_UNKNOWN = - "Returns `\"unknown\"` if the algorithm implementation has "* - "failed to overload the trait. " -const DOC_ON_TYPE = "The value of the trait must depend only on the type of `algorithm`. " - -const DOC_ONLY_ONE = - "No more than one of the following should be overloaded for an algorithm type: "* - "`LearnAPI.fit_scitype`, `LearnAPI.fit_type`, `LearnAPI.fit_observation_scitype`, "* - "`LearnAPI.fit_observation_type`." - - -const TRAITS = [ - :functions, - :preferred_kind_of_proxy, - :position_of_target, - :position_of_weights, - :descriptors, - :is_pure_julia, - :pkg_name, - :pkg_license, - :doc_url, - :load_path, - :is_wrapper, - :human_name, - :iteration_parameter, - :fit_keywords, - :fit_scitype, - :fit_observation_scitype, - :fit_type, - :fit_observation_type, - :predict_input_scitype, - :predict_output_scitype, - :predict_input_type, - :predict_output_type, - :transform_input_scitype, - :transform_output_scitype, - :transform_input_type, - :transform_output_type, - :name, - :is_algorithm, -] - -# # OVERLOADABLE TRAITS - -functions() = METHODS = (TRAINING_FUNCTIONS..., OPERATIONS..., ACCESSOR_FUNCTIONS...) -const FUNCTIONS = map(d -> "`:$d`", functions()) - -""" - LearnAPI.functions(algorithm) - -Return a tuple of symbols, such as `(:fit, :predict)`, corresponding to LearnAPI methods -specifically implemented for objects having the same type as `algorithm`. If non-empty, -this also guarantees `algorithm` is an algorithm, in the LearnAPI sense. See the Reference -section of the manual for details. - -# New implementations - -Every LearnAPI method that is not a trait and which is specifically implemented for -`typeof(algorithm)` must be included in the return value of this trait. Specifically, the -return value is a tuple of symbols from this list: $(join(FUNCTIONS, ", ")). To regenerate -this list, do `LearnAPI.functions()`. - -See also [`LearnAPI.Algorithm`](@ref). - -""" -functions(::Any) = () - - -""" - LearnAPI.preferred_kind_of_proxy(algorithm) - -Returns an instance of [`LearnAPI.KindOfProxy`](@ref), unless `LearnAPI.predict` is not -implemented for objects of type `typeof(algorithm)`, in which case it returns `nothing`. - -The returned target proxy is generally the one with the smallest computational cost, if -more than one type is supported. - -See also [`LearnAPI.predict`](@ref), [`LearnAPI.KindOfProxy`](@ref). - -# New implementations - -Any algorithm implementing `LearnAPI.predict` must overload this trait. - -The trait must return a lone instance `T()` for some concrete subtype `T <: -LearnAPI.KindOfProxy`. List these with `subtypes(LearnAPI.KindOfProxy)` and -`subtypes(LearnAPI.IID)`. - -Suppose, for example, we have the following implementation of a supervised learner -returning only probablistic predictions: - -```julia -LearnAPI.predict(algorithm::MyNewAlgorithmType, LearnAPI.Distribution(), Xnew) = ... -``` - -Then we can declare - -```julia -@trait MyNewAlgorithmType preferred_kind_of_proxy = LearnAPI.LiteralTarget() -``` - -which is shorthand for - -```julia -LearnAPI.preferred_kind_of_proxy(::MyNewAlgorithmType) = LearnAPI.Distribution() -``` - -For more on target variables and target proxies, refer to the LearnAPI documentation. - -""" -preferred_kind_of_proxy(::Any) = nothing - -""" - LearnAPI.position_of_target(algorithm) - -Return the expected position of the target variable within `data` in calls of the form -[`LearnAPI.fit`](@ref)`(algorithm, verbosity, data...)`. - -If this number is `0`, then no target is expected. If this number exceeds `length(data)`, -then `data` is understood to exclude the target variable. - -""" -position_of_target(::Any) = 0 - -""" - LearnAPI.position_of_weights(algorithm) - -Return the expected position of per-observation weights within `data` in -calls of the form [`LearnAPI.fit`](@ref)`(algorithm, verbosity, data...)`. - -If this number is `0`, then no weights are expected. If this number exceeds -`length(data)`, then `data` is understood to exclude weights, which are assumed to be -uniform. - -""" -position_of_weights(::Any) = 0 - -descriptors() = [ - :regression, - :classification, - :clustering, - :gradient_descent, - :iterative_algorithms, - :incremental_algorithms, - :dimension_reduction, - :encoders, - :static_algorithms, - :missing_value_imputation, - :ensemble_algorithms, - :wrappers, - :time_series_forecasting, - :time_series_classification, - :survival_analysis, - :distribution_fitters, - :Bayesian_algorithms, - :outlier_detection, - :collaborative_filtering, - :text_analysis, - :audio_analysis, - :natural_language_processing, - :image_processing, -] - -const DOC_DESCRIPTORS_LIST = join(map(d -> "`:$d`", descriptors()), ", ") - -""" - LearnAPI.descriptors(algorithm) - -Lists one or more suggestive algorithm descriptors from this list: $DOC_DESCRIPTORS_LIST (do -`LearnAPI.descriptors()` to reproduce). - -!!! warning - The value of this trait guarantees no particular behavior. The trait is - intended for informal classification purposes only. - -# New implementations - -This trait should return a tuple of symbols, as in `(:classifier, :probabilistic)`. - -""" -descriptors(::Any) = () - -""" - LearnAPI.is_pure_julia(algorithm) - -Returns `true` if training `algorithm` requires evaluation of pure Julia code only. - -# New implementations - -The fallback is `false`. - -""" -is_pure_julia(::Any) = false - -""" - LearnAPI.pkg_name(algorithm) - -Return the name of the package module which supplies the core training algorithm for -`algorithm`. This is not necessarily the package providing the LearnAPI -interface. - -$DOC_UNKNOWN - -# New implementations - -Must return a string, as in `"DecisionTree"`. - -""" -pkg_name(::Any) = "unknown" - -""" - LearnAPI.pkg_license(algorithm) - -Return the name of the software license, such as `"MIT"`, applying to the package where the -core algorithm for `algorithm` is implemented. - -""" -pkg_license(::Any) = "unknown" - -""" - LearnAPI.doc_url(algorithm) - -Return a url where the core algorithm for `algorithm` is documented. - -$DOC_UNKNOWN - -# New implementations - -Must return a string, such as `"https://en.wikipedia.org/wiki/Decision_tree_learning"`. - -""" -doc_url(::Any) = "unknown" - -""" - LearnAPI.load_path(algorithm) - -Return a string indicating where the `struct` for `typeof(algorithm)` can be found, beginning -with the name of the package module defining it. For example, a return value of -`"FastTrees.LearnAPI.DecisionTreeClassifier"` means the following julia code will return the -algorithm type: - -```julia -import FastTrees -FastTrees.LearnAPI.DecisionTreeClassifier -``` - -$DOC_UNKNOWN - - -""" -load_path(::Any) = "unknown" - - -""" - LearnAPI.is_wrapper(algorithm) - -Returns `true` if one or more properties (fields) of `algorithm` may themselves be -algorithms, and `false` otherwise. - -# New implementations - -This trait must be overloaded if one or more properties (fields) of `algorithm` may take -algorithm values. Fallback return value is `false`. - -$DOC_ON_TYPE - - -""" -is_wrapper(::Any) = false - -""" - LearnAPI.human_name(algorithm) - -A human-readable string representation of `typeof(algorithm)`. Primarily intended for -auto-generation of documentation. - -# New implementations - -Optional. A fallback takes the type name, inserts spaces and removes capitalization. For -example, `KNNRegressor` becomes `"knn regressor"`. Better would be to overload the trait -to return `"K-nearest neighbors regressor"`. Ideally, this is a "concrete" noun like -`"ridge regressor"` rather than an "abstract" noun like `"ridge regression"`. - -""" -human_name(M) = snakecase(name(M), delim=' ') # `name` defined below - -""" - LearnAPI.iteration_parameter(algorithm) - -The name of the iteration parameter of `algorithm`, or `nothing` if the algorithm is not -iterative. - -# New implementations - -Implement if algorithm is iterative. Returns a symbol or `nothing`. - -""" -iteration_parameter(::Any) = nothing - -""" - LearnAPI.fit_keywords(algorithm) - -Return a list of keywords that can be provided to `fit` that correspond to -metadata; $DOC_METADATA - -# New implementations - -If `LearnAPI.fit(algorithm, ...)` supports keyword arguments, then this trait must be -overloaded, and otherwise not. Fallback returns `()`. - -Here's a sample implementation for a classifier that implements a `LearnAPI.fit` method -with signature `fit(algorithm::MyClassifier, verbosity, X, y; class_weights=nothing)`: - -``` -LearnAPI.fit_keywords(::Any{<:MyClassifier}) = (:class_weights,) -``` - -or the shorthand - -``` -@trait MyClassifier fit_keywords=(:class_weights,) -``` - - -""" -fit_keywords(::Any) = () - -""" - LearnAPI.fit_scitype(algorithm) - -Return an upper bound on the scitype of data guaranteeing it to work when training -`algorithm`. - -Specifically, if the return value is `S` and `ScientificTypes.scitype(data) <: S`, then -the following low-level calls are allowed (assuming `metadata` is also valid and -`verbosity` is an integer): - -```julia -# apply data front-end: -data2, metadata2 = LearnAPI.reformat(algorithm, LearnAPI.fit, data...; metadata...) - -# train: -LearnAPI.fit(algorithm, verbosity, data2...; metadata2...) -``` - -See also [`LearnAPI.fit_type`](@ref), [`LearnAPI.fit_observation_scitype`](@ref), -[`LearnAPI.fit_observation_type`](@ref). - -# New implementations - -Optional. The fallback return value is `Union{}`. $DOC_ONLY_ONE - -""" -fit_scitype(::Any) = Union{} - -""" - LearnAPI.fit_observation_scitype(algorithm) - -Return an upper bound on the scitype of observations guaranteed to work when training -`algorithm` (independent of the type/scitype of the data container itself). - -Specifically, denoting the type returned above by `S`, suppose a user supplies training -data, `data` - typically a tuple, such as `(X, y)` - and valid metadata, `metadata`, and -one computes - - data2, metadata2 = LearnAPI.reformat(algorithm, LearnAPI.fit, data...; metadata...) - -Then, assuming - - ScientificTypes.scitype(LearnAPI.getobs(algorithm, LearnAPI.fit, data2, i)) <: S - -for any valid index `i`, the following is guaranteed to work: - - -```julia -LearnAPI.fit(algorithm, verbosity, data2...; metadata2...) -``` - -See also See also [`LearnAPI.fit_type`](@ref), [`LearnAPI.fit_scitype`](@ref), -[`LearnAPI.fit_observation_type`](@ref). - -# New implementations - -Optional. The fallback return value is `Union{}`. $DOC_ONLY_ONE - -""" -fit_observation_scitype(::Any) = Union{} - -""" - LearnAPI.fit_type(algorithm) - -Return an upper bound on the type of data guaranteeing it to work when training `algorithm`. - -Specifically, if the return value is `T` and `typeof(data) <: T`, then the following -low-level calls are allowed (assuming `metadata` is also valid and `verbosity` is an -integer): - -```julia -# apply data front-end: -data2, metadata2 = LearnAPI.reformat(algorithm, LearnAPI.fit, data...; metadata...) - -# train: -LearnAPI.fit(algorithm, verbosity, data2...; metadata2...) -``` - -See also [`LearnAPI.fit_scitype`](@ref), [`LearnAPI.fit_observation_type`](@ref). -[`LearnAPI.fit_observation_scitype`](@ref) - -# New implementations - -Optional. The fallback return value is `Union{}`. $DOC_ONLY_ONE - -""" -fit_type(::Any) = Union{} - -""" - LearnAPI.fit_observation_type(algorithm) - -Return an upper bound on the type of observations guaranteed to work when training -`algorithm` (independent of the type/scitype of the data container itself). - -Specifically, denoting the type returned above by `T`, suppose a user supplies training -data, `data` - typically a tuple, such as `(X, y)` - and valid metadata, `metadata`, and -one computes - - data2, metadata2 = LearnAPI.reformat(algorithm, LearnAPI.fit, data...; metadata...) - -Then, assuming - - typeof(LearnAPI.getobs(algorithm, LearnAPI.fit, data2, i)) <: T - -for any valid index `i`, the following is guaranteed to work: - - -```julia -LearnAPI.fit(algorithm, verbosity, data2...; metadata2...) -``` - -See also See also [`LearnAPI.fit_type`](@ref), [`LearnAPI.fit_scitype`](@ref), -[`LearnAPI.fit_observation_scitype`](@ref). - -# New implementations - -Optional. The fallback return value is `Union{}`. $DOC_ONLY_ONE - -""" -fit_observation_type(::Any) = Union{} - -DOC_INPUT_SCITYPE(op) = - """ - LearnAPI.$(op)_input_scitype(algorithm) - - Return an upper bound on the scitype of input data guaranteed to work with the `$op` - operation. - - Specifically, if `S` is the value returned and `ScientificTypes.scitype(data) <: S`, - then the following low-level calls are allowed - - data2 = LearnAPI.reformat(algorithm, LearnAPI.$op, data...) - LearnAPI.$op(algorithm, fitted_params, data2...) - - Here `fitted_params` are the learned parameters returned by an appropriate call to - `LearnAPI.fit`. - - See also [`LearnAPI.$(op)_input_type`](@ref). - - # New implementations - - Implementation is optional. The fallback return value is `Union{}`. Should not be - overloaded if `LearnAPI.$(op)_input_type` is overloaded. - - """ - -DOC_INPUT_TYPE(op) = - """ - LearnAPI.$(op)_input_type(algorithm) - - Return an upper bound on the type of input data guaranteed to work with the `$op` - operation. - - Specifically, if `T` is the value returned and `typeof(data) <: S`, then the following - low-level calls are allowed - - data2 = LearnAPI.reformat(algorithm, LearnAPI.$op, data...) - LearnAPI.$op(algorithm, fitted_params, data2...) - - Here `fitted_params` are the learned parameters returned by an appropriate call to - `LearnAPI.fit`. - - See also [`LearnAPI.$(op)_input_scitype`](@ref). - - # New implementations - - Implementation is optional. The fallback return value is `Union{}`. Should not be - overloaded if `LearnAPI.$(op)_input_scitype` is overloaded. - - """ - -DOC_OUTPUT_SCITYPE(op) = - """ - LearnAPI.$(op)_output_scitype(algorithm) - - Return an upper bound on the scitype of the output of the `$op` operation. - - Specifically, if `S` is the value returned, and if - - output, report = LearnAPI.$op(algorithm, fitted_params, data...) - - for suitable `fitted_params` and `data`, then - - ScientificTypes.scitype(output) <: S - - See also [`LearnAPI.$(op)_input_scitype`](@ref). - - # New implementations - - Implementation is optional. The fallback return value is `Any`. - - """ - -DOC_OUTPUT_TYPE(op) = - """ - LearnAPI.$(op)_output_type(algorithm) - - Return an upper bound on the type of the output of the `$op` operation. - - Specifically, if `T` is the value returned, and if - - output, report = LearnAPI.$op(algorithm, fitted_params, data...) - - for suitable `fitted_params` and `data`, then - - typeof(output) <: T - - See also [`LearnAPI.$(op)_input_type`](@ref). - - # New implementations - - Implementation is optional. The fallback return value is `Any`. - - """ - -"$(DOC_INPUT_SCITYPE(:predict))" -predict_input_scitype(::Any) = Union{} - -"$(DOC_INPUT_TYPE(:predict))" -predict_input_type(::Any) = Union{} - -"$(DOC_INPUT_SCITYPE(:transform))" -transform_input_scitype(::Any) = Union{} - -"$(DOC_OUTPUT_SCITYPE(:transform))" -transform_output_scitype(::Any) = Any - -"$(DOC_INPUT_TYPE(:transform))" -transform_input_type(::Any) = Union{} - -"$(DOC_OUTPUT_TYPE(:transform))" -transform_output_type(::Any) = Any - - -# # TWO-ARGUMENT TRAITS - -# Here `s` is `:type` or `:scitype`: -const DOC_PREDICT_OUTPUT(s) = - """ - LearnAPI.predict_output_$s(algorithm, kind_of_proxy::KindOfProxy) - - Return an upper bound for the $(s)s of predictions of the specified form where - supported, and otherwise return `Any`. For example, if - - ŷ, report = LearnAPI.predict(algorithm, LearnAPI.Distribution(), data...) - - successfully returns (i.e., `algorithm` supports predictions of target probability - distributions) then the following is guaranteed to hold: - - $(s)(ŷ) <: LearnAPI.predict_output_$(s)(algorithm, LearnAPI.Distribution()) - - **Note.** This trait has a single-argument "convenience" version - `LearnAPI.predict_output_$(s)(algorithm)` derived from this one, which returns a - dictionary keyed on target proxy types. - - See also [`LearnAPI.KindOfProxy`](@ref), [`LearnAPI.predict`](@ref), - [`LearnAPI.predict_input_$(s)`](@ref). - - # New implementations - - Overloading the trait is optional. Here's a sample implementation for a supervised - regressor type `MyRgs` that only predicts actual values of the target: - - LearnAPI.predict(alogrithm::MyRgs, ::LearnAPI.LiteralTarget, data...) = ... - LearnAPI.predict_output_$(s)(::MyRgs, ::LearnAPI.LiteralTarget) = - AbstractVector{ScientificTypesBase.Continuous} - - The fallback method returns `Any`. - - """ - -"$(DOC_PREDICT_OUTPUT(:scitype))" -predict_output_scitype(algorithm, kind_of_proxy) = Any - -"$(DOC_PREDICT_OUTPUT(:type))" -predict_output_type(algorithm, kind_of_proxy) = Any - - -# # DERIVED TRAITS - -name(A) = string(typename(A)) - -is_algorithm(A) = !isempty(functions(A)) - -const DOC_PREDICT_OUTPUT2(s) = - """ - LearnAPI.predict_output_$(s)(algorithm) - - Return a dictionary of upper bounds on the $(s) of predictions, keyed on concrete - subtypes of [`LearnAPI.KindOfProxy`](@ref). Each of these subtypes respresents a - different form of target prediction (`LiteralTarget`, `Distribution`, `SurvivalFunction`, - etc) possibly supported by `algorithm`, but the existence of a key does not guarantee - that form is supported. - - As an example, if - - ŷ, report = LearnAPI.predict(algorithm, LearnAPI.Distribution(), data...) - - successfully returns (i.e., `algorithm` supports predictions of target probability - distributions) then the following is guaranteed to hold: - - $(s)(ŷ) <: LearnAPI.predict_output_$(s)(algorithm)[LearnAPI.Distribution] - - See also [`LearnAPI.KindOfProxy`](@ref), [`LearnAPI.predict`](@ref), - [`LearnAPI.predict_input_$(s)`](@ref). - - # New implementations - - This single argument trait should not be overloaded. Instead, overload - [`LearnAPI.predict_output_$(s)`](@ref)(algorithm, kind_of_proxy). See above. - - """ - -"$(DOC_PREDICT_OUTPUT2(:scitype))" -predict_output_scitype(algorithm) = - Dict(T => predict_output_scitype(algorithm, T()) - for T in CONCRETE_TARGET_PROXY_TYPES) - -"$(DOC_PREDICT_OUTPUT2(:type))" -predict_output_type(algorithm) = - Dict(T => predict_output_type(algorithm, T()) - for T in CONCRETE_TARGET_PROXY_TYPES) - - diff --git a/src/algorithms.jl b/src/algorithms.jl deleted file mode 100644 index 02e9614c..00000000 --- a/src/algorithms.jl +++ /dev/null @@ -1,19 +0,0 @@ -abstract type LearnAPIType end - -""" - LearnAPI.Algorithm - -An optional abstract type for algorithms implementing LearnAPI.jl. - -If `typeof(alg) <: LearnAPI.Algorithm`, then `alg` is guaranteed to be an ML/statistical -algorithm in the strict LearnAPI sense. - -# New implementations - -While not a formal requirement, algorithm types implementing the LearnAPI.jl are -encouraged to subtype `LearnAPI.Algorithm`, unless it is disruptive to do so. - -See also [`LearnAPI.functions`](@ref). - -""" -abstract type Algorithm <: LearnAPIType end diff --git a/src/clone.jl b/src/clone.jl new file mode 100644 index 00000000..d3e6c872 --- /dev/null +++ b/src/clone.jl @@ -0,0 +1,27 @@ +""" + LearnAPI.clone(learner, replacements...) + LearnAPI.clone(learner; replacements...) + +Return a shallow copy of `learner` with the specified hyperparameter replacements. Two +syntaxes are supported, as shown in the following examples: + +```julia +clone(learner, :epochs => 100, :learner_rate => 0.01) +clone(learner; epochs=100, learning_rate=0.01) +``` + +A LearnAPI.jl contract ensures that `LearnAPI.clone(learner) == learner`. + +A new learner implementation does not overload `clone`. + +""" +function clone(learner, args...; kwargs...) + reps = merge(NamedTuple(args), NamedTuple(kwargs)) + names = propertynames(learner) + rep_names = keys(reps) + new_values = map(names) do name + name in rep_names && return getproperty(reps, name) + getproperty(learner, name) + end + return LearnAPI.constructor(learner)(NamedTuple{names}(new_values)...) +end diff --git a/src/data_interface.jl b/src/data_interface.jl deleted file mode 100644 index 8db5f486..00000000 --- a/src/data_interface.jl +++ /dev/null @@ -1,107 +0,0 @@ -""" - LearnAPI.getobs(algorithm, LearnAPI.fit, I, data...) - -Return a subsample of `data` consisting of all observations with indices in `I`. Here -`data` is data of the form expected in a call like `LearnAPI.fit(algorithm, verbosity, -data...; metadata...)`. - -Always returns a tuple of the same length as `data`. - - LearnAPI.getobs(algorithm, operation, I, data...) - -Return a subsample of `data` consisting of all observations with indices in `I`. Here -`data` is data of the form expected in a call of the specified `operation`, e.g., in a -call like `LearnAPI.predict(algorithm, data...)`, if `operation = LearnAPI.predict`. Possible -values for `operation` are: $DOC_OPERATIONS_LIST_FUNCTION. - -Always returns a tuple of the same length as `data`. - -# New implementations - -Implementation is optional. If implemented, then ordinarily implemented for each signature -of `fit` and operation implemented for `algorithm`. - -$(DOC_IMPLEMENTED_METHODS(:reformat)) - -The subsample returned must be acceptable in place of `data` in calls of the function -named in the second argument. - -## Sample implementation - -Suppose that `MyClassifier` is an algorithm type for simple supervised classification, with -`LearnAPI.fit(algorithm::MyClassifier, verbosity, A, y)` and `predict(algorithm::MyClassifier, -fitted_params, A)` implemented assuming the target `y` is an ordinary abstract vector and -the features `A` is an abstract matrix with columns as observations. Then the following is -a valid implementation of `getobs`: - -```julia -LearnAPI.getobs(::MyClassifier, ::typeof(LearnAPI.fit), I, A, y) = - (view(A, :, I), view(y, I)) -LearnAPI.getobs(::MyClassifier, ::typeof(LearnAPI.predict), I, A) = (view(A, :, I),) -``` - -""" -function getobs end - -""" - LearnAPI.reformat(algorithm, LearnAPI.fit, user_data...; metadata...) - -Return the algorithm-specific representations `(data, metadata)` of user-supplied `(user_data, -user_metadata)`, for consumption, after splatting, by `LearnAPI.fit`, `LearnAPI.update!` -or `LearnAPI.ingest!`. - - LearnAPI.reformat(algorithm, operation, user_data...) - -Return the algorithm-specific representation `data` of user-supplied `user_data`, for -consumption, after splatting, by the specified `operation`, dispatched on `algorithm`. Here -`operation` is one of: $DOC_OPERATIONS_LIST_FUNCTION. - -The following sample workflow illustrates the use of both versions of `reformat`above. The -data objects `X`, `y`, and `Xtest` are the user-supplied versions of data. - -```julia -data, metadata = LearnAPI.reformat(algorithm, LearnAPI.fit, X, y; class_weights=some_dictionary) -fitted_params, state, fit_report = LearnAPI.fit(algorithm, 0, data...; metadata...) - -test_data = LearnAPI.reformat(algorithm, LearnAPI.predict, Xtest) -ŷ, predict_report = LearnAPI.predict(algorithm, fitted_params, test_data...) -``` - -# New implementations - -Implementation of `reformat` is optional. The fallback simply slurps the supplied -data/metadata. You will want to implement for each `fit` or operation signature -implemented for `algorithm`. - -$(DOC_IMPLEMENTED_METHODS(:reformat, overloaded=true)) - -Ideally, any potentially expensive transformation of user-supplied data that is carried -out during training only once, at the beginning, should occur in `reformat` instead of -`fit`/`update!`/`ingest!`. - -Note that the first form of `reformat`, for operations, should always return a tuple, -because the output is splat in calls to the operation (see the sample workflow -above). Similarly, in the return value `(data, metadata)` for the `fit` variant, `data` is -always a tuple and `metadata` always a named tuple (or `Base.Pairs` object). If there is -no metadata, a `NamedTuple()` can be returned in its place. - -## Example implementation - -Suppose that `MyClassifier` is an algorithm type for simple supervised classification, with -`LearnAPI.fit(algorithm::MyClassifier, verbosity, A, y; names=...)` and -`predict(algorithm::MyClassifier, fitted_params, A)` implemented assuming that the target `y` -is an ordinary vector, the features `A` is a matrix with columns as observations, and -`names` are the names of the features. Then, supposing users supply features in tabular -form, but target as expected, then we can provide the following implementation of -`reformat`: - -```julia -using Tables -function LearnAPI.reformat(::MyClassifier, ::typeof(LearnAPI.fit), X, y) - names = Tables.schema(Tables.rows(X)).names - return ((Tables.matrix(X)', y), (; names)) -end -LearnAPI.reformat(::MyClassifier, ::typeof(LearnAPI.predict), X) = (Tables.matrix(X)',) -``` -""" -reformat(::Any, ::Any, data...; algorithm_data...) = (data, algorithm_data) diff --git a/src/fit_update.jl b/src/fit_update.jl new file mode 100644 index 00000000..015669e7 --- /dev/null +++ b/src/fit_update.jl @@ -0,0 +1,173 @@ +# # FIT + +""" + fit(learner, data; verbosity=LearnAPI.default_verbosity()) + fit(learner; verbosity=LearnAPI.default_verbosity()) + +Execute the machine learning or statistical algorithm with configuration `learner` using +the provided training `data`, returning an object, `model`, on which other methods, such +as [`predict`](@ref) or [`transform`](@ref), can be dispatched. +[`LearnAPI.functions(learner)`](@ref) returns a list of methods that can be applied to +either `learner` or `model`. + +For example, a supervised classifier might have a workflow like this: + +```julia +model = fit(learner, (X, y)) +ŷ = predict(model, Xnew) +``` + +The signature `fit(learner; verbosity=...)` (no `data`) is provided by learners that do +not generalize to new observations (called *static algorithms*). In that case, +`transform(model, data)` or `predict(model, ..., data)` carries out the actual algorithm +execution, writing any byproducts of that operation to the mutable object `model` returned +by `fit`. Inspect the value of [`LearnAPI.is_static(learner)`](@ref) to determine whether +`fit` consumes `data` or not. + +Use `verbosity=0` for warnings only, and `-1` for silent training. + +See also [`LearnAPI.default_verbosity`](@ref), [`predict`](@ref), [`transform`](@ref), +[`inverse_transform`](@ref), [`LearnAPI.functions`](@ref), [`obs`](@ref). + +# Extended help + +# New implementations + +Implementation of exactly one of the signatures is compulsory. If `fit(learner; +verbosity=...)` is implemented, then the trait [`LearnAPI.is_static`](@ref) must be +overloaded to return `true`. + +The signature must include `verbosity` with [`LearnAPI.default_verbosity()`](@ref) as +default. + +If `data` encapsulates a *target* variable, as defined in LearnAPI.jl documentation, then +[`LearnAPI.target(data)`](@ref) must be overloaded to return it. If [`predict`](@ref) or +[`transform`](@ref) are implemented and consume data, then +[`LearnAPI.features(data)`](@ref) must return something that can be passed as data to +these methods. A fallback returns `first(data)` if `data` is a tuple, and `data` +otherwise. + +The LearnAPI.jl specification has nothing to say regarding `fit` signatures with more than +two arguments. For convenience, for example, an implementation is free to implement a +slurping signature, such as `fit(learner, X, y, extras...) = fit(learner, (X, y, +extras...))` but LearnAPI.jl does not guarantee such signatures are actually implemented. + +$(DOC_DATA_INTERFACE(:fit)) + +""" +function fit end + + +# # UPDATE AND COUSINS + +""" + update(model, data, param_replacements...; verbosity=1) + +Return an updated version of the `model` object returned by a previous [`fit`](@ref) or +`update` call, but with the specified hyperparameter replacements, in the form `p1 => +value1, p2 => value2, ...`. + +```julia +learner = MyForest(ntrees=100) + +# train with 100 trees: +model = fit(learner, data) + +# add 50 more trees: +model = update(model, data, :ntrees => 150) +``` + +Provided that `data` is identical with the data presented in a preceding `fit` call *and* +there is at most one hyperparameter replacement, as in the above example, execution is +semantically equivalent to the call `fit(learner, data)`, where `learner` is +`LearnAPI.learner(model)` with the specified replacements. In some cases (typically, +when changing an iteration parameter) there may be a performance benefit to using `update` +instead of retraining ab initio. + +If `data` differs from that in the preceding `fit` or `update` call, or there is more than +one hyperparameter replacement, then behaviour is learner-specific. + +See also [`fit`](@ref), [`update_observations`](@ref), [`update_features`](@ref). + +# New implementations + +Implementation is optional. The signature must include `verbosity`. It should be true that +`LearnAPI.learner(newmodel) == newlearner`, where `newmodel` is the return value and +`newlearner = LearnAPI.clone(learner, replacements...)`. + + +$(DOC_IMPLEMENTED_METHODS(":(LearnAPI.update)")) + +See also [`LearnAPI.clone`](@ref) + +""" +function update end + +""" + update_observations(model, new_data, param_replacements...; verbosity=1) + +Return an updated version of the `model` object returned by a previous [`fit`](@ref) or +`update` call given the new observations present in `new_data`. One may additionally +specify hyperparameter replacements in the form `p1 => value1, p2 => value2, ...`. + +```julia-repl +learner = MyNeuralNetwork(epochs=10, learning_rate => 0.01) + +# train for ten epochs: +model = fit(learner, data) + +# train for two more epochs using new data and new learning rate: +model = update_observations(model, new_data, epochs => 12, learning_rate => 0.1) +``` + +When following the call `fit(learner, data)`, the `update` call is semantically +equivalent to retraining ab initio using a concatenation of `data` and `new_data`, +*provided there are no hyperparameter replacements* (which rules out the example +above). Behaviour is otherwise learner-specific. + +See also [`fit`](@ref), [`update`](@ref), [`update_features`](@ref). + +# Extended help + +# New implementations + +Implementation is optional. The signature must include `verbosity`. It should be true that +`LearnAPI.learner(newmodel) == newlearner`, where `newmodel` is the return value and +`newlearner = LearnAPI.clone(learner, replacements...)`. + +$(DOC_IMPLEMENTED_METHODS(":(LearnAPI.update_observations)")) + +See also [`LearnAPI.clone`](@ref). + +""" +function update_observations end + +""" + update_features(model, new_data, param_replacements...; verbosity=1) + ) + +Return an updated version of the `model` object returned by a previous [`fit`](@ref) or +`update` call given the new features encapsulated in `new_data`. One may additionally +specify hyperparameter replacements in the form `p1 => value1, p2 => value2, ...`. + +When following the call `fit(learner, data)`, the `update` call is semantically +equivalent to retraining ab initio using a concatenation of `data` and `new_data`, +*provided there are no hyperparameter replacements.* Behaviour is otherwise +learner-specific. + +See also [`fit`](@ref), [`update`](@ref), [`update_features`](@ref). + +# Extended help + +# New implementations + +Implementation is optional. The signature must include `verbosity`. It should be true that +`LearnAPI.learner(newmodel) == newlearner`, where `newmodel` is the return value and +`newlearner = LearnAPI.clone(learner, replacements...)`. + +$(DOC_IMPLEMENTED_METHODS(":(LearnAPI.update_features)")) + +See also [`LearnAPI.clone`](@ref). + +""" +function update_features end diff --git a/src/fit_update_ingest.jl b/src/fit_update_ingest.jl deleted file mode 100644 index e38ccc40..00000000 --- a/src/fit_update_ingest.jl +++ /dev/null @@ -1,177 +0,0 @@ -# # DOC STRING HELPERS - -const TRAINING_FUNCTIONS = (:fit, :update!, :ingest!) - -const DOC_OPERATIONS = "An *operation* is a method dispatched on an algorithm, "* - "associated learned parameters, and data. "* - "The LearnAPI operations are: $DOC_OPERATIONS_LIST_FUNCTION. " - -const DOC_METADATA = - "`metadata` is for extra information pertaining to the data that is never "* - "iterated or subsampled. Examples, include target *class* weights and group "* - "lasso feature groupings. Further examples include feature names, and the "* - "pool of target classes, when these are not embedded in the data representation. " - -const DOC_WHAT_IS_DATA = - """ - In LearnAPI, *data* is any tuple of objects sharing a - common number of "observations. " - """ - -const DOC_MUTATING_MODELS = - """ - !!! note - - The method is not permitted to mutate `algorithm`. In particular, if `algorithm` - has a random number generator as a hyperparameter (property) then it must be - copied before use. - """ - -# # FIT - -""" - LearnAPI.fit(algorithm, verbosity, data...; metadata...) - -Perform training associated with `algorithm` using the provided `data` and `metadata`. With -the exception of warnings, training will be silent if `verbosity == 0`. Lower values -should suppress warnings; any integer ought to be admissible. Here: - -- `algorithm` is a property-accessible object whose properties are the hyperparameters of - some ML/statistical algorithm. - -- `data` is a tuple of data objects with a common number of observations, for example, - `data = (X, y, w)` where `X` is a table of features, `y` is a target vector with the - same number of rows, and `w` a vector of per-observation weights. - -- $DOC_METADATA To see the keyword names for metadata supported by `algorithm`, do - `LearnAPI.fit_keywords(algorithm)`. " - - -# Return value - -Returns a tuple (`fitted_params`, `state`, `report`) where: - -- The `fitted_params` is the algorithm's learned parameters (eg, the coefficients in a linear - algorithm) in a form understood by operations. $DOC_OPERATIONS If some training - outcome of user-interest is not needed for operations, it should be part of `report` - instead (see below). - -- The `state` is for passing to [`LearnAPI.update!`](@ref) or - [`LearnAPI.ingest!`](@ref). For algorithms that implement neither, `state` should be - `nothing`. - -- The `report` records byproducts of training not in the `fitted_params`, such as feature - rankings, or out-of-sample estimates of performance. - - -# New implementations - -Overloading this method for new algorithms is optional. A fallback performs no -computation, returning `(nothing, nothing, nothing)`. - -See the LearnAPI.jl documentation for the detailed requirements of LearnAPI.jl algorithm -objects. - -$DOC_WHAT_IS_DATA - -$DOC_MUTATING_MODELS - -$(DOC_IMPLEMENTED_METHODS(:fit)) - -If supporting metadata, you must also implement [`LearnAPI.fit_keywords`](@ref) to list -the supported keyword argument names (e.g., `class_weights`). - -See also [`LearnAPI.update!`](@ref), [`LearnAPI.ingest!`](@ref). - -""" -fit(::Any, ::Any, ::Integer, data...; metadata...) = nothing, nothing, nothing - - -# # UPDATE - -""" - LearnAPI.update!(algorithm, verbosity, fitted_params, state, data...; metadata...) - -Based on the values of `state`, and `fitted_params` returned by a preceding call to -[`LearnAPI.fit`](@ref), [`LearnAPI.ingest!`](@ref), or [`LearnAPI.update!`](@ref), update a -algorithm's fitted parameters, returning new (or mutated) `state` and `fitted_params`. - -Intended for retraining when the training data has not changed, but `algorithm` -properties (hyperparameters) may have changed, e.g., when increasing an iteration -parameter. Specifically, the assumption is that `data` and `metadata` have the same values -seen in the most recent call to `fit/update!/ingest!`. - -For incremental training (same algorithm, new data) see instead [`LearnAPI.ingest!`](@ref). - -# Return value - -Same as [`LearnAPI.fit`](@ref), namely a tuple (`fitted_params`, `state`, `report`). See -[`LearnAPI.fit`](@ref) for details. - - -# New implementations - -Overloading this method is optional. A fallback calls `LearnAPI.fit`: - -```julia -LearnAPI.update!(algorithm, verbosity, fitted_params, state, data...; metadata...) = - fit(algorithm, verbosity, data; metadata...) -``` -$(DOC_IMPLEMENTED_METHODS(:fit)) - -$DOC_WHAT_IS_DATA - -The most common use case is continuing training of an iterative algorithm: `state` is -simply a copy of the algorithm used in the last training call (`fit`, `update!` or `ingest!`) -and this will include the current number of iterations as a property. If `algorithm` and -`state` differ only in the number of iterations (e.g., epochs in a neural network), which -has increased, then the fitted parameters (weights) are updated, rather than computed from -scratch. Otherwise, `update!` simply calls `fit`, to force retraining from scratch. - -It is permitted to return mutated versions of `state` and `fitted_params`. - -$DOC_MUTATING_MODELS - -See also [`LearnAPI.fit`](@ref), [`LearnAPI.ingest!`](@ref). - -""" -update!(algorithm, verbosity, fitted_params, state, data...; metadata...) = - fit(algorithm, verbosity, data...; metadata...) - - -# # INGEST - -""" - LernAPI.ingest!(algorithm, verbosity, fitted_params, state, data...; metadata...) - -For an algorithm that supports incremental learning, update the fitted parameters using -`data`, which has typically not been seen before. The arguments `state` and -`fitted_params` are the output of a preceding call to [`LearnAPI.fit`](@ref), -[`LearnAPI.ingest!`](@ref), or [`LearnAPI.update!`](@ref), of which mutated or new -versions are returned. - -For updating fitted parameters using the *same* data but new hyperparameters, see instead -[`LearnAPI.update!`](@ref). - -For training an algorithm with new hyperparameters but *unchanged* data, see instead -[`LearnAPI.update!`](@ref). - - -# Return value - -Same as [`LearnAPI.fit`](@ref), namely a tuple (`fitted_params`, `state`, `report`). See -[`LearnAPI.fit`](@ref) for details. - - -# New implementations - -Implementing this method is optional. It has no fallback. - -$(DOC_IMPLEMENTED_METHODS(:fit)) - -$DOC_MUTATING_MODELS - -See also [`LearnAPI.fit`](@ref), [`LearnAPI.update!`](@ref). - -""" -function ingest!(algorithm, verbosity, fitted_params, state, data...; metadata...) end diff --git a/src/obs.jl b/src/obs.jl new file mode 100644 index 00000000..2e631d30 --- /dev/null +++ b/src/obs.jl @@ -0,0 +1,89 @@ +""" + obs(learner, data) + obs(model, data) + +Return learner-specific representation of `data`, suitable for passing to `fit`, `update`, + `update_observations`, or `update_features` (first signature) or to `predict` and + `transform` (second signature), in place of `data`. Here `model` is the return value of + `fit(learner, ...)` for some LearnAPI.jl learner, `learner`. + +The returned object is guaranteed to implement observation access as indicated by +[`LearnAPI.data_interface(learner)`](@ref), typically +[`LearnAPI.RandomAccess()`](@ref). + +Calling `fit`/`predict`/`transform` on the returned objects may have performance +advantages over calling directly on `data` in some contexts. + +# Example + +Usual workflow, using data-specific resampling methods: + +```julia +data = (X, y) # a DataFrame and a vector +data_train = (Tables.select(X, 1:100), y[1:100]) +model = fit(learner, data_train) +ŷ = predict(model, Point(), X[101:150]) +``` + +Alternative workflow using `obs` and the MLUtils.jl method `getobs` to carry out +subsampling (assumes `LearnAPI.data_interface(learner) == RandomAccess()`): + +```julia +import MLUtils +fit_observations = obs(learner, data) +model = fit(learner, MLUtils.getobs(fit_observations, 1:100)) +predict_observations = obs(model, X) +ẑ = predict(model, Point(), MLUtils.getobs(predict_observations, 101:150)) +@assert ẑ == ŷ +``` + +See also [`LearnAPI.data_interface`](@ref). + +# Extended help + +# New implementations + +Implementation is typically optional. + +For each supported form of `data` in `fit(learner, data)`, it must be true that `model = +fit(learner, observations)` is equivalent to `model = fit(learner, data)`, whenever +`observations = obs(learner, data)`. For each supported form of `data` in calls +`predict(model, ..., data)` and `transform(model, data)`, where implemented, the calls +`predict(model, ..., observations)` and `transform(model, observations)` must be supported +alternatives with the same output, whenever `observations = obs(model, data)`. + +If `LearnAPI.data_interface(learner) == RandomAccess()` (the default), then `fit`, +`predict` and `transform` must additionally accept `obs` output that has been *subsampled* +using `MLUtils.getobs`, with the obvious interpretation applying to the outcomes of such +calls (e.g., if *all* observations are subsampled, then outcomes should be the same as if +using the original data). + +It is required that `obs(learner, _)` and `obs(model, _)` are involutive, meaning both the +following hold: + +```julia +obs(learner, obs(learner, data)) == obs(learner, data) +obs(model, obs(model, data) == obs(model, obs(model, data) +``` + +If one overloads `obs`, one typically needs additionally overloadings to guarantee +involutivity. + +The fallback for `obs` is `obs(model_or_learner, data) = data`, and the fallback for +`LearnAPI.data_interface(learner)` is `LearnAPI.RandomAccess()`. For details refer to +the [`LearnAPI.data_interface`](@ref) document string. + +In particular, if the `data` to be consumed by `fit`, `predict` or `transform` consists +only of suitable tables and arrays, then `obs` and `LearnAPI.data_interface` do not need +to be overloaded. However, the user will get no performance benefits by using `obs` in +that case. + +## Sample implementation + +Refer to the ["Anatomy of an +Implementation"](https://juliaai.github.io/LearnAPI.jl/dev/anatomy_of_an_implementation/#Providing-an-advanced-data-interface) +section of the LearnAPI.jl manual. + + +""" +obs(learner_or_model, data) = data diff --git a/src/operations.jl b/src/operations.jl deleted file mode 100644 index 2782df3d..00000000 --- a/src/operations.jl +++ /dev/null @@ -1,184 +0,0 @@ -function DOC_IMPLEMENTED_METHODS(name; overloaded=false) - word = overloaded ? "overloaded" : "implemented" - "If $word, you must include `:$name` in the tuple returned by the "* - "[`LearnAPI.functions`](@ref) trait. " -end - -const OPERATIONS = (:predict, :transform, :inverse_transform) -const DOC_OPERATIONS_LIST_SYMBOL = join(map(op -> "`:$op`", OPERATIONS), ", ") -const DOC_OPERATIONS_LIST_FUNCTION = join(map(op -> "`LearnAPI.$op`", OPERATIONS), ", ") - -const DOC_NEW_DATA = - "The `report` contains ancilliary byproducts of the computation, or "* - "is `nothing`; `data` is a tuple of data objects, "* - "generally a single object representing new observations "* - "not seen in training. " - - -# # METHOD STUBS/FALLBACKS - -""" - LearnAPI.predict(algorithm, kind_of_proxy::LearnAPI.KindOfProxy, fitted_params, data...) - -Return `(ŷ, report)` where `ŷ` is the predictions (a data object with target predictions -as observations) or a proxy for these, for the specified `algorithm` having learned -parameters `fitted_params` (first object returned by [`LearnAPI.fit`](@ref)`(algorithm, -...)`). $DOC_NEW_DATA - -Where available, use `kind_of_proxy=LiteralTarget()` for ordinary target predictions, and -`kind_of_proxy=Distribution()` for PDF/PMF predictions. Always available is -`kind_of_proxy=`LearnAPI.preferred_kind_of_proxy(algorithm)`. - -For a full list of target proxy types, run `subtypes(LearnAPI.KindOfProxy)` and -`subtypes(LearnAPI.IID)`. - -# New implementations - -$(DOC_IMPLEMENTED_METHODS(:predict)) - -If implementing `LearnAPI.predict`, then a -[`LearnAPI.preferred_kind_of_proxy`](@ref) declaration is required, as in - -```julia -LearnAPI.preferred_kind_of_proxy(::Type{<:SomeAlgorithm}) = LearnAPI.Distribution() -``` - -which has the shorthand - -```julia -@trait SomeAlgorithm preferred_kind_of_proxy=LearnAPI.Distribution() -``` - -The value of this trait must be an instance `T()`, where `T <: LearnAPI.KindOfProxy`. - -See also [`LearnAPI.fit`](@ref). - -""" -function predict end - -""" - LearnAPI.transform(algorithm, fitted_params, data...) - -Return `(output, report)`, where `output` is some kind of transformation of `data`, -provided by `algorithm`, based on the learned parameters `fitted_params` (the first object -returned by [`LearnAPI.fit`](@ref)`(algorithm, ...)`). The `fitted_params` could be -`nothing`, in the case of algorithms that do not generalize to new data. $DOC_NEW_DATA - - -# New implementations - -$(DOC_IMPLEMENTED_METHODS(:transform)) - -See also [`LearnAPI.inverse_transform`](@ref), [`LearnAPI.fit`](@ref), -[`LearnAPI.predict`](@ref), - -""" -function transform end - -""" - LearnAPI.inverse_transform(algorithm, fitted_params, data) - -Return `(data_inverted, report)`, where `data_inverted` is valid input to the call - -```julia -LearnAPI.transform(algorithm, fitted_params, data_inverted) -``` -$DOC_NEW_DATA - -Typically, the map - -```julia -data -> first(inverse_transform(algorithm, fitted_params, data)) -``` - -will be an inverse, approximate inverse, right inverse, or approximate right inverse, for -the map - -```julia -data -> first(transform(algorithm, fitted_params, data)) -``` - -For example, if `transform` corresponds to a projection, `inverse_transform` might be the -corresponding embedding. - - -# New implementations - -$(DOC_IMPLEMENTED_METHODS(:transform)) - -See also [`LearnAPI.fit`](@ref), [`LearnAPI.predict`](@ref), - -""" -function inverse_transform end - -function save end -function restore end - - -# # TARGET PROXIES - -""" - - LearnAPI.KindOfProxy - -Abstract type whose concrete subtypes `T` each represent a different kind of proxy for the -target variable, associated with some algorithm. Instances `T()` are used to request the -form of target predictions in [`LearnAPI.predict`](@ref) calls. - -For example, `LearnAPI.Distribution` is a concrete subtype of `LearnAPI.KindOfProxy` and -the call `LearnAPI.predict(algorithm , LearnAPI.Distribution(), data...)` returns a data -object whose observations are probability density/mass functions, assuming `algorithm` -supports predictions of that form. - -Run `subtypes(LearnAPI.KindOfProxy)` and `subtypes(LearnAPI.IID)` to list all concrete -subtypes of `KindOfProxy`. - -""" -abstract type KindOfProxy end - -""" - LearnAPI.IID <: LearnAPI.KindOfProxy - -Abstract subtype of [`LearnAPI.KindOfProxy`](@ref). If `kind_of_proxy` is an instance of -`LearnAPI.IID` then, given `data` constisting of ``n`` observations, the following must -hold: - -- `LearnAPI.predict(algorithm, kind_of_proxy, data...) == (ŷ, report)` where `ŷ` is data - also consisting of ``n`` observations; and - -- The ``j``th observation of `ŷ`, for any ``j``, depends only on the ``j``th - observation of the provided `data` (no correlation between observations). - -See also [`LearnAPI.KindOfProxy`](@ref). - -""" -abstract type IID <: KindOfProxy end - -struct LiteralTarget <: IID end -struct Sampleable <: IID end -struct Distribution <: IID end -struct LogDistribution <: IID end -struct Probability <: IID end -struct LogProbability <: IID end -struct Parametric <: IID end -struct LabelAmbiguous <: IID end -struct LabelAmbiguousSampleable <: IID end -struct LabelAmbiguousDistribution <: IID end -struct ConfidenceInterval <: IID end -struct Set <: IID end -struct ProbabilisticSet <: IID end -struct SurvivalFunction <: IID end -struct SurvivalDistribution <: IID end -struct OutlierScore <: IID end -struct Continuous <: IID end - -struct JointSampleable <: KindOfProxy end -struct JointDistribution <: KindOfProxy end -struct JointLogDistribution <: KindOfProxy end - -const CONCRETE_TARGET_PROXY_TYPES = [ - subtypes(IID)..., - JointSampleable, - JointDistribution, - JointLogDistribution, -] diff --git a/src/predict_transform.jl b/src/predict_transform.jl new file mode 100644 index 00000000..d4bfe0c8 --- /dev/null +++ b/src/predict_transform.jl @@ -0,0 +1,207 @@ +function DOC_IMPLEMENTED_METHODS(name; overloaded=false) + word = overloaded ? "overloaded" : "implemented" + "If $word, you must include `$name` in the tuple returned by the "* + "[`LearnAPI.functions`](@ref) trait. " +end + +DOC_MUTATION(op) = + """ + + If [`LearnAPI.is_static(learner)`](@ref) is `true`, then `$op` may mutate it's first + argument (to record byproducts of the computation not naturally part of the return + value) but not in a way that alters the result of a subsequent call to `predict`, + `transform` or `inverse_transform`. See more at [`fit`](@ref). + + """ + +DOC_SLURPING(op) = + """ + + An implementation is free to implement `$op` signatures with additional positional + arguments (eg., data-slurping signatures) but LearnAPI.jl is silent about their + interpretation or existence. + + """ + +DOC_MINIMIZE(func) = + """ + + If, additionally, [`LearnAPI.strip(model)`](@ref) is overloaded, then the following + identity must hold: + + ```julia + $func(LearnAPI.strip(model), args...) == $func(model, args...) + ``` + + """ + +DOC_DATA_INTERFACE(method) = + """ + + ## Assumptions about data + + By default, it is assumed that `data` supports the [`LearnAPI.RandomAccess`](@ref) + interface; this includes all matrices, with observations-as-columns, most tables, and + tuples thereof. See [`LearnAPI.RandomAccess`](@ref) for details. If this is not the + case then an implementation must either: (i) overload [`obs`](@ref) to articulate how + provided data can be transformed into a form that does support + [`LearnAPI.RandomAccess`](@ref); or (ii) overload the trait + [`LearnAPI.data_interface`](@ref) to specify a more relaxed data API. Refer tbo + document strings for details. + + """ + + +# # METHOD STUBS/FALLBACKS + +""" + predict(model, kind_of_proxy::LearnAPI.KindOfProxy, data) + predict(model, data) + +The first signature returns target predictions, or proxies for target predictions, for +input features `data`, according to some `model` returned by [`fit`](@ref). Where +supported, these are literally target predictions if `kind_of_proxy = Point()`, +and probability density/mass functions if `kind_of_proxy = Distribution()`. List all +options with [`LearnAPI.kinds_of_proxy(learner)`](@ref), where `learner = +LearnAPI.learner(model)`. + +```julia +model = fit(learner, (X, y)) +predict(model, Point(), Xnew) +``` + +The shortcut `predict(model, data)` calls the first method with learner-specific +`kind_of_proxy`, namely the first element of [`LearnAPI.kinds_of_proxy(learner)`](@ref), +which lists all supported target proxies. + +The argument `model` is anything returned by a call of the form `fit(learner, ...)`. + +If `LearnAPI.features(LearnAPI.learner(model)) == nothing`, then the argument `data` is +omitted in both signatures. An example is density estimators. + +See also [`fit`](@ref), [`transform`](@ref), [`inverse_transform`](@ref). + +# Extended help + +In the special case `LearnAPI.is_static(learner) == true`, it is possible that +`predict(model, ...)` will mutate `model`, but not in a way that affects subsequent +`predict` calls. + +# New implementations + +If there is no notion of a "target" variable in the LearnAPI.jl sense, or you need an +operation with an inverse, implement [`transform`](@ref) instead. + +Implementation is optional. Only the first signature (with or without the `data` argument) +is implemented, but each `kind_of_proxy::`[`KindOfProxy`](@ref) that gets an +implementation must be added to the list returned by +[`LearnAPI.kinds_of_proxy(learner)`](@ref). List all available kinds of proxy by doing +`LearnAPI.kinds_of_proxy()`. + +If `data` is not present in the implemented signature (eg., for density estimators) then +[`LearnAPI.features(learner, data)`](@ref) must return `nothing`. + +$(DOC_IMPLEMENTED_METHODS(":(LearnAPI.predict)")) + +$(DOC_MINIMIZE(:predict)) + +$(DOC_MUTATION(:predict)) + +$(DOC_DATA_INTERFACE(:predict)) + +""" +predict(model, data) = predict(model, kinds_of_proxy(learner(model)) |> first, data) +predict(model) = predict(model, kinds_of_proxy(learner(model)) |> first) + +""" + transform(model, data) + +Return a transformation of some `data`, using some `model`, as returned by +[`fit`](@ref). + +# Example + +Below, `X` and `Xnew` are data of the same form. + +For a `learner` that generalizes to new data ("learns"): + +```julia +model = fit(learner, X; verbosity=0) +transform(model, Xnew) +``` + +or, in one step (where supported): + +```julia +W = transform(learner, X) # `fit` implied +``` + +For a static (non-generalizing) transformer: + +```julia +model = fit(learner) +W = transform(model, X) +``` + +or, in one step (where supported): + +```julia +W = transform(learner, X) # `fit` implied +``` + +In the special case `LearnAPI.is_static(learner) == true`, it is possible that +`transform(model, ...)` will mutate `model`, but not in a way that affects subsequent +`transform` calls. + +See also [`fit`](@ref), [`predict`](@ref), +[`inverse_transform`](@ref). + +# Extended help + +# New implementations + +Implementation for new LearnAPI.jl learners is +optional. $(DOC_IMPLEMENTED_METHODS(":(LearnAPI.transform)")) + +$(DOC_SLURPING(:transform)) + +$(DOC_MINIMIZE(:transform)) + +$(DOC_MUTATION(:transform)) + +$(DOC_DATA_INTERFACE(:transform)) + +""" +function transform end + + +""" + inverse_transform(model, data) + +Inverse transform `data` according to some `model` returned by [`fit`](@ref). Here +"inverse" is to be understood broadly, e.g, an approximate +right or left inverse for [`transform`](@ref). + +# Example + +In the following, `learner` is some dimension-reducing algorithm that generalizes to new +data (such as PCA); `Xtrain` is the training input and `Xnew` the input to be reduced: + +```julia +model = fit(learner, Xtrain) +W = transform(model, Xnew) # reduced version of `Xnew` +Ŵ = inverse_transform(model, W) # embedding of `W` in original space +``` + +See also [`fit`](@ref), [`transform`](@ref), [`predict`](@ref). + +# Extended help + +# New implementations + +Implementation is optional. $(DOC_IMPLEMENTED_METHODS(":(LearnAPI.inverse_transform)")) + +$(DOC_MINIMIZE(:inverse_transform)) + +""" +function inverse_transform end diff --git a/src/target_weights_features.jl b/src/target_weights_features.jl new file mode 100644 index 00000000..c14f467b --- /dev/null +++ b/src/target_weights_features.jl @@ -0,0 +1,113 @@ +""" + LearnAPI.target(learner, observations) -> target + +Return, for every conceivable `observations` returned by a call of the form [`obs(learner, +data)`](@ref), the target variable part of `observations`. If `nothing` is returned, the +`learner` does not see a target variable in training (is unsupervised). + +The returned object `y` has the same number of observations as `observations` does and is +guaranteed to implement the data interface specified by +[`LearnAPI.data_interface(learner)`](@ref). It's form should be suitable for pairing with +the output of [`predict`](@ref), for example in a loss function. + +# Extended help + +## What is a target variable? + +Examples of target variables are house prices in real estate pricing estimates, the +"spam"/"not spam" labels in an email spam filtering task, "outlier"/"inlier" labels in +outlier detection, cluster labels in clustering problems, and censored survival times in +survival analysis. For more on targets and target proxies, see the "Reference" section of +the LearnAPI.jl documentation. + +## New implementations + +A fallback returns `nothing`. The method must be overloaded if [`fit`](@ref) consumes data +that includes a target variable. If `obs` is not being overloaded, then `observations` +above is any `data` supported in calls of the form [`fit(learner, data)`](@ref). The form +of the output `y` should be suitable for pairing with the output of [`predict`](@ref), in +the evaluation of a loss function, for example. + +Ensure the object `y` returned by `LearnAPI.target`, unless `nothing`, implements the data +interface specified by [`LearnAPI.data_interface(learner)`](@ref). + +$(DOC_IMPLEMENTED_METHODS(":(LearnAPI.target)"; overloaded=true)) + +""" +target(::Any, observations) = nothing + +""" + LearnAPI.weights(learner, observations) -> weights + +Return, for every conceivable `observations` returned by a call of the form [`obs(learner, +data)`](@ref), the weights part of `observations`. Where `nothing` is returned, no weights +are part of `data`, which is to be interpreted as uniform weighting. + +The returned object `w` has the same number of observations as `observations` does and is +guaranteed to implement the data interface specified by +[`LearnAPI.data_interface(learner)`](@ref). + +# Extended help + +# New implementations + +Overloading is optional. A fallback returns `nothing`. If `obs` is not being overloaded, +then `observations` above is any `data` supported in calls of the form [`fit(learner, +data)`](@ref). + +Ensure the returned object, unless `nothing`, implements the data interface specified by +[`LearnAPI.data_interface(learner)`](@ref). + +$(DOC_IMPLEMENTED_METHODS(":(LearnAPI.weights)"; overloaded=true)) + +""" +weights(::Any, observations) = nothing + +""" + LearnAPI.features(learner, observations) + +Return, for every conceivable `observations` returned by a call of the form [`obs(learner, +data)`](@ref), the "features" part of `observations` (as opposed to the target variable, +for example). + +It must always be possible to pass the returned object `X` to `predict` or `transform`, +where implemented, as in the following sample workflow: + +```julia +observations = obs(learner, data) +model = fit(learner, observations) +X = LearnAPI.features(learner, observations) +ŷ = predict(model, kind_of_proxy, X) # eg, `kind_of_proxy = Point()` +``` + +For supervised models (i.e., where `:(LearnAPI.target) in LearnAPI.functions(learner)`) +`ŷ` above is generally intended to be an approximate proxy for the target variable. + +The object `X` returned by `LearnAPI.features` has the same number of observations as +`observations` does and is guaranteed to implement the data interface specified by +[`LearnAPI.data_interface(learner)`](@ref). + +# Extended help + +# New implementations + +A fallback returns `first(observations)` if `observations` is a tuple, and otherwise +returns `observations`. New implementations may need to overload this method if this +fallback is inadequate. + +For density estimators, whose `fit` typically consumes *only* a target variable, you +should overload this method to return `nothing`. If `obs` is not being overloaded, then +`observations` above is any `data` supported in calls of the form [`fit(learner, +data)`](@ref). + +It must otherwise be possible to pass the return value `X` to `predict` and/or +`transform`, and `X` must have same number of observations as `data`. + +Ensure the returned object, unless `nothing`, implements the data interface specified by +[`LearnAPI.data_interface(learner)`](@ref). + +""" +features(learner, observations) = _first(observations) +_first(observations) = observations +_first(observations::Tuple) = first(observations) +# note the factoring above guards against method ambiguities diff --git a/src/tools.jl b/src/tools.jl index 88e1b788..e051b64b 100644 --- a/src/tools.jl +++ b/src/tools.jl @@ -8,53 +8,53 @@ function name_value_pair(ex) return (ex.args[1], ex.args[2]) end -macro trait(algorithm_ex, exs...) +""" + @trait(LearnerType, trait1=value1, trait2=value2, ...) + +Simultaneously overload a number of traits for learners of type `LearnerType`. For +example, the code + +```julia +@trait( + RidgeRegressor, + tags = ("regression", ), + doc_url = "https://some.cool.documentation", +) +``` + +is equivalent to + +```julia +LearnAPI.tags(::RidgeRegressor) = ("regression", ), +LearnAPI.doc_url(::RidgeRegressor) = "https://some.cool.documentation", +``` + +""" +macro trait(learner_ex, exs...) program = quote end for ex in exs trait_ex, value_ex = name_value_pair(ex) push!( program.args, - :($LearnAPI.$trait_ex(::$algorithm_ex) = $value_ex), + :($LearnAPI.$trait_ex(::$learner_ex) = $value_ex), ) end return esc(program) end -""" - typename(x) - -Return a symbolic representation of the name of `type(x)`, stripped of any type-parameters -and module qualifications. For example, if - - typeof(x) = MLJBase.Machine{MLJAlgorithms.ConstantRegressor,true} - -Then `typename(x)` returns `:Machine`. - -""" -function typename(x) - M = typeof(x) - if isdefined(M, :name) - return M.name.name - elseif isdefined(M, :body) - return typename(M.body) - else - return Symbol(string(M)) - end -end - function is_uppercase(char::Char) i = Int(char) i > 64 && i < 91 end -""" - snakecase(str, del='_') +# """ +# snakecase(str, del='_') -Return the snake case version of the abstract string or symbol, `str`, as in +# Return the snake case version of the abstract string or symbol, `str`, as in - snakecase("TheLASERBeam") == "the_laser_beam" +# snakecase("TheLASERBeam") == "the_laser_beam" -""" +# """ function snakecase(str::AbstractString; delim='_') snake = Char[] n = length(str) diff --git a/src/traits.jl b/src/traits.jl new file mode 100644 index 00000000..46004d17 --- /dev/null +++ b/src/traits.jl @@ -0,0 +1,493 @@ +# There are two types of traits - ordinary traits that an implementation overloads to make +# promises of learner behavior, and derived traits, which are never overloaded. + +const DOC_UNKNOWN = + "Returns `\"unknown\"` if the learner implementation has "* + "not overloaded the trait. " +const DOC_ON_TYPE = "The value of the trait must depend only on the type of `learner`. " + +const DOC_EXPLAIN_EACHOBS = + """ + + Here, "for each `o` in `observations`" is understood in the sense of + [`LearnAPI.data_interface(learner)`](@ref). For example, if + `LearnAPI.data_interface(learner) == Base.HasLength()`, then this means "for `o` in + `MLUtils.eachobs(observations)`". + + """ + +# # OVERLOADABLE TRAITS + +""" + Learn.API.constructor(learner) + +Return a keyword constructor that can be used to clone `learner`: + +```julia-repl +julia> learner.lambda +0.1 +julia> C = LearnAPI.constructor(learner) +julia> learner2 = C(lambda=0.2) +julia> learner2.lambda +0.2 +``` + +# New implementations + +All new implementations must overload this trait. + +Attach public LearnAPI.jl-related documentation for learner to the constructor, not +the learner struct. + +It must be possible to recover learner from the constructor returned as follows: + +```julia +properties = propertynames(learner) +named_properties = NamedTuple{properties}(getproperty.(Ref(learner), properties)) +@assert learner == LearnAPI.constructor(learner)(; named_properties...) +``` + +which can be tested with `@assert LearnAPI.clone(learner) == learner`. + +The keyword constructor provided by `LearnAPI.constructor` must provide default values for +all properties, with the exception of those that can take other LearnAPI.jl learners as +values. These can be provided with the default `nothing`, with the constructor throwing an +error if the default value persists. + +""" +function constructor end + +""" + LearnAPI.functions(learner) + +Return a tuple of expressions representing functions that can be meaningfully applied with +`learner`, or an associated model (object returned by `fit(learner, ...)`), as the first +argument. Learner traits (methods for which `learner` is the *only* argument) are +excluded. + +To return actual functions, instead of symbols, use [`@functions`](@ref)` learner` +instead. + +The returned tuple may include expressions like `:(DecisionTree.print_tree)`, which +reference functions not owned by LearnAPI.jl. + +The understanding is that `learner` is a LearnAPI-compliant object whenever the return +value is non-empty. + +Do `LearnAPI.functions()` to list all possible elements of the return value owned by +LearnAPI.jl. + +# Extended help + +# New implementations + +All new implementations must implement this trait. Here's a checklist for elements in the +return value: + +| expression | implementation compulsory? | include in returned tuple? | +|:----------------------------------|:---------------------------|:-----------------------------------| +| `:(LearnAPI.fit)` | yes | yes | +| `:(LearnAPI.learner)` | yes | yes | +| `:(LearnAPI.clone)` | never overloaded | yes | +| `:(LearnAPI.strip)` | no | yes | +| `:(LearnAPI.obs)` | no | yes | +| `:(LearnAPI.features)` | no | yes, unless `fit` consumes no data | +| `:(LearnAPI.target)` | no | only if implemented | +| `:(LearnAPI.weights)` | no | only if implemented | +| `:(LearnAPI.update)` | no | only if implemented | +| `:(LearnAPI.update_observations)` | no | only if implemented | +| `:(LearnAPI.update_features)` | no | only if implemented | +| `:(LearnAPI.predict)` | no | only if implemented | +| `:(LearnAPI.transform)` | no | only if implemented | +| `:(LearnAPI.inverse_transform)` | no | only if implemented | +| < accessor functions> | no | only if implemented | + +Also include any implemented accessor functions, both those owned by LearnaAPI.jl, and any +learner-specific ones. The LearnAPI.jl accessor functions are: $ACCESSOR_FUNCTIONS_LIST +(`LearnAPI.strip` is always included). + +""" +functions(::Any) = () +functions() = ( + :(LearnAPI.fit), + :(LearnAPI.learner), + :(LearnAPI.clone), + :(LearnAPI.strip), + :(LearnAPI.obs), + :(LearnAPI.features), + :(LearnAPI.target), + :(LearnAPI.weights), + :(LearnAPI.update), + :(LearnAPI.update_observations), + :(LearnAPI.update_features), + :(LearnAPI.predict), + :(LearnAPI.transform), + :(LearnAPI.inverse_transform), + ACCESSOR_FUNCTIONS..., +) + +""" + @functions learner + +Return a tuple of functions that can be meaningfully applied with `learner`, or an +associated model, as the first argument. An "associated model" is an object returned by +`fit(learner, ...)`. Learner traits (methods for which `learner` always the *only* +argument) are excluded. + +``` +julia> @functions my_feature_selector +(fit, LearnAPI.learner, strip, obs, transform) + +``` + +New learner implementations should overload [`LearnAPI.functions`](@ref). + +See also [`LearnAPI.functions`](@ref). + +""" +macro functions(learner) + quote + exs = LearnAPI.functions(learner) + eval.(exs) + end |> esc +end + +""" + LearnAPI.kinds_of_proxy(learner) + +Returns a tuple of all instances, `kind`, for which for which `predict(learner, kind, +data...)` has a guaranteed implementation. Each such `kind` subtypes +[`LearnAPI.KindOfProxy`](@ref). Examples are `Point()` (for predicting actual +target values) and `Distributions()` (for predicting probability mass/density functions). + +The call `predict(model, data)` always returns `predict(model, kind, data)`, where `kind` +is the first element of the trait's return value. + +See also [`LearnAPI.predict`](@ref), [`LearnAPI.KindOfProxy`](@ref). + +# Extended help + +# New implementations + +Must be overloaded whenever `predict` is implemented. + +Elements of the returned tuple must be instances of [`LearnAPI.KindOfProxy`](@ref). List +all possibilities by running `LearnAPI.kinds_of_proxy()`. + +Suppose, for example, we have the following implementation of a supervised learner +returning only probabilistic predictions: + +```julia +LearnAPI.predict(learner::MyNewLearnerType, LearnAPI.Distribution(), Xnew) = ... +``` + +Then we can declare + +```julia +@trait MyNewLearnerType kinds_of_proxy = (LearnaAPI.Distribution(),) +``` + +LearnAPI.jl provides the fallback for `predict(model, data)`. + +For more on target variables and target proxies, refer to the LearnAPI documentation. + +""" +kinds_of_proxy(::Any) = () +kinds_of_proxy() = map(CONCRETE_TARGET_PROXY_SYMBOLS) do ex + quote + $ex() + end |> eval +end + +tags() = [ + "regression", + "classification", + "clustering", + "gradient descent", + "iterative algorithms", + "incremental algorithms", + "feature engineering", + "dimension reduction", + "missing value imputation", + "transformers", + "static algorithms", + "ensembling", + "time series forecasting", + "time series classification", + "survival analysis", + "density estimation", + "Bayesian algorithms", + "outlier detection", + "collaborative filtering", + "text analysis", + "audio analysis", + "natural language processing", + "image processing", + "meta-algorithms" +] + +""" + LearnAPI.tags(learner) + +Lists one or more suggestive learner tags. Do `LearnAPI.tags()` to list +all possible. + +!!! warning + The value of this trait guarantees no particular behavior. The trait is + intended for informal classification purposes only. + +# New implementations + +This trait should return a tuple of strings, as in `("classifier", "text analysis")`. + +""" +tags(::Any) = () + +""" + LearnAPI.is_pure_julia(learner) + +Returns `true` if training `learner` requires evaluation of pure Julia code only. + +# New implementations + +The fallback is `false`. + +""" +is_pure_julia(::Any) = false + +""" + LearnAPI.pkg_name(learner) + +Return the name of the package module which supplies the core training algorithm for +`learner`. This is not necessarily the package providing the LearnAPI +interface. + +$DOC_UNKNOWN + +# New implementations + +Must return a string, as in `"DecisionTree"`. + +""" +pkg_name(::Any) = "unknown" + +""" + LearnAPI.pkg_license(learner) + +Return the name of the software license, such as `"MIT"`, applying to the package where the +core algorithm for `learner` is implemented. + +""" +pkg_license(::Any) = "unknown" + +""" + LearnAPI.doc_url(learner) + +Return a url where the core algorithm for `learner` is documented. + +$DOC_UNKNOWN + +# New implementations + +Must return a string, such as `"https://en.wikipedia.org/wiki/Decision_tree_learning"`. + +""" +doc_url(::Any) = "unknown" + +""" + LearnAPI.load_path(learner) + +Return a string indicating where in code the definition of the learner's constructor can +be found, beginning with the name of the package module defining it. By "constructor" we +mean the return value of [`LearnAPI.constructor(learner)`](@ref). + +# Implementation + +For example, a return value of `"FastTrees.LearnAPI.DecisionTreeClassifier"` means the +following julia code will not error: + +```julia +import FastTrees +import LearnAPI +@assert FastTrees.LearnAPI.DecisionTreeClassifier == LearnAPI.constructor(learner) +``` + +$DOC_UNKNOWN + + +""" +load_path(::Any) = "unknown" + + +""" + LearnAPI.nonlearners(learner) + +Return the properties of `learner` whose corresponding values are not themselves +learners. + +See also [`LearnAPI.learners`](@ref). + +# New implementations + +This trait should be overloaded if one or more properties (fields) of `learner` take +learner values. The fallback returns `propertynames(learner)`, meaning no properties have +learner values. If overloaded, implementation of the accessor function +[`LearnAPI.components`](@ref) is recommended. + +$DOC_ON_TYPE + + +""" +nonlearners(learner) = propertynames(learner) + +""" + LearnAPI.human_name(learner) + +Return a human-readable string representation of `typeof(learner)`. Primarily intended +for auto-generation of documentation. + +# New implementations + +Optional. A fallback takes the type name, inserts spaces and removes capitalization. For +example, `KNNRegressor` becomes `"knn regressor"`. Better would be to overload the trait +to return `"K-nearest neighbors regressor"`. Ideally, this is a "concrete" noun like +`"ridge regressor"` rather than an "abstract" noun like `"ridge regression"`. + +""" +human_name(learner) = snakecase(name(learner), delim=' ') # `name` defined below + +""" + LearnAPI.data_interface(learner) + +Return the data interface supported by `learner` for accessing individual observations +in representations of input data returned by [`obs(learner, data)`](@ref) or +[`obs(model, data)`](@ref), whenever `learner == LearnAPI.learner(model)`. Here `data` +is `fit`, `predict`, or `transform`-consumable data. + +Possible return values are [`LearnAPI.RandomAccess`](@ref), +[`LearnAPI.FiniteIterable`](@ref), and [`LearnAPI.Iterable`](@ref). + +See also [`obs`](@ref). + +# New implementations + +The fallback returns [`LearnAPI.RandomAccess`](@ref), which applies to arrays, most +tables, and tuples of these. See the doc-string for details. + +""" +data_interface(::Any) = LearnAPI.RandomAccess() + +""" + LearnAPI.is_static(learner) + +Returns `true` if [`fit`](@ref) is called with no data arguments, as in +`fit(learner)`. That is, `learner` does not generalize to new data, and data is only +provided at the `predict` or `transform` step. + +For example, some clustering algorithms are applied with this workflow, to assign labels +to the observations in `X`: + +```julia +model = fit(learner) # no training data +labels = predict(model, X) # may mutate `model`! + +# extract some byproducts of the clustering algorithm (e.g., outliers): +LearnAPI.extras(model) +``` + +# New implementations + +This trait, falling back to `false`, may only be overloaded when `fit` has no data +arguments. See more at [`fit`](@ref). + +""" +is_static(::Any) = false + +""" + LearnAPI.iteration_parameter(learner) + +The name of the iteration parameter of `learner`, or `nothing` if the algorithm is not +iterative. + +# New implementations + +Implement if algorithm is iterative. Returns a symbol or `nothing`. + +""" +iteration_parameter(::Any) = nothing + + +""" + LearnAPI.fit_observation_scitype(learner) + +Return an upper bound `S` on the scitype of individual observations guaranteed to work +when calling `fit`: if `observations = obs(learner, data)` and +`ScientificTypes.scitype(collect(o)) <:S` for each `o` in `observations`, then the call +`fit(learner, data)` is supported. + +$DOC_EXPLAIN_EACHOBS + +See also [`LearnAPI.target_observation_scitype`](@ref). + +# New implementations + +Optional. The fallback return value is `Union{}`. + +""" +fit_observation_scitype(::Any) = Union{} + +""" + LearnAPI.target_observation_scitype(learner) + +Return an upper bound `S` on the scitype of each observation of an applicable target +variable. Specifically: + +- If `:(LearnAPI.target) in LearnAPI.functions(learner)` (i.e., `fit` consumes target + variables) then "target" means anything returned by `LearnAPI.target(learner, data)`, + where `data` is an admissible argument in the call `fit(learner, data)`. + +- `S` will always be an upper bound on the scitype of (point) observations that could be + conceivably extracted from the output of [`predict`](@ref). + +To illustate the second case, suppose we have + +```julia +model = fit(learner, data) +ŷ = predict(model, Sampleable(), data_new) +``` + +Then each individual sample generated by each "observation" of `ŷ` (a vector of sampleable +objects, say) will be bound in scitype by `S`. + +See also See also [`LearnAPI.fit_observation_scitype`](@ref). + +# New implementations + +Optional. The fallback return value is `Any`. + +""" +target_observation_scitype(::Any) = Any + + +# # DERIVED TRAITS + +name(learner) = split(string(constructor(learner)), ".") |> last + +""" + LearnAPI.learners(learner) + +Return the properties of `learner` whose corresponding values are themselves +learners. + +See also [`LearnAPI.learners`](@ref). + +# New implementations + +This trait should not be overloaded. Instead overload [`LearnAPI.nonlearners`](@ref). + +""" +learners(learner) = setdiff(propertynames(learner), nonlearners(learner)) +is_learner(learner) = !isempty(functions(learner)) +preferred_kind_of_proxy(learner) = first(kinds_of_proxy(learner)) +target(learner) = :(LearnAPI.target) in functions(learner) +weights(learner) = :(LearnAPI.weights) in functions(learner) diff --git a/src/types.jl b/src/types.jl new file mode 100644 index 00000000..faa6d250 --- /dev/null +++ b/src/types.jl @@ -0,0 +1,261 @@ +# # TARGET PROXIES + +# see later for doc string: +abstract type KindOfProxy end + +""" + LearnAPI.IID <: LearnAPI.KindOfProxy + +Abstract subtype of [`LearnAPI.KindOfProxy`](@ref). If `kind_of_proxy` is an instance of +`LearnAPI.IID` then, given `data` constisting of ``n`` observations, the +following must hold: + +- `ŷ = LearnAPI.predict(model, kind_of_proxy, data)` is + data also consisting of ``n`` observations. + +- The ``j``th observation of `ŷ`, for any ``j``, depends only on the ``j``th + observation of the provided `data` (no correlation between observations). + +See also [`LearnAPI.KindOfProxy`](@ref). + +# Extended help + +| type | form of an observation | +|:-------------------------------------:|:----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `Point` | same as target observations; may have the interpretation of a 50% quantile, 50% expectile or mode | +| `Sampleable` | object that can be sampled to obtain object of the same form as target observation | +| `Distribution` | explicit probability density/mass function whose sample space is all possible target observations | +| `LogDistribution` | explicit log-probability density/mass function whose sample space is possible target observations | +| `Probability`¹ | numerical probability or probability vector | +| `LogProbability`¹ | log-probability or log-probability vector | +| `Parametric`¹ | a list of parameters (e.g., mean and variance) describing some distribution | +| `LabelAmbiguous` | collections of labels (in case of multi-class target) but without a known correspondence to the original target labels (and of possibly different number) as in, e.g., clustering | +| `LabelAmbiguousSampleable` | sampleable version of `LabelAmbiguous`; see `Sampleable` above | +| `LabelAmbiguousDistribution` | pdf/pmf version of `LabelAmbiguous`; see `Distribution` above | +| `LabelAmbiguousFuzzy` | same as `LabelAmbiguous` but with multiple values of indeterminant number | +| `Quantile`² | same as target but with quantile interpretation | +| `Expectile`² | same as target but with expectile interpretation | +| `ConfidenceInterval`² | confidence interval | +| `Fuzzy` | finite but possibly varying number of target observations | +| `ProbabilisticFuzzy` | as for `Fuzzy` but labeled with probabilities (not necessarily summing to one) | +| `SurvivalFunction` | survival function | +| `SurvivalDistribution` | probability distribution for survival time | +| `SurvivalHazardFunction` | hazard function for survival time | +| `OutlierScore` | numerical score reflecting degree of outlierness (not necessarily normalized) | +| `Continuous` | real-valued approximation/interpolation of a discrete-valued target, such as a count (e.g., number of phone calls) | + +¹Provided for completeness but discouraged to avoid [ambiguities in +representation](https://github.com/alan-turing-institute/MLJ.jl/blob/dev/paper/paper.md#a-unified-approach-to-probabilistic-predictions-and-their-evaluation). + +²The level will be controlled by a hyper-parameter; models providing only quantiles or +expectiles at 50% will provide `Point` instead. + +""" +abstract type IID <: KindOfProxy end + +const IID_SYMBOLS = [ + :Point, + :Sampleable, + :Distribution, + :LogDistribution, + :Probability, + :LogProbability, + :Parametric, + :LabelAmbiguous, + :LabelAmbiguousSampleable, + :LabelAmbiguousDistribution, + :LabelAmbiguousFuzzy, + :ConfidenceInterval, + :Fuzzy, + :ProbabilisticFuzzy, + :SurvivalFunction, + :SurvivalDistribution, + :HazardFunction, + :OutlierScore, + :Continuous, + :Quantile, + :Expectile, +] + +for S in IID_SYMBOLS + quote + struct $S <: IID end + end |> eval +end + + +""" + Joint <: KindOfProxy + +Abstract subtype of [`LearnAPI.KindOfProxy`](@ref). If `kind_of_proxy` is an instance of +`LearnAPI.Joint` then, given `data` consisting of ``n`` observations, `predict(model, +kind_of_proxy, data)` represents a *single* probability distribution for the sample +space ``Y^n``, where ``Y`` is the space from which the target variable takes its values. + +| type `T` | form of output of `predict(model, ::T, data)` | +|:-------------------------------:|:---------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `JointSampleable` | object that can be sampled to obtain a *vector* whose elements have the form of target observations; the vector length matches the number of observations in `data`. | +| `JointDistribution` | explicit probability density/mass function whose sample space is vectors of target observations; the vector length matches the number of observations in `data` | +| `JointLogDistribution` | explicit log-probability density/mass function whose sample space is vectors of target observations; the vector length matches the number of observations in `data` | + +""" +abstract type Joint <: KindOfProxy end + +const JOINT_SYMBOLS = [ + :JointSampleable, + :JointDistribution, + :JointLogDistribution, +] + +for S in JOINT_SYMBOLS + quote + struct $S <: Joint end + end |> eval +end + +""" + Single <: KindOfProxy + +Abstract subtype of [`LearnAPI.KindOfProxy`](@ref). It applies only to learners for +which `predict` has no data argument, i.e., is of the form `predict(model, +kind_of_proxy)`. An example is an algorithm learning a probability distribution from +samples, and we regard the samples as drawn from the "target" variable. If in this case, +`kind_of_proxy` is an instance of `LearnAPI.Single` then, `predict(learner)` returns a +single object representing a probability distribution. + +| type `T` | form of output of `predict(model, ::T)` | +|:--------------------------------:|:-----------------------------------------------------------------------| +| `SingleSampleable` | object that can be sampled to obtain a single target observation | +| `SingleDistribution` | explicit probability density/mass function for sampling the target | +| `SingleLogDistribution` | explicit log-probability density/mass function for sampling the target | + +""" +abstract type Single <: KindOfProxy end + +const SINGLE_SYMBOLS = [ + :SingleSampeable, + :SingleDistribution, + :SingleLogDistribution, +] + +for S in SINGLE_SYMBOLS + quote + struct $S <: Single end + end |> eval +end + +const CONCRETE_TARGET_PROXY_SYMBOLS = [ + IID_SYMBOLS..., + SINGLE_SYMBOLS..., + JOINT_SYMBOLS..., +] + +""" + + LearnAPI.KindOfProxy + +Abstract type whose concrete subtypes `T` each represent a different kind of proxy for +some target variable, associated with some learner. Instances `T()` are used to request +the form of target predictions in [`predict`](@ref) calls. + +See LearnAPI.jl documentation for an explanation of "targets" and "target proxies". + +For example, `Distribution` is a concrete subtype of `IID <: LearnAPI.KindOfProxy` and a +call like `predict(model, Distribution(), Xnew)` returns a data object whose observations +are probability density/mass functions, assuming `learner = LearnAPI.learner(model)` +supports predictions of that form, which is true if `Distribution() in` +[`LearnAPI.kinds_of_proxy(learner)`](@ref). + +Proxy types are grouped under three abstract subtypes: + +- [`LearnAPI.IID`](@ref): The main type, for proxies consisting of uncorrelated individual + components, one for each input observation + +- [`LearnAPI.Joint`](@ref): For learners that predict a single probabilistic structure + encapsulating correlations between target predictions for different input observations + +- [`LearnAPI.Single`](@ref): For learners, such as density estimators, that are trained on + a target variable only (no features); `predict` consumes no data and the returned target + proxy is a single probabilistic structure. + +For lists of all concrete instances, refer to documentation for the relevant subtype. + +""" +KindOfProxy + + +# # DATA INTERFACES + +abstract type DataInterface end +abstract type Finite <: DataInterface end + +""" + LearnAPI.RandomAccess + +A data interface type. We say that `data` implements the `RandomAccess` interface if +`data` implements the methods `getobs` and `numobs` from MLUtils.jl. The first method +allows one to grab observations specified by an arbitrary index set, as in +`MLUtils.getobs(data, [2, 3, 5])`, while the second method returns the total number of +available observations, which is assumed to be known and finite. + +All arrays implement `RandomAccess`, with the last index being the observation index +(observations-as-columns in matrices). + +A Tables.jl compatible table `data` implements `RandomAccess` if `Tables.istable(data)` is +true and if `data` implements `DataAPI.nrow`. This includes many tables, and in +particular, `DataFrame`s. Tables that are also tuples are explicitly excluded. + +Any tuple of objects implementing `RandomAccess` also implements `RandomAccess`. + +If [`LearnAPI.data_interface(learner)`](@ref) takes the value `RandomAccess()`, then +[`obs`](@ref)`(learner, ...)` is guaranteed to return objects implementing the +`RandomAccess` interface, and the same holds for `obs(model, ...)`, whenever +`LearnAPI.learner(model) == learner`. + +# Implementing `RandomAccess` for new data types + +Typically, to implement `RandomAccess` for a new data type requires only implementing +`Base.getindex` and `Base.length`, which are the fallbacks for `MLUtils.getobs` and +`MLUtils.numobs`, and this avoids making MLUtils.jl a package dependency. + +See also [`LearnAPI.FiniteIterable`](@ref), [`LearnAPI.Iterable`](@ref). +""" +struct RandomAccess <: Finite end + +""" + LearnAPI.FiniteIterable + +A data interface type. We say that `data` implements the `FiniteIterable` interface if +it implements Julia's `iterate` interface, including `Base.length`, and if +`Base.IteratorSize(typeof(data)) == Base.HasLength()`. For example, this is true if: + +- `data` implements the [`LearnAPI.RandomAccess`](@ref) interface (arrays and most + tables); or + +- `data isa MLUtils.DataLoader`, which includes output from `MLUtils.eachobs`. + +If [`LearnAPI.data_interface(learner)`](@ref) takes the value `FiniteIterable()`, then +[`obs`](@ref)`(learner, ...)` is guaranteed to return objects implementing the +`FiniteIterable` interface, and the same holds for `obs(model, ...)`, whenever +`LearnAPI.learner(model) == learner`. + +See also [`LearnAPI.RandomAccess`](@ref), [`LearnAPI.Iterable`](@ref). +""" +struct FiniteIterable <: Finite end + +""" + LearnAPI.Iterable + +A data interface type. We say that `data` implements the `Iterable` interface if it +implements Julia's basic `iterate` interface. (Such objects may not implement +`MLUtils.numobs` or `Base.length`.) + +If [`LearnAPI.data_interface(learner)`](@ref) takes the value `Iterable()`, then +[`obs`](@ref)`(learner, ...)` is guaranteed to return objects implementing `Iterable`, +and the same holds for `obs(model, ...)`, whenever `LearnAPI.learner(model) == +learner`. + +See also [`LearnAPI.FiniteIterable`](@ref), [`LearnAPI.RandomAccess`](@ref). + +""" +struct Iterable <: DataInterface end diff --git a/src/verbosity.jl b/src/verbosity.jl new file mode 100644 index 00000000..3723bb77 --- /dev/null +++ b/src/verbosity.jl @@ -0,0 +1,25 @@ +const DEFAULT_VERBOSITY = Ref(1) + +""" + LearnAPI.default_verbosity() + LearnAPI.default_verbosity(verbosity::Int) + +Respectively return, or set, the default `verbosity` level for LearnAPI.jl methods that +support it, which includes [`fit`](@ref), [`update`](@ref), +[`update_observations`](@ref), and [`update_features`](@ref). The effect in a top-level +call is generally: + + + +| `verbosity` | behaviour | +|:------------|:--------------| +| 1 | informational | +| 0 | warnings only | + + +Methods consuming `verbosity` generally call other verbosity-supporting methods +at one level lower, so increasing `verbosity` beyond `1` may be useful. + +""" +default_verbosity() = DEFAULT_VERBOSITY[] +default_verbosity(level) = (DEFAULT_VERBOSITY[] = level) diff --git a/test/accessor_functions.jl b/test/accessor_functions.jl new file mode 100644 index 00000000..0ee61e55 --- /dev/null +++ b/test/accessor_functions.jl @@ -0,0 +1,6 @@ +using Test +using LearnAPI + +@test LearnAPI.strip("junk") == "junk" + +true diff --git a/test/clone.jl b/test/clone.jl new file mode 100644 index 00000000..13c5ce5a --- /dev/null +++ b/test/clone.jl @@ -0,0 +1,22 @@ +using Test +using LearnAPI + +struct Potato + x + y +end + +Potato(; x=1, y=2) = Potato(x, y) +LearnAPI.constructor(::Potato) = Potato + +@test LearnAPI.clone(Potato()) == Potato() + +p = LearnAPI.clone(Potato(), y=20) +@test p.y == 20 +@test p.x == 1 + +q = LearnAPI.clone(Potato(), y=20, x=10) +@test q.y == 20 +@test q.x == 10 + +true diff --git a/test/obs.jl b/test/obs.jl new file mode 100644 index 00000000..b9ab8119 --- /dev/null +++ b/test/obs.jl @@ -0,0 +1,8 @@ +using Test +using LearnAPI + +@testset "`obs` fallback" begin + @test obs("some learner", 42) == 42 +end + +true diff --git a/test/predict_transform.jl b/test/predict_transform.jl new file mode 100644 index 00000000..3f9648c9 --- /dev/null +++ b/test/predict_transform.jl @@ -0,0 +1,27 @@ +using Test +using LearnAPI + +struct Cherry end + +LearnAPI.fit(learner::Cherry, data; verbosity=1) = Ref(learner) +LearnAPI.learner(model::Base.RefValue{Cherry}) = model[] +LearnAPI.predict(model::Base.RefValue{Cherry}, ::Point, x) = 2x +@trait Cherry kinds_of_proxy=(Point(),) + +struct Ripe end + +LearnAPI.fit(learner::Ripe, data; verbosity=1) = Ref(learner) +LearnAPI.learner(model::Base.RefValue{Ripe}) = model[] +LearnAPI.predict(model::Base.RefValue{Ripe}, ::Distribution) = "a distribution" +LearnAPI.features(::Ripe, data) = nothing +@trait Ripe kinds_of_proxy=(Distribution(),) + +@testset "`predict` with no kind of proxy specified" begin + model = fit(Cherry(), "junk") + @test predict(model, 42) == 84 + + model = fit(Ripe(), "junk") + @test predict(model) == "a distribution" +end + +true diff --git a/test/runtests.jl b/test/runtests.jl index a5c2a854..056fa491 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -1,7 +1,22 @@ -using LearnAPI using Test -using SparseArrays -@testset "tools.jl" begin - include("tools.jl") +test_files = [ + "tools.jl", + "verbosity.jl", + "traits.jl", + "clone.jl", + "predict_transform.jl", + "obs.jl", + "accessor_functions.jl", + "target_features.jl", +] + +files = isempty(ARGS) ? test_files : ARGS + +for file in files + quote + @testset $file begin + include($file) + end + end |> eval end diff --git a/test/target_features.jl b/test/target_features.jl new file mode 100644 index 00000000..b84ded25 --- /dev/null +++ b/test/target_features.jl @@ -0,0 +1,11 @@ +using Test +using LearnAPI + +struct Avocado end + +@test isnothing(LearnAPI.target(Avocado(), "salsa")) +@test isnothing(LearnAPI.weights(Avocado(), "salsa")) +@test LearnAPI.features(Avocado(), "salsa") == "salsa" +@test LearnAPI.features(Avocado(), (:X, :y)) == :X + +true diff --git a/test/tools.jl b/test/tools.jl index 15d5c29c..523f40e1 100644 --- a/test/tools.jl +++ b/test/tools.jl @@ -1,3 +1,6 @@ +using LearnAPI +using Test + module Fruit using LearnAPI @@ -18,13 +21,6 @@ import .Fruit ## HELPERS -@testset "typename" begin - @test LearnAPI.typename(Fruit.RedApple(1)) == :RedApple - @test LearnAPI.typename(nothing) == :Nothing - m = SparseArrays.sparse([1,2], [1,3], [0.5, 0.6]) - @test LearnAPI.typename(m) == :SparseMatrixCSC -end - @testset "snakecase" begin @test LearnAPI.snakecase("AnthonyBlaomsPetElk") == "anthony_blaoms_pet_elk" diff --git a/test/traits.jl b/test/traits.jl new file mode 100644 index 00000000..8b0353f3 --- /dev/null +++ b/test/traits.jl @@ -0,0 +1,77 @@ +using Test +using LearnAPI + +# A MINIMUM IMPLEMENTATION OF A LEARNER + +# does nothing useful +struct SmallLearner end +LearnAPI.fit(learner::SmallLearner, data; verbosity=1) = learner +LearnAPI.learner(model::SmallLearner) = model +@trait( + SmallLearner, + constructor = SmallLearner, + functions = ( + :(LearnAPI.fit), + :(LearnAPI.learner), + :(LearnAPI.clone), + :(LearnAPI.strip), + :(LearnAPI.obs), + :(LearnAPI.features), + ), +) +######## END OF IMPLEMENTATION ################## + +# ZERO ARGUMENT METHODS + +@test :(LearnAPI.fit) in LearnAPI.functions() +@test Point() in LearnAPI.kinds_of_proxy() +@test "regression" in LearnAPI.tags() + +# OVERLOADABLE TRAITS + +small = SmallLearner() +@test LearnAPI.constructor(small) == SmallLearner +@test :(LearnAPI.learner) in LearnAPI.functions(small) +@test isempty(LearnAPI.kinds_of_proxy(small)) +@test isempty(LearnAPI.tags(small)) +@test !LearnAPI.is_pure_julia(small) +@test LearnAPI.pkg_name(small) == "unknown" +@test isempty(LearnAPI.nonlearners(small)) +@test LearnAPI.pkg_license(small) == "unknown" +@test LearnAPI.doc_url(small) == "unknown" +@test LearnAPI.load_path(small) == "unknown" +@test LearnAPI.human_name(small) == "small learner" +@test isnothing(LearnAPI.iteration_parameter(small)) +@test LearnAPI.data_interface(small) == LearnAPI.RandomAccess() +@test !(6 isa LearnAPI.fit_observation_scitype(small)) +@test 6 isa LearnAPI.target_observation_scitype(small) +@test !LearnAPI.is_static(small) + +# DERIVED TRAITS + +@test isempty(LearnAPI.learners(small)) +@trait SmallLearner kinds_of_proxy=(Point(),) +@test LearnAPI.is_learner(small) +@test !LearnAPI.is_learner("junk") +@test !LearnAPI.target(small) +@test !LearnAPI.weights(small) +@test LearnAPI.preferred_kind_of_proxy(small) == Point() + +module FruitSalad +import LearnAPI + +struct RedApple{T} + x::T +end + +LearnAPI.constructor(::RedApple) = RedApple + +end + +import .FruitSalad + +@testset "name" begin + @test LearnAPI.name(FruitSalad.RedApple(1)) == "RedApple" +end + +true diff --git a/test/verbosity.jl b/test/verbosity.jl new file mode 100644 index 00000000..72ce29c8 --- /dev/null +++ b/test/verbosity.jl @@ -0,0 +1,7 @@ +using Test + +@test LearnAPI.default_verbosity() ==1 +LearnAPI.default_verbosity(42) +@test LearnAPI.default_verbosity() == 42 + +true diff --git a/typos.toml b/typos.toml new file mode 100644 index 00000000..8f5d6f5a --- /dev/null +++ b/typos.toml @@ -0,0 +1,6 @@ +[default.extend-words] +# Don't correct "mape" to "map" +mape = "mape" +yhat = "yhat" +LSO ="LSO" +datas = "datas"