data utils (#3)

Author: tlienart

Manifest.toml

@@ -7,12 +7,17 @@ version = "0.1.0"
 ScientificTypes = "321657f4-b219-11e9-178b-2701a2544e81"
 
 [compat]
-ScientificTypes = "^0.6"
+ScientificTypes = "^0.7"
 julia = "1"
 
 [extras]
+CategoricalArrays = "324d7699-5711-5eae-9e2f-1d82baa6b597"
+DataFrames = "a93c6f00-e57d-5684-b7b6-d8193f3e46c0"
+Distances = "b4f34e82-e78d-54a5-968a-f98e89d6e8f7"
+InteractiveUtils = "b77e0a4c-d291-57a0-90e8-8db25a27a240"
 Tables = "bd369af6-aec1-5ad0-b16a-f7cc5008161c"
 Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
 
 [targets]
-test = ["Test", "Tables"]
+test = ["Test", "Tables", "Distances", "CategoricalArrays", "InteractiveUtils",
+        "DataFrames"]

Project.toml

@@ -2,13 +2,43 @@ module MLJModelInterface
 
 # ------------------------------------------------------------------------
 # Dependency (note that ScientificTypes itself does not have dependencies)
-import ScientificTypes: trait
+using ScientificTypes
 
 # ------------------------------------------------------------------------
-# Single export: matrix, everything else is qualified in MLJBase
-export matrix
+# exports
+
+# types
+export LightInterface, FullInterface
+export MLJType, Model, Supervised, Unsupervised,
+       Probabilistic, Deterministic, Interval, Static,
+       UnivariateFinite
+
+# rexport types from ScientificTypes
+export Scientific, Found, Unknown, Known, Finite, Infinite,
+       OrderedFactor, Multiclass, Count, Continuous, Textual,
+       Binary, ColorImage, GrayImage, Table
+
+# constructor + metadata
+export @mlj_model, metadata_pkg, metadata_model, metadata_measure
+# api
+export fit, update, update_data, transform, inverse_transform,
+       fitted_params, predict, predict_mode, predict_mean, predict_median,
+       evaluate, clean!
+# traits
+export input_scitype, output_scitype, target_scitype,
+       is_pure_julia, package_name, package_license,
+       load_path, package_uuid, package_url,
+       is_wrapper, supports_weights, supports_online,
+       docstring, name, is_supervised,
+       prediction_type, implemented_methods, hyperparameters,
+       hyperparameter_types, hyperparameter_ranges
+
+# data operations
+export matrix, int, classes, decoder, table,
+       nrows, selectrows, selectcols, select
 
 # ------------------------------------------------------------------------
+# Mode trick
 
 abstract type Mode end
 struct LightInterface <: Mode end
@@ -24,24 +54,32 @@ struct InterfaceError <: Exception
     m::String
 end
 
-vtrait(X) = X |> trait |> Val
+# ------------------------------------------------------------------------
+# Model types
+
+abstract type MLJType end
+
+abstract type Model   <: MLJType end
+
+abstract type   Supervised <: Model end
+abstract type Unsupervised <: Model end
 
-"""
-    matrix(X; transpose=false)
+abstract type Probabilistic <: Supervised end
+abstract type Deterministic <: Supervised end
+abstract type      Interval <: Supervised end
 
-If `X <: AbstractMatrix`, return `X` or `permutedims(X)` if `transpose=true`.
-If `X` is a Tables.jl compatible table source, convert `X` into a `Matrix`.
-"""
-matrix(X; kw...) = matrix(vtrait(X), X, get_interface_mode(); kw...)
+abstract type Static <: Unsupervised end
+
+# ------------------------------------------------------------------------
+# includes
 
-matrix(::Val{:other}, X::AbstractMatrix, ::Mode; transpose=false) =
-    transpose ? permutedims(X) : X
+include("utils.jl")
 
-matrix(::Val{:other}, X, ::Mode; kw...) =
-    throw(ArgumentError("Function `matrix` only supports AbstractMatrix or " *
-                        "containers implementing the Tables interface."))
+include("data_utils.jl")
+include("metadata_utils.jl")
 
-matrix(::Val{:table}, X, ::LightInterface; kw...) =
-    throw(InterfaceError("Only `MLJModelInterface` loaded. Import `MLJBase`."))
+include("model_traits.jl")
+include("model_def.jl")
+include("model_api.jl")
 
 end # module

src/MLJModelInterface.jl

@@ -0,0 +1,276 @@
+
+vtrait(X) = X |> trait |> Val
+
+const REQUIRE = "(requires MLJBase to be loaded)"
+
+errlight(s) = throw(InterfaceError("Only `MLJModelInterface` is loaded. " *
+                                    "Import `MLJBase` in order to use `$s`."))
+
+# ------------------------------------------------------------------------
+# matrix
+
+"""
+    matrix(X; transpose=false)
+
+If `X <: AbstractMatrix`, return `X` or `permutedims(X)` if `transpose=true`.
+If `X` is a Tables.jl compatible table source, convert `X` into a `Matrix`
+$REQUIRE.
+"""
+matrix(X; kw...) = matrix(get_interface_mode(), vtrait(X), X; kw...)
+
+matrix(::Mode, ::Val{:other}, X::AbstractMatrix; transpose=false) =
+    transpose ? permutedims(X) : X
+
+matrix(::Mode, ::Val{:other}, X; kw...) =
+    throw(ArgumentError("Function `matrix` only supports AbstractMatrix or " *
+                        "containers implementing the Tables interface."))
+
+matrix(::LightInterface, ::Val{:table}, X; kw...) = errlight("matrix")
+
+# ------------------------------------------------------------------------
+# int
+
+"""
+   int(x)
+
+The positional integer of the `CategoricalString` or `CategoricalValue` `x`, in
+the ordering defined by the pool of `x`. The type of `int(x)` is the reference
+type of `x` $REQUIRE.
+
+Not to be confused with `x.ref`, which is unchanged by reordering of the pool
+of `x`, but has the same type.
+
+    int(X::CategoricalArray)
+    int(W::Array{<:CategoricalString})
+    int(W::Array{<:CategoricalValue})
+
+Broadcasted versions of `int`.
+
+    julia> v = categorical([:c, :b, :c, :a])
+    julia> levels(v)
+    3-element Array{Symbol,1}:
+     :a
+     :b
+     :c
+    julia> int(v)
+    4-element Array{UInt32,1}:
+     0x00000003
+     0x00000002
+     0x00000003
+     0x00000001
+
+See also: [`decoder`](@ref).
+"""
+int(x; kw...) = int(get_interface_mode(), x; kw...)
+
+int(::LightInterface, x; kw...) = errlight("int")
+
+# ------------------------------------------------------------------------
+# classes
+
+"""
+    classes(x)
+
+All the categorical elements with in the same pool as `x` (including `x`),
+returned as a list, with an ordering consistent with the pool $REQUIRE.
+Here `x` has `CategoricalValue` or `CategoricalString` type, and `classes(x)`
+is a vector of the same eltype. Note that `x in classes(x)` is always true.
+
+Not to be confused with `levels(x.pool)`. See the example below.
+
+    julia>  v = categorical([:c, :b, :c, :a])
+    4-element CategoricalArrays.CategoricalArray{Symbol,1,UInt32}:
+     :c
+     :b
+     :c
+     :a
+
+    julia> levels(v)
+    3-element Array{Symbol,1}:
+     :a
+     :b
+     :c
+
+    julia> x = v[4]
+    CategoricalArrays.CategoricalValue{Symbol,UInt32} :a
+
+    julia> classes(x)
+    3-element CategoricalArrays.CategoricalArray{Symbol,1,UInt32}:
+     :a
+     :b
+     :c
+
+    julia> levels(x.pool)
+    3-element Array{Symbol,1}:
+     :a
+     :b
+     :c
+
+"""
+classes(x) = classes(get_interface_mode(), x)
+
+classes(::LightInterface, x) = errlight("classes")
+
+# ------------------------------------------------------------------------
+# decoder
+
+"""
+    d = decoder(x)
+
+A callable object for decoding the integer representation of a
+`CategoricalString` or `CategoricalValue` sharing the same pool as `x`
+$REQUIRE. (Here `x` is of one of these two types.) Specifically, one has
+`d(int(y)) == y` for all `y in classes(x)`. One can also call `d` on integer
+arrays, in which case `d` is broadcast over all elements.
+
+    julia> v = categorical([:c, :b, :c, :a])
+    julia> int(v)
+    4-element Array{UInt32,1}:
+     0x00000003
+     0x00000002
+     0x00000003
+     0x00000001
+    julia> d = decoder(v[3])
+    julia> d(int(v)) == v
+    true
+
+*Warning:* It is *not* true that `int(d(u)) == u` always holds.
+
+See also: [`int`](@ref), [`classes`](@ref).
+"""
+decoder(x) = decoder(get_interface_mode(), x)
+
+decoder(::LightInterface, x) = errlight("decoder")
+
+# ------------------------------------------------------------------------
+# table
+
+"""
+    table(columntable; prototype=nothing)
+
+Convert a named tuple of vectors or tuples `columntable`, into a table of the
+"preferred sink type" of `prototype` $REQUIRE. This is often the type of
+`prototype` itself, when `prototype` is a sink; see the Tables.jl
+documentation. If `prototype` is not specified, then a named tuple of vectors
+is returned.
+
+    table(A::AbstractMatrix; names=nothing, prototype=nothing)
+
+Wrap an abstract matrix `A` as a Tables.jl compatible table with the specified
+column `names` (a tuple of symbols). If `names` are not specified,
+`names=(:x1, :x2, ..., :xn)` is used, where `n=size(A, 2)` $REQUIRE.
+
+If a `prototype` is specified, then the matrix is materialized as a table of
+the preferred sink type of `prototype`, rather than wrapped. Note that if
+`prototype` is *not* specified, then `matrix(table(A))` is essentially a no-op.
+"""
+table(X; kw...) = table(get_interface_mode(), X; kw...)
+
+table(::LightInterface, X; kw...) = errlight("table")
+
+# ------------------------------------------------------------------------
+# nrows, select, selectrows, selectcols
+
+"""
+    nrows(X)
+
+Return the number of rows for a table, abstract vector or matrix `X` $REQUIRE.
+"""
+nrows(X) = nrows(get_interface_mode(), vtrait(X), X)
+
+nrows(::Mode, ::Val{:other}, X::AbstractVecOrMat) = size(X, 1)
+
+nrows(::Mode, ::Val{:other}, X) =
+    throw(ArgumentError("Function `nrows` only supports AbstractVector or " *
+                        "AbstractMatrix or containers implementing the " *
+                        "Tables interface."))
+
+nrows(::LightInterface, ::Val{:table}, X) = errlight("table")
+
+"""
+    selectrows(X, r)
+
+Select single or multiple rows from a table, abstract vector or matrix `X`
+$REQUIRE. If `X` is tabular, the object returned is a table of the
+preferred sink type of `typeof(X)`, even if only a single row is selected.
+"""
+selectrows(X, r) = selectrows(get_interface_mode(), vtrait(X), X, r)
+
+selectrows(::Mode, ::Val{:other}, ::Nothing, r) = nothing
+
+selectrows(::Mode, ::Val{:other}, X::AbstractVector, r)          = X[r]
+selectrows(::Mode, ::Val{:other}, X::AbstractVector, r::Integer) = X[r:r]
+selectrows(::Mode, ::Val{:other}, X::AbstractVector, ::Colon)    = X
+
+selectrows(::Mode, ::Val{:other}, X::AbstractMatrix, r)          = X[r, :]
+selectrows(::Mode, ::Val{:other}, X::AbstractMatrix, r::Integer) = X[r:r, :]
+selectrows(::Mode, ::Val{:other}, X::AbstractMatrix, ::Colon)    = X
+
+selectrows(::Mode, ::Val{:other}, X, r) =
+    throw(ArgumentError("Function `selectrows` only supports AbstractVector " *
+                        "or AbstractMatrix or containers implementing the " * "Tables interface."))
+
+selectrows(::LightInterface, ::Val{:table}, X, r; kw...) =
+    errlight("selectrows")
+
+"""
+    selectcols(X, c)
+
+Select single or multiple columns from a matrix or table `X` $REQUIRE. If `c`
+is an abstract vector of integers or symbols, then the object returned
+is a table of the preferred sink type of `typeof(X)`. If `c` is a
+*single* integer or column, then an `AbstractVector` is returned.
+"""
+selectcols(X, c) = selectcols(get_interface_mode(), vtrait(X), X, c)
+
+selectcols(::Mode, ::Val{:other}, ::Nothing, c) = nothing
+
+selectcols(::Mode, ::Val{:other}, X::AbstractMatrix, r)       = X[:, r]
+selectcols(::Mode, ::Val{:other}, X::AbstractMatrix, ::Colon) = X
+
+selectcols(::Mode, ::Val{:other}, X, r) =
+    throw(ArgumentError("Function `selectcols` only supports AbstractMatrix " *
+                        "or containers implementing the Tables interface."))
+
+selectcols(::LightInterface, ::Val{:table}, X, c; kw...) =
+    errlight("selectcols")
+
+"""
+    select(X, r, c)
+
+Select element(s) of a table or matrix at row(s) `r` and column(s) `c`. In the
+case of sparse data where the key `(r, c)`, zero or `missing` is returned,
+depending on the value type. See also: [`selectrows`](@ref),
+[`selectcols`](@ref).
+"""
+select(X, r, c) = select(get_interface_mode(), vtrait(X), X, r, c)
+
+select(::Mode, ::Val, X, r, c) = selectcols(selectrows(X, r), c)
+
+# ------------------------------------------------------------------------
+# UnivariateFinite
+
+"""
+    UnivariateFinite(classes, p)
+
+A discrete univariate distribution whose finite support is the elements of the
+vector `classes`, and whose corresponding probabilities are elements of the
+vector `p`, which must sum to one $REQUIRE.. Here `classes` must have type
+`AbstractVector{<:CategoricalElement}` where
+
+    CategoricalElement = Union{CategoricalValue,CategoricalString}
+
+and all classes are assumed to share the same categorical pool.
+
+    UnivariateFinite(prob_given_class)
+
+A discrete univariate distribution whose finite support is the set of keys of
+the provided dictionary, `prob_given_class` $REQUIRE.. The dictionary keys must
+be of type `CategoricalElement` (see above) and the dictionary values specify
+the corresponding probabilities.
+"""
+UnivariateFinite(d::AbstractDict) = UnivariateFinite(get_interface_mode(), d)
+UnivariateFinite(c::AbstractVector, p::AbstractVector) =
+    UnivariateFinite(get_interface_mode(), c, p)
+
+UnivariateFinite(::LightInterface, a...) = errlight("UnivariateFinite")