🦙 Missingness encoder is here

EssamWisam · EssamWisam · commit 9fb4c2749506 · 2024-07-24T13:15:36.000-05:00
diff --git a/Project.toml b/Project.toml
@@ -9,6 +9,7 @@ CategoricalArrays = "324d7699-5711-5eae-9e2f-1d82baa6b597"
 Combinatorics = "861a8166-3701-5b0c-9a16-15d98fcdc6aa"
 Dates = "ade2ca70-3891-5945-98fb-dc099432e06a"
 Distributions = "31c24e10-a181-5473-b8eb-7969acd0382f"
+LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e"
 MLJModelInterface = "e80e1ace-859a-464e-9ed9-23947d8ae3ea"
 OrderedCollections = "bac558e1-5e72-5ebc-8fee-abe8a469f55d"
 Parameters = "d96e819e-fc66-5662-9728-84c9c7592b0a"
@@ -20,7 +21,7 @@ Tables = "bd369af6-aec1-5ad0-b16a-f7cc5008161c"
 
 [compat]
 CategoricalArrays = "0.10"
-MLJModelInterface = "1.10"
+MLJModelInterface = "1.11"
 ScientificTypes = "3.0"
 StatsBase = "0.34"
 TableOperations = "1.2"
@@ -29,10 +30,11 @@ julia = "1.6.7"
 
 [extras]
 DataFrames = "a93c6f00-e57d-5684-b7b6-d8193f3e46c0"
+StableRNGs = "860ef19b-820b-49d6-a774-d7a799459cd3"
 MLJBase = "a7f614a8-145f-11e9-1d2a-a57a1082229d"
 Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c"
+StatsModels = "3eaba693-59b7-5ba5-a881-562e759f1c8d"
 Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
-StableRNGs = "860ef19b-820b-49d6-a774-d7a799459cd3"
 
 [targets]
-test = ["Test", "DataFrames", "MLJBase", "Random", "StableRNGs"]
+test = ["Test", "DataFrames", "MLJBase", "Random", "StableRNGs", "StatsModels"]
diff --git a/src/MLJTransforms.jl b/src/MLJTransforms.jl
@@ -24,20 +24,27 @@ include("encoders/target_encoding/errors.jl")
 include("encoders/target_encoding/target_encoding.jl")
 include("encoders/target_encoding/interface_mlj.jl")
 export target_encoder_fit, target_encoder_transform, TargetEncoder
+export  TargetEncoder
 
 # Ordinal encoding
 include("encoders/ordinal_encoding/ordinal_encoding.jl")
 include("encoders/ordinal_encoding/interface_mlj.jl")
 export ordinal_encoder_fit, ordinal_encoder_transform, OrdinalEncoder
+export  OrdinalEncoder
 
 # Frequency encoding
 include("encoders/frequency_encoding/frequency_encoding.jl")
 include("encoders/frequency_encoding/interface_mlj.jl")
 export frequency_encoder_fit, frequency_encoder_transform, FrequencyEncoder
+export  FrequencyEncoder
 
 # Cardinality reduction
 include("transformers/cardinality_reducer/cardinality_reducer.jl")
 include("transformers/cardinality_reducer/interface_mlj.jl")
 export cardinality_reducer_fit, cardinality_reducer_transform, CardinalityReducer
+export  CardinalityReducer
+include("encoders/missingness_encoding/missingness_encoding.jl")
+include("encoders/missingness_encoding/interface_mlj.jl")
+export  MissingnessEncoder
 
 end
diff --git a/src/encoders/missingness_encoding/errors.jl b/src/encoders/missingness_encoding/errors.jl
@@ -0,0 +1,9 @@
+UNSUPPORTED_COL_TYPE_ME(col_type) =
+    "In MissingnessEncoder, elements have type $(col_type). The supported types are `Union{Char, AbstractString, Number}`"
+VALID_TYPES_NEW_VAL_ME(possible_col_type) =
+    "In MissingnessEncoder, label_for_missing keys have type $(possible_col_type). The supported types are `Union{Char, AbstractString, Number}`"
+COLLISION_NEW_VAL_ME(value) =
+    "In MissingnessEncoder, label_for_missing specifies new feature name $(value). However, this name already exists in one of the features. Please respecify label_for_missing."
+UNSPECIFIED_COL_TYPE_ME(col_type, label_for_missing) =
+    "In MissingnessEncoder, $(col_type) does not appear in label_for_missing which only has keys $(keys(label_for_missing))"
+
diff --git a/src/encoders/missingness_encoding/interface_mlj.jl b/src/encoders/missingness_encoding/interface_mlj.jl
@@ -0,0 +1,161 @@
+### MissingnessEncoder with MLJ Interface
+
+# 1. Interface Struct
+mutable struct MissingnessEncoder{
+    AS <: AbstractVector{Symbol},
+    T <: Type,
+    A <: Any,
+} <: Unsupervised
+    features::AS
+    ignore::Bool
+    ordered_factor::Bool
+    label_for_missing::Dict{T, A}
+end;
+
+# 2. Constructor
+function MissingnessEncoder(;
+    features = Symbol[],
+    ignore = true,
+    ordered_factor = false,
+    label_for_missing = Dict(
+        AbstractString => "missing",
+        Char => 'm',
+    ),
+)
+    return MissingnessEncoder(features, ignore, ordered_factor, label_for_missing)
+end;
+
+
+# 4. Fitted parameters (for user access)
+MMI.fitted_params(::MissingnessEncoder, fitresult) = (
+    new_cat_given_col_val = fitresult,
+)
+
+# 5. Fit method
+function MMI.fit(transformer::MissingnessEncoder, verbosity::Int, X)
+    generic_cache = missingness_encoder_fit(
+        X,
+        transformer.features;
+        ignore = transformer.ignore,
+        ordered_factor = transformer.ordered_factor,
+        label_for_missing = transformer.label_for_missing,
+    )
+    fitresult = generic_cache[:new_cat_given_col_val]
+
+    report = (encoded_features = generic_cache[:encoded_features],)        # report only has list of encoded features
+    cache = nothing
+    return fitresult, cache, report
+end;
+
+
+# 6. Transform method
+function MMI.transform(transformer::MissingnessEncoder, fitresult, Xnew)
+    generic_cache = Dict(
+        :new_cat_given_col_val =>
+            fitresult,
+    )
+    Xnew_transf = missingness_encoder_transform(Xnew, generic_cache)
+    return Xnew_transf
+end
+
+# 8. Extra metadata
+MMI.metadata_pkg(
+    MissingnessEncoder,
+    package_name = "MLJTransforms",
+    package_uuid = "23777cdb-d90c-4eb0-a694-7c2b83d5c1d6",
+    package_url = "https://github.com/JuliaAI/MLJTransforms.jl",
+    is_pure_julia = true,
+)
+
+MMI.metadata_model(
+    MissingnessEncoder,
+    input_scitype = Table,
+    output_scitype = Table,
+    load_path = "MLJTransforms.MissingnessEncoder",
+)
+
+
+
+"""
+$(MMI.doc_header(MissingnessEncoder))
+
+`MissingnessEncoder` maps any missing level of a categorical feature into a new level (e.g., "Missing"). 
+By this, missingness will be treated as a new
+level by any subsequent model. This assumes that the categorical features have raw
+types that are in `Union{Char, AbstractString, Number}`.
+
+# Training data
+
+In MLJ (or MLJBase) bind an instance unsupervised `model` to data with
+
+    mach = machine(model, X)
+
+Here:
+
+- `X` is any table of input features (eg, a `DataFrame`). Features to be transformed must
+   have element scitype `Multiclass` or `OrderedFactor`. Use `schema(X)` to 
+   check scitypes. 
+
+Train the machine using `fit!(mach, rows=...)`.
+
+# Hyper-parameters
+
+- `features=[]`: A list of names of categorical features given as symbols to exclude or include from encoding
+- `ignore=true`: Whether to exclude or includes the features given in `features`
+- `ordered_factor=false`: Whether to encode `OrderedFactor` or ignore them
+- `label_for_missing=Dict{<:Type, <:Any}()= Dict( AbstractString => "missing", Char => 'm', )`: A
+dictionary where the possible values for keys are the types in `Union{Char, AbstractString, Number}` and where each value
+signifies the new level to map into given a column raw super type. By default, if the raw type of the column subtypes `AbstractString`
+then missing values will be replaced with `"missing"` and if the raw type subtypes `Char` then the new value is `'m'`
+and if the raw type subtypes `Number` then the new value is the lowest value in the column - 1.
+
+# Operations
+
+- `transform(mach, Xnew)`: Apply cardinality reduction to selected `Multiclass` or `OrderedFactor` features of `Xnew` specified by hyper-parameters, and 
+   return the new table.   Features that are neither `Multiclass` nor `OrderedFactor`
+   are always left unchanged.
+
+# Fitted parameters
+
+The fields of `fitted_params(mach)` are:
+
+- `new_cat_given_col_val`: A dictionary that for each column, maps `missing` into some value according to `label_for_missing`
+
+# Report
+
+The fields of `report(mach)` are:
+
+- `encoded_features`: The subset of the categorical features of X that were encoded
+
+# Examples
+
+```julia
+import StatsBase.proportionmap
+using MLJ
+
+# Define a table with missing values
+Xm = (
+    A = categorical(["Ben", "John", missing, missing, "Mary", "John", missing]),
+    B = [1.85, 1.67, missing, missing, 1.5, 1.67, missing],
+    C= categorical([7, 5, missing, missing, 10, 0, missing]),
+    D = [23, 23, 44, 66, 14, 23, 11],
+    E = categorical([missing, 'g', 'r', missing, 'r', 'g', 'p'])
+)
+
+encoder = MissingnessEncoder(ordered_factor = false)
+mach = fit!(machine(encoder, Xm))
+Xnew = transform(mach, Xm)
+
+julia> Xnew
+(A = ["Ben", "John", "missing", "missing", "Mary", "John", "missing"],
+ B = Union{Missing, Float64}[1.85, 1.67, missing, missing, 1.5, 1.67, missing],
+ C = [7, 5, -1, -1, 10, 0, -1],
+ D = [23, 23, 44, 66, 14, 23, 11],
+ E = ['m', 'g', 'r', 'm', 'r', 'g', 'p'],)
+
+```
+
+See also
+[`CardinalityReducer`](@ref)
+"""
+MissingnessEncoder
diff --git a/src/encoders/missingness_encoding/missingness_encoding.jl b/src/encoders/missingness_encoding/missingness_encoding.jl
@@ -0,0 +1,121 @@
+include("errors.jl")
+
+"""
+**Private method.**
+
+Fit a transformer that maps any missing value into a new level (e.g., "Missing"). By this, missingness will be treated as a new
+level by any subsequent model. This assumes that the categorical features have raw
+types that are in `Union{Char, AbstractString, Number}`.
+
+# Arguments
+
+  - `X`: A table where the elements of the categorical features have [scitypes](https://juliaai.github.io/ScientificTypes.jl/dev/)
+    `Multiclass` or `OrderedFactor`
+  - `features=[]`: A list of names of categorical features given as symbols to exclude or include from encoding
+  - `ignore=true`: Whether to exclude or includes the features given in `features`
+  - `ordered_factor=false`: Whether to encode `OrderedFactor` or ignore them
+  - `label_for_missing=Dict{<:Type, <:Any}()= Dict( AbstractString => "missing", Char => 'm', )`: A
+    dictionary where the possible values for keys are the types in `Union{Char, AbstractString, Number}` and where each value
+    signifies the new level to map into given a column raw super type. By default, if the raw type of the column subtypes `AbstractString`
+    then missing values will be replaced with `"missing"` and if the raw type subtypes `Char` then the new value is `'m'`
+    and if the raw type subtypes `Number` then the new value is the lowest value in the column - 1.
+
+# Returns (in a dict)
+
+  - `new_cat_given_col_val`: A dictionary that for each column, maps `missing` into some value according to `label_for_missing`
+  - `encoded_features`: The subset of the categorical features of X that were encoded
+"""
+function missingness_encoder_fit(
+    X,
+    features::AbstractVector{Symbol} = Symbol[];
+    ignore::Bool = true,
+    ordered_factor::Bool = false,
+    label_for_missing::Dict{<:Type, <:Any} = Dict(    
+        AbstractString => "missing",
+        Char => 'm',
+    ),
+)
+    supportedtypes = Union{Char, AbstractString, Number}
+
+    # 1. Define feature mapper
+    function feature_mapper(col, name)
+        col_type = nonmissingtype(eltype(col)).parameters[1]
+        feat_levels = levels(col; skipmissing=true)
+
+        # Ensure column type is valid (can't test because never occurs)
+        # Converting array elements to strings before wrapping in a `CategoricalArray`, as...
+        if !(col_type <: supportedtypes)
+            throw(ArgumentError(UNSUPPORTED_COL_TYPE_ME(col_type)))
+        end
+
+        # Ensure label_for_missing keys are valid types
+        for possible_col_type in keys(label_for_missing)
+            if !(possible_col_type in union_types(supportedtypes))
+                throw(ArgumentError(VALID_TYPES_NEW_VAL_ME(possible_col_type)))
+            end
+        end
+
+        # Check no collision between keys(label_for_missing) and feat_levels
+        for value in values(label_for_missing)
+            if !ismissing(value) 
+                if value in feat_levels
+                    throw(ArgumentError(COLLISION_NEW_VAL_ME(value)))
+                end
+            end
+        end
+
+        # Get ancestor type of column
+        elgrandtype = nothing
+        for allowed_type in union_types(supportedtypes)
+            if col_type <: allowed_type
+                elgrandtype = allowed_type
+                break
+            end
+        end
+        
+        # Nonmissing levels remain as is
+        new_cat_given_col_val = Dict{Missing, col_type}()
+
+        # Missing levels are mapped
+        if elgrandtype in keys(label_for_missing)
+            new_cat_given_col_val[missing] = label_for_missing[elgrandtype]
+        elseif elgrandtype == Number
+            new_cat_given_col_val[missing] = minimum(feat_levels) - 1
+        else
+            throw(ArgumentError(UNSPECIFIED_COL_TYPE_ME(col_type, label_for_missing)))
+        end
+
+        return new_cat_given_col_val::Dict{Missing, col_type}
+    end
+
+    # 2. Pass it to generic_fit
+    new_cat_given_col_val, encoded_features = generic_fit(
+        X, features; ignore = ignore, ordered_factor = ordered_factor, feature_mapper = feature_mapper,
+    )
+    cache = Dict(
+        :new_cat_given_col_val => new_cat_given_col_val,
+        :encoded_features => encoded_features,
+    )
+    return cache
+end
+
+"""
+**Private method.**
+
+Apply a fitted missingness encoder to a table given the output of `missingness_encoder_fit`
+
+# Arguments
+
+  - `X`: A table where the elements of the categorical features have [scitypes](https://juliaai.github.io/ScientificTypes.jl/dev/)
+    `Multiclass` or `OrderedFactor`
+  - `cache`: The output of `missingness_encoder_fit`
+
+# Returns
+
+  - `X_tr`: The table with selected features after the selected features are transformed by missingness encoder
+"""
+function missingness_encoder_transform(X, cache::Dict)
+    new_cat_given_col_val = cache[:new_cat_given_col_val]
+    return generic_transform(X, new_cat_given_col_val; ignore_unknown = true)
+end
+
diff --git a/test/encoders/missingness_encoding.jl b/test/encoders/missingness_encoding.jl
diff --git a/test/runtests.jl b/test/runtests.jl
