adapt to new behaviour of CategoricalArrays.levels

Author: ablaom

src/MLJTransforms.jl

@@ -20,6 +20,9 @@ using OrderedCollections
 
 const MMI = MLJModelInterface
 
+# old behaviour of `levels` (before CategoricalArrays 1.0):
+rawlevels(A) = unwrap.(levels(A))
+
 # Functions of generic use across transformers
 include("common_docs.jl")
 include("generic.jl")

src/encoders/contrast_encoder/contrast_encoder.jl

@@ -102,7 +102,7 @@ function contrast_encoder_fit(
 
     # ensure mode is one of :contrast, :dummy, :sum, :backward_diff, :forward_diff, :helmert, :polynomial, :hypothesis
     function feature_mapper(col, name)
-        feat_levels = levels(col)
+        feat_levels = rawlevels(col)
         k = length(feat_levels)
         feat_mode = (mode_is_vector) ? mode[findfirst(isequal(name), features)] : mode
         if feat_mode == :contrast

src/encoders/frequency_encoding/frequency_encoding.jl

@@ -29,7 +29,7 @@ function frequency_encoder_fit(
     # 1. Define feature mapper
     function feature_mapper(col, name)
         frequency_map = (!normalize) ? countmap(col) : proportionmap(col)
-        feat_levels = levels(col)
+        feat_levels = rawlevels(col)
         statistic_given_feat_val = Dict{eltype(feat_levels), output_type}(
             level => get(frequency_map, level, 0) for level in feat_levels
         )

src/encoders/missingness_encoding/missingness_encoding.jl

@@ -39,7 +39,7 @@ function missingness_encoder_fit(
 
     # 1. Define feature mapper
     function feature_mapper(col, name)
-        feat_levels = levels(col; skipmissing = true)
+        feat_levels = unwrap.(levels(col; skipmissing = true))
         col_type = nonmissingtype(eltype(feat_levels))
 
         # Ensure column type is valid (can't test because never occurs)

src/encoders/ordinal_encoding/ordinal_encoding.jl

@@ -25,7 +25,7 @@ function ordinal_encoder_fit(
 )
     # 1. Define feature mapper
     function feature_mapper(col, name)
-        feat_levels = levels(col)
+        feat_levels = rawlevels(col)
         index_given_feat_val =
             Dict{eltype(feat_levels), output_type}(
                 value => index for (index, value) in enumerate(feat_levels)

src/encoders/target_encoding/target_encoding.jl

@@ -148,12 +148,12 @@ function target_encoder_fit(
         "Your target must be Continuous/Count for regression or Multiclass/OrderedFactor for classification",
     )
 
-    # 2. Setup prior statistics 
+    # 2. Setup prior statistics
     if task == "Regression"
         y_mean = mean(y)                             # for mixing
         m == :auto && (y_var = std(y)^2)              # for empirical Bayes estimation
     else
-        y_classes = levels(y)
+        y_classes = rawlevels(y)
         is_multiclass = length(y_classes) > 2
         if !is_multiclass       # binary case
             y_prior = sum(y .== y_classes[1]) / length(y)   # for mixing
@@ -165,10 +165,10 @@ function target_encoder_fit(
 
     # 3. Define function to compute the new value(s) for each level given a column
     function feature_mapper(col, name)
-        feat_levels = levels(col)
+        feat_levels = rawlevels(col)
         y_stat_given_feat_level_for_col =
             Dict{eltype(feat_levels), Any}()
-        for level in levels(col)
+        for level in rawlevels(col)
             # Get the targets of an example that belong to this level
             targets_for_level = y[col.==level]
 
@@ -230,14 +230,14 @@ end
 Transform given data with fitted target encoder cache.
 
 # Arguments
-- `X`: A table where the elements of the categorical features have [scitypes](https://juliaai.github.io/ScientificTypes.jl/dev/) 
+- `X`: A table where the elements of the categorical features have [scitypes](https://juliaai.github.io/ScientificTypes.jl/dev/)
 `Multiclass` or `OrderedFactor`
-- `cache`: A dictionary containing a dictionary `y_stat_given_feat_level` with the necessary statistics for 
+- `cache`: A dictionary containing a dictionary `y_stat_given_feat_level` with the necessary statistics for
 every categorical feature as well as other metadata needed for transform
 
 # Returns
 - `X`: A table where the categorical features as specified during fitting are transformed by target encoding. Other features will remain
-    the same. This will attempt to preserve the type of the table but may not succeed. 
+    the same. This will attempt to preserve the type of the table but may not succeed.
 """
 
 function target_encoder_transform(X, cache)
@@ -253,4 +253,3 @@ function target_encoder_transform(X, cache)
         use_levelnames = true,
         custom_levels = y_classes)
 end
-

src/generic.jl

@@ -207,7 +207,7 @@ function generic_transform(
         if feat_name in keys(mapping_per_feat_level)
             if !ignore_unknown
                 train_levels = keys(mapping_per_feat_level[feat_name])
-                test_levels = levels(col)
+                test_levels = rawlevels(col)
                 # test levels must be a subset of train levels
                 if !issubset(test_levels, train_levels)
                     # get the levels in test that are not in train

src/transformers/cardinality_reducer/cardinality_reducer.jl

@@ -46,7 +46,7 @@ function cardinality_reducer_fit(
     # 1. Define feature mapper
     function feature_mapper(col, name)
         val_to_freq = (min_frequency isa AbstractFloat) ? proportionmap(col) : countmap(col)
-        feat_levels = levels(col)
+        feat_levels = rawlevels(col)
         col_type = eltype(feat_levels)
 
         # Ensure column type is valid (can't test because never occurs)

src/transformers/other_transformers/one_hot_encoder.jl

@@ -61,7 +61,7 @@ function MMI.fit(transformer::OneHotEncoder, verbosity::Int, X)
         if T <: allowed_scitypes && ftr in specified_features
             ref_name_pairs_given_feature[ftr] = Pair{<:Unsigned,Symbol}[]
             shift = transformer.drop_last ? 1 : 0
-            levels = classes(col)
+            levels = CategoricalArrays.levels(col)
             fitted_levels_given_feature[ftr] = levels
             if verbosity > 0
                 @info "Spawning $(length(levels)-shift) sub-features "*
@@ -136,7 +136,7 @@ function MMI.transform(transformer::OneHotEncoder, fitresult, X)
         col = MMI.selectcols(X, ftr)
         if ftr in features_to_be_transformed
             Set(fitresult.fitted_levels_given_feature[ftr]) ==
-                Set(classes(col)) ||
+                Set(levels(col)) ||
             error("Found category level mismatch in feature `$(ftr)`. "*
             "Consider using `levels!` to ensure fitted and transforming "*
             "features have the same category levels.")
@@ -289,4 +289,4 @@ julia> schema(W)
 See also [`ContinuousEncoder`](@ref).
 
 """
-OneHotEncoder
+OneHotEncoder

test/encoders/contrast_encoder.jl

@@ -82,9 +82,9 @@ end
     end
     # test that fit is correct for dummy Coding
     cache = contrast_encoder_fit(X, [:name]; ignore = false, mode = :dummy)
-    k = length(levels(X.name))
+    k = length(rawlevels(X.name))
     contrast_matrix = get_dummy_contrast(k)
-    for (i, level) in enumerate(levels(X.name))
+    for (i, level) in enumerate(rawlevels(X.name))
         @test cache.vector_given_value_given_feature[:name][level] == contrast_matrix[i, :]
     end
 end
@@ -110,9 +110,9 @@ end
     @test size(contrast_matrix_3) == (3, 2)
     # test that fit is correct for sum Coding
     cache = contrast_encoder_fit(X, [:name, :favnum]; ignore = false, mode = :sum)
-    k = length(levels(X.favnum))
+    k = length(rawlevels(X.favnum))
     contrast_matrix = get_sum_contrast(k)
-    for (i, level) in enumerate(levels(X.favnum))
+    for (i, level) in enumerate(rawlevels(X.favnum))
         @test cache.vector_given_value_given_feature[:favnum][level] ==
               contrast_matrix[i, :]
     end
@@ -130,9 +130,9 @@ end
 
     # Test that fit is correct for backward Coding
     cache = contrast_encoder_fit(X, [:name, :favnum]; ignore = false, mode = :backward_diff)
-    k = length(levels(X.favnum))
+    k = length(rawlevels(X.favnum))
     contrast_matrix = get_backward_diff_contrast(k)
-    for (i, level) in enumerate(levels(X.favnum))
+    for (i, level) in enumerate(rawlevels(X.favnum))
         @test cache.vector_given_value_given_feature[:favnum][level] ==
               contrast_matrix[i, :]
     end
@@ -148,9 +148,9 @@ end
 
     # Test that fit is correct for forward Coding
     cache = contrast_encoder_fit(X, [:name, :favnum]; ignore = false, mode = :forward_diff)
-    k = length(levels(X.favnum))
+    k = length(rawlevels(X.favnum))
     contrast_matrix = get_forward_diff_contrast(k)
-    for (i, level) in enumerate(levels(X.favnum))
+    for (i, level) in enumerate(rawlevels(X.favnum))
         @test cache.vector_given_value_given_feature[:favnum][level] ==
               contrast_matrix[i, :]
     end
@@ -171,9 +171,9 @@ end
          0.0   0.0   3.0]
     # test that fit is correct for helmert Coding
     cache = contrast_encoder_fit(X, [:name, :favnum]; ignore = false, mode = :helmert)
-    k = length(levels(X.name))
+    k = length(rawlevels(X.name))
     contrast_matrix = get_helmert_contrast(k)
-    for (i, level) in enumerate(levels(X.name))
+    for (i, level) in enumerate(rawlevels(X.name))
         @test cache.vector_given_value_given_feature[:name][level] == contrast_matrix[i, :]
     end
 end
@@ -227,12 +227,12 @@ end
         contrasts = Dict(
             :name => StatsModels.HypothesisCoding(
                 buildrandomhypothesis(nothing, 3);
-                levels = levels(X.name),
+                levels = rawlevels(X.name),
                 labels = [],
             ),
             :favnum => StatsModels.HypothesisCoding(
                 buildrandomhypothesis(nothing, 4);
-                levels = levels(X.favnum),
+                levels = rawlevels(X.favnum),
                 labels = [],
             ),
         ),
@@ -263,7 +263,7 @@ end
             StatsModels.HelmertCoding(),
             StatsModels.HypothesisCoding(
                 buildrandomhypothesis(nothing, k);
-                levels = (k == 3) ? levels(X.name) : levels(X.favnum),
+                levels = (k == 3) ? rawlevels(X.name) : rawlevels(X.favnum),
                 labels = [],
             ),
         ][ind]
@@ -304,7 +304,7 @@ end
                 StatsModels.HelmertCoding(),
                 StatsModels.HypothesisCoding(
                     buildrandomhypothesis(nothing, k);
-                    levels = (k == 3) ? levels(X.name) : levels(X.favnum),
+                    levels = (k == 3) ? rawlevels(X.name) : rawlevels(X.favnum),
                     labels = [],
                 ),
             ][ind]