960

Author: mathias-von-ottenbreit

API_REFERENCE_FOR_CLASSIFICATION.md

@@ -1,6 +1,6 @@
 # APLRClassifier
 
-## class aplr.APLRClassifier(m:int=3000, v:float=0.1, random_state:int=0, n_jobs:int=0, cv_folds:int=5, bins:int=300, verbosity:int=0, max_interaction_level:int=1, max_interactions:int=100000, min_observations_in_split:int=20, ineligible_boosting_steps_added:int=10, max_eligible_terms:int=5, boosting_steps_before_interactions_are_allowed: int = 0, monotonic_constraints_ignore_interactions: bool = False, early_stopping_rounds: int = 500, num_first_steps_with_linear_effects_only: int = 0)
+## class aplr.APLRClassifier(m:int=3000, v:float=0.1, random_state:int=0, n_jobs:int=0, cv_folds:int=5, bins:int=300, verbosity:int=0, max_interaction_level:int=1, max_interactions:int=100000, min_observations_in_split:int=20, ineligible_boosting_steps_added:int=10, max_eligible_terms:int=5, boosting_steps_before_interactions_are_allowed: int = 0, monotonic_constraints_ignore_interactions: bool = False, early_stopping_rounds: int = 500, num_first_steps_with_linear_effects_only: int = 0, penalty_for_non_linearity: float = 0.0, penalty_for_interactions: float = 0.0)
 
 ### Constructor parameters
 
@@ -52,6 +52,12 @@ If validation loss does not improve during the last ***early_stopping_rounds***
 #### num_first_steps_with_linear_effects_only (default = 0)
 Specifies the number of initial boosting steps that are reserved only for linear effects. 0 means that non-linear effects are allowed from the first boosting step. Reasons for setting this parameter to a higher value than 0 could be to 1) build a more interpretable model with more emphasis on linear effects or 2) build a linear only model by setting ***num_first_steps_with_linear_effects_only*** to no less than ***m***.
 
+#### penalty_for_non_linearity (default = 0.0)
+Specifies a penalty in the range [0.0, 1.0] on terms that are not linear effects. A higher value increases model interpretability but can hurt predictiveness.
+
+#### penalty_for_interactions (default = 0.0)
+Specifies a penalty in the range [0.0, 1.0] on interaction terms. A higher value increases model interpretability but can hurt predictiveness.
+
 
 ## Method: fit(X:npt.ArrayLike, y:List[str], sample_weight:npt.ArrayLike = np.empty(0), X_names:List[str]=[], cv_observations: npt.ArrayLike = np.empty([0, 0]), prioritized_predictors_indexes:List[int]=[], monotonic_constraints:List[int]=[], interaction_constraints:List[List[int]]=[])
 

API_REFERENCE_FOR_REGRESSION.md

@@ -1,6 +1,6 @@
 # APLRRegressor
 
-## class aplr.APLRRegressor(m:int=3000, v:float=0.1, random_state:int=0, loss_function:str="mse", link_function:str="identity", n_jobs:int=0, cv_folds:int=5, bins:int=300, max_interaction_level:int=1, max_interactions:int=100000, min_observations_in_split:int=20, ineligible_boosting_steps_added:int=10, max_eligible_terms:int=5, verbosity:int=0, dispersion_parameter:float=1.5, validation_tuning_metric:str="default", quantile:float=0.5, calculate_custom_validation_error_function:Optional[Callable[[npt.ArrayLike, npt.ArrayLike, npt.ArrayLike, npt.ArrayLike, npt.ArrayLike], float]]=None, calculate_custom_loss_function:Optional[Callable[[npt.ArrayLike, npt.ArrayLike, npt.ArrayLike, npt.ArrayLike, npt.ArrayLike], float]]=None, calculate_custom_negative_gradient_function:Optional[Callable[[npt.ArrayLike, npt.ArrayLike, npt.ArrayLike, npt.ArrayLike], npt.ArrayLike]]=None, calculate_custom_transform_linear_predictor_to_predictions_function:Optional[Callable[[npt.ArrayLike], npt.ArrayLike]]=None, calculate_custom_differentiate_predictions_wrt_linear_predictor_function:Optional[Callable[[npt.ArrayLike], npt.ArrayLike]]=None, boosting_steps_before_interactions_are_allowed: int = 0, monotonic_constraints_ignore_interactions: bool = False, group_mse_by_prediction_bins: int = 10, group_mse_cycle_min_obs_in_bin: int = 30, early_stopping_rounds: int = 500, num_first_steps_with_linear_effects_only: int = 0)
+## class aplr.APLRRegressor(m:int=3000, v:float=0.1, random_state:int=0, loss_function:str="mse", link_function:str="identity", n_jobs:int=0, cv_folds:int=5, bins:int=300, max_interaction_level:int=1, max_interactions:int=100000, min_observations_in_split:int=20, ineligible_boosting_steps_added:int=10, max_eligible_terms:int=5, verbosity:int=0, dispersion_parameter:float=1.5, validation_tuning_metric:str="default", quantile:float=0.5, calculate_custom_validation_error_function:Optional[Callable[[npt.ArrayLike, npt.ArrayLike, npt.ArrayLike, npt.ArrayLike, npt.ArrayLike], float]]=None, calculate_custom_loss_function:Optional[Callable[[npt.ArrayLike, npt.ArrayLike, npt.ArrayLike, npt.ArrayLike, npt.ArrayLike], float]]=None, calculate_custom_negative_gradient_function:Optional[Callable[[npt.ArrayLike, npt.ArrayLike, npt.ArrayLike, npt.ArrayLike], npt.ArrayLike]]=None, calculate_custom_transform_linear_predictor_to_predictions_function:Optional[Callable[[npt.ArrayLike], npt.ArrayLike]]=None, calculate_custom_differentiate_predictions_wrt_linear_predictor_function:Optional[Callable[[npt.ArrayLike], npt.ArrayLike]]=None, boosting_steps_before_interactions_are_allowed: int = 0, monotonic_constraints_ignore_interactions: bool = False, group_mse_by_prediction_bins: int = 10, group_mse_cycle_min_obs_in_bin: int = 30, early_stopping_rounds: int = 500, num_first_steps_with_linear_effects_only: int = 0,         penalty_for_non_linearity: float = 0.0, penalty_for_interactions: float = 0.0)
 
 ### Constructor parameters
 
@@ -120,6 +120,12 @@ If validation loss does not improve during the last ***early_stopping_rounds***
 #### num_first_steps_with_linear_effects_only (default = 0)
 Specifies the number of initial boosting steps that are reserved only for linear effects. 0 means that non-linear effects are allowed from the first boosting step. Reasons for setting this parameter to a higher value than 0 could be to 1) build a more interpretable model with more emphasis on linear effects or 2) build a linear only model by setting ***num_first_steps_with_linear_effects_only*** to no less than ***m***. 
 
+#### penalty_for_non_linearity (default = 0.0)
+Specifies a penalty in the range [0.0, 1.0] on terms that are not linear effects. A higher value increases model interpretability but can hurt predictiveness.
+
+#### penalty_for_interactions (default = 0.0)
+Specifies a penalty in the range [0.0, 1.0] on interaction terms. A higher value increases model interpretability but can hurt predictiveness.
+
 
 ## Method: fit(X:npt.ArrayLike, y:npt.ArrayLike, sample_weight:npt.ArrayLike = np.empty(0), X_names:List[str]=[], cv_observations: npt.ArrayLike = np.empty([0, 0]), prioritized_predictors_indexes:List[int]=[], monotonic_constraints:List[int]=[], group:npt.ArrayLike = np.empty(0), interaction_constraints:List[List[int]]=[], other_data: npt.ArrayLike = np.empty([0, 0]))
 

aplr/aplr.py

@@ -66,6 +66,8 @@ def __init__(
         group_mse_cycle_min_obs_in_bin: int = 30,
         early_stopping_rounds: int = 500,
         num_first_steps_with_linear_effects_only: int = 0,
+        penalty_for_non_linearity: float = 0.0,
+        penalty_for_interactions: float = 0.0,
     ):
         self.m = m
         self.v = v
@@ -109,6 +111,8 @@ def __init__(
         self.num_first_steps_with_linear_effects_only = (
             num_first_steps_with_linear_effects_only
         )
+        self.penalty_for_non_linearity = penalty_for_non_linearity
+        self.penalty_for_interactions = penalty_for_interactions
 
         # Creating aplr_cpp and setting parameters
         self.APLRRegressor = aplr_cpp.APLRRegressor()
@@ -166,6 +170,8 @@ def __set_params_cpp(self):
         self.APLRRegressor.num_first_steps_with_linear_effects_only = (
             self.num_first_steps_with_linear_effects_only
         )
+        self.APLRRegressor.penalty_for_non_linearity = self.penalty_for_non_linearity
+        self.APLRRegressor.penalty_for_interactions = self.penalty_for_interactions
 
     def fit(
         self,
@@ -294,6 +300,8 @@ def get_params(self, deep=True):
             "group_mse_cycle_min_obs_in_bin": self.group_mse_cycle_min_obs_in_bin,
             "early_stopping_rounds": self.early_stopping_rounds,
             "num_first_steps_with_linear_effects_only": self.num_first_steps_with_linear_effects_only,
+            "penalty_for_non_linearity": self.penalty_for_non_linearity,
+            "penalty_for_interactions": self.penalty_for_interactions,
         }
 
     # For sklearn
@@ -323,6 +331,8 @@ def __init__(
         monotonic_constraints_ignore_interactions: bool = False,
         early_stopping_rounds: int = 500,
         num_first_steps_with_linear_effects_only: int = 0,
+        penalty_for_non_linearity: float = 0.0,
+        penalty_for_interactions: float = 0.0,
     ):
         self.m = m
         self.v = v
@@ -346,6 +356,8 @@ def __init__(
         self.num_first_steps_with_linear_effects_only = (
             num_first_steps_with_linear_effects_only
         )
+        self.penalty_for_non_linearity = penalty_for_non_linearity
+        self.penalty_for_interactions = penalty_for_interactions
 
         # Creating aplr_cpp and setting parameters
         self.APLRClassifier = aplr_cpp.APLRClassifier()
@@ -377,6 +389,8 @@ def __set_params_cpp(self):
         self.APLRClassifier.num_first_steps_with_linear_effects_only = (
             self.num_first_steps_with_linear_effects_only
         )
+        self.APLRClassifier.penalty_for_non_linearity = self.penalty_for_non_linearity
+        self.APLRClassifier.penalty_for_interactions = self.penalty_for_interactions
 
     def fit(
         self,
@@ -450,6 +464,8 @@ def get_params(self, deep=True):
             "monotonic_constraints_ignore_interactions": self.monotonic_constraints_ignore_interactions,
             "early_stopping_rounds": self.early_stopping_rounds,
             "num_first_steps_with_linear_effects_only": self.num_first_steps_with_linear_effects_only,
+            "penalty_for_non_linearity": self.penalty_for_non_linearity,
+            "penalty_for_interactions": self.penalty_for_interactions,
         }
 
     # For sklearn

cpp/APLRClassifier.h

@@ -46,12 +46,15 @@ class APLRClassifier
     bool monotonic_constraints_ignore_interactions;
     size_t early_stopping_rounds;
     size_t num_first_steps_with_linear_effects_only;
+    double penalty_for_non_linearity;
+    double penalty_for_interactions;
 
     APLRClassifier(size_t m = 3000, double v = 0.1, uint_fast32_t random_state = std::numeric_limits<uint_fast32_t>::lowest(), size_t n_jobs = 0,
                    size_t cv_folds = 5, size_t reserved_terms_times_num_x = 100, size_t bins = 300, size_t verbosity = 0, size_t max_interaction_level = 1,
                    size_t max_interactions = 100000, size_t min_observations_in_split = 20, size_t ineligible_boosting_steps_added = 10, size_t max_eligible_terms = 5,
                    size_t boosting_steps_before_interactions_are_allowed = 0, bool monotonic_constraints_ignore_interactions = false,
-                   size_t early_stopping_rounds = 500, size_t num_first_steps_with_linear_effects_only = 0);
+                   size_t early_stopping_rounds = 500, size_t num_first_steps_with_linear_effects_only = 0,
+                   double penalty_for_non_linearity = 0.0, double penalty_for_interactions = 0.0);
     APLRClassifier(const APLRClassifier &other);
     ~APLRClassifier();
     void fit(const MatrixXd &X, const std::vector<std::string> &y, const VectorXd &sample_weight = VectorXd(0),
@@ -72,14 +75,16 @@ APLRClassifier::APLRClassifier(size_t m, double v, uint_fast32_t random_state, s
                                size_t reserved_terms_times_num_x, size_t bins, size_t verbosity, size_t max_interaction_level, size_t max_interactions,
                                size_t min_observations_in_split, size_t ineligible_boosting_steps_added, size_t max_eligible_terms,
                                size_t boosting_steps_before_interactions_are_allowed, bool monotonic_constraints_ignore_interactions,
-                               size_t early_stopping_rounds, size_t num_first_steps_with_linear_effects_only)
+                               size_t early_stopping_rounds, size_t num_first_steps_with_linear_effects_only,
+                               double penalty_for_non_linearity, double penalty_for_interactions)
     : m{m}, v{v}, random_state{random_state}, n_jobs{n_jobs}, cv_folds{cv_folds},
       reserved_terms_times_num_x{reserved_terms_times_num_x}, bins{bins}, verbosity{verbosity}, max_interaction_level{max_interaction_level},
       max_interactions{max_interactions}, min_observations_in_split{min_observations_in_split},
       ineligible_boosting_steps_added{ineligible_boosting_steps_added}, max_eligible_terms{max_eligible_terms},
       boosting_steps_before_interactions_are_allowed{boosting_steps_before_interactions_are_allowed},
       monotonic_constraints_ignore_interactions{monotonic_constraints_ignore_interactions}, early_stopping_rounds{early_stopping_rounds},
-      num_first_steps_with_linear_effects_only{num_first_steps_with_linear_effects_only}
+      num_first_steps_with_linear_effects_only{num_first_steps_with_linear_effects_only}, penalty_for_non_linearity{penalty_for_non_linearity},
+      penalty_for_interactions{penalty_for_interactions}
 {
 }
 
@@ -94,7 +99,8 @@ APLRClassifier::APLRClassifier(const APLRClassifier &other)
       boosting_steps_before_interactions_are_allowed{other.boosting_steps_before_interactions_are_allowed},
       monotonic_constraints_ignore_interactions{other.monotonic_constraints_ignore_interactions},
       early_stopping_rounds{other.early_stopping_rounds},
-      num_first_steps_with_linear_effects_only{other.num_first_steps_with_linear_effects_only}
+      num_first_steps_with_linear_effects_only{other.num_first_steps_with_linear_effects_only},
+      penalty_for_non_linearity{other.penalty_for_non_linearity}, penalty_for_interactions{other.penalty_for_interactions}
 {
 }
 
@@ -121,6 +127,8 @@ void APLRClassifier::fit(const MatrixXd &X, const std::vector<std::string> &y, c
         logit_models[categories[0]].monotonic_constraints_ignore_interactions = monotonic_constraints_ignore_interactions;
         logit_models[categories[0]].early_stopping_rounds = early_stopping_rounds;
         logit_models[categories[0]].num_first_steps_with_linear_effects_only = num_first_steps_with_linear_effects_only;
+        logit_models[categories[0]].penalty_for_non_linearity = penalty_for_non_linearity;
+        logit_models[categories[0]].penalty_for_interactions = penalty_for_interactions;
         logit_models[categories[0]].fit(X, response_values[categories[0]], sample_weight, X_names, cv_observations, prioritized_predictors_indexes,
                                         monotonic_constraints, VectorXi(0), interaction_constraints);
 
@@ -138,6 +146,8 @@ void APLRClassifier::fit(const MatrixXd &X, const std::vector<std::string> &y, c
             logit_models[category].monotonic_constraints_ignore_interactions = monotonic_constraints_ignore_interactions;
             logit_models[category].early_stopping_rounds = early_stopping_rounds;
             logit_models[category].num_first_steps_with_linear_effects_only = num_first_steps_with_linear_effects_only;
+            logit_models[category].penalty_for_non_linearity = penalty_for_non_linearity;
+            logit_models[category].penalty_for_interactions = penalty_for_interactions;
             logit_models[category].fit(X, response_values[category], sample_weight, X_names, cv_observations, prioritized_predictors_indexes,
                                        monotonic_constraints, VectorXi(0), interaction_constraints);
         }

cpp/APLRRegressor.h

@@ -212,6 +212,8 @@ class APLRRegressor
     VectorXi term_interaction_levels;
     size_t early_stopping_rounds;
     size_t num_first_steps_with_linear_effects_only;
+    double penalty_for_non_linearity;
+    double penalty_for_interactions;
 
     APLRRegressor(size_t m = 3000, double v = 0.1, uint_fast32_t random_state = std::numeric_limits<uint_fast32_t>::lowest(), std::string loss_function = "mse",
                   std::string link_function = "identity", size_t n_jobs = 0, size_t cv_folds = 5,
@@ -225,7 +227,8 @@ class APLRRegressor
                   const std::function<VectorXd(VectorXd)> &calculate_custom_differentiate_predictions_wrt_linear_predictor_function = {},
                   size_t boosting_steps_before_interactions_are_allowed = 0, bool monotonic_constraints_ignore_interactions = false,
                   size_t group_mse_by_prediction_bins = 10, size_t group_mse_cycle_min_obs_in_bin = 30, size_t early_stopping_rounds = 500,
-                  size_t num_first_steps_with_linear_effects_only = 0);
+                  size_t num_first_steps_with_linear_effects_only = 0, double penalty_for_non_linearity = 0.0,
+                  double penalty_for_interactions = 0.0);
     APLRRegressor(const APLRRegressor &other);
     ~APLRRegressor();
     void fit(const MatrixXd &X, const VectorXd &y, const VectorXd &sample_weight = VectorXd(0), const std::vector<std::string> &X_names = {},
@@ -266,7 +269,7 @@ APLRRegressor::APLRRegressor(size_t m, double v, uint_fast32_t random_state, std
                              const std::function<VectorXd(VectorXd)> &calculate_custom_differentiate_predictions_wrt_linear_predictor_function,
                              size_t boosting_steps_before_interactions_are_allowed, bool monotonic_constraints_ignore_interactions,
                              size_t group_mse_by_prediction_bins, size_t group_mse_cycle_min_obs_in_bin, size_t early_stopping_rounds,
-                             size_t num_first_steps_with_linear_effects_only)
+                             size_t num_first_steps_with_linear_effects_only, double penalty_for_non_linearity, double penalty_for_interactions)
     : reserved_terms_times_num_x{reserved_terms_times_num_x}, intercept{NAN_DOUBLE}, m{m}, v{v},
       loss_function{loss_function}, link_function{link_function}, cv_folds{cv_folds}, n_jobs{n_jobs}, random_state{random_state},
       bins{bins}, verbosity{verbosity}, max_interaction_level{max_interaction_level},
@@ -281,7 +284,8 @@ APLRRegressor::APLRRegressor(size_t m, double v, uint_fast32_t random_state, std
       boosting_steps_before_interactions_are_allowed{boosting_steps_before_interactions_are_allowed},
       monotonic_constraints_ignore_interactions{monotonic_constraints_ignore_interactions}, group_mse_by_prediction_bins{group_mse_by_prediction_bins},
       group_mse_cycle_min_obs_in_bin{group_mse_cycle_min_obs_in_bin}, cv_error{NAN_DOUBLE}, early_stopping_rounds{early_stopping_rounds},
-      num_first_steps_with_linear_effects_only{num_first_steps_with_linear_effects_only}
+      num_first_steps_with_linear_effects_only{num_first_steps_with_linear_effects_only}, penalty_for_non_linearity{penalty_for_non_linearity},
+      penalty_for_interactions{penalty_for_interactions}
 {
 }
 
@@ -307,7 +311,8 @@ APLRRegressor::APLRRegressor(const APLRRegressor &other)
       group_mse_cycle_min_obs_in_bin{other.group_mse_cycle_min_obs_in_bin}, cv_error{other.cv_error},
       term_main_predictor_indexes{other.term_main_predictor_indexes}, term_interaction_levels{other.term_interaction_levels},
       early_stopping_rounds{other.early_stopping_rounds},
-      num_first_steps_with_linear_effects_only{other.num_first_steps_with_linear_effects_only}
+      num_first_steps_with_linear_effects_only{other.num_first_steps_with_linear_effects_only},
+      penalty_for_non_linearity{other.penalty_for_non_linearity}, penalty_for_interactions{other.penalty_for_interactions}
 {
 }
 
@@ -1145,7 +1150,7 @@ void APLRRegressor::estimate_split_point_for_each_term(std::vector<Term> &terms,
     for (size_t i = 0; i < terms_indexes.size(); ++i)
     {
         terms[terms_indexes[i]].estimate_split_point(X_train, neg_gradient_current, sample_weight_train, bins, v, min_observations_in_split,
-                                                     linear_effects_only_in_this_boosting_step);
+                                                     linear_effects_only_in_this_boosting_step, penalty_for_non_linearity, penalty_for_interactions);
     }
 }