boosting_steps_before_interactions_are_allowed

Author: mathias-von-ottenbreit

API_REFERENCE_FOR_CLASSIFICATION.md

@@ -1,6 +1,6 @@
 # APLRClassifier
 
-## class aplr.APLRClassifier(m:int=9000, v:float=0.1, random_state:int=0, n_jobs:int=0, validation_ratio:float=0.2, bins:int=300, verbosity:int=0, max_interaction_level:int=1, max_interactions:int=100000, min_observations_in_split:int=20, ineligible_boosting_steps_added:int=10, max_eligible_terms:int=5, boosting_steps_before_pruning_is_done:int = 0)
+## class aplr.APLRClassifier(m:int=9000, v:float=0.1, random_state:int=0, n_jobs:int=0, validation_ratio:float=0.2, bins:int=300, verbosity:int=0, max_interaction_level:int=1, max_interactions:int=100000, min_observations_in_split:int=20, ineligible_boosting_steps_added:int=10, max_eligible_terms:int=5, boosting_steps_before_pruning_is_done:int = 0, boosting_steps_before_interactions_are_allowed: int = 0)
 
 ### Constructor parameters
 
@@ -43,6 +43,9 @@ Limits 1) the number of terms already in the model that can be considered as int
 #### boosting_steps_before_pruning_is_done (default = 0)
 Specifies how many boosting steps to wait before pruning the model. If 0 (default) then pruning is not done. If for example 500 then the model will be pruned in boosting steps 500, 1000, and so on. When pruning, terms are removed as long as this reduces the training error. This can be a computationally costly operation especially if the model gets many terms. Pruning may improve predictiveness.
 
+#### boosting_steps_before_interactions_are_allowed (default = 0)
+Specifies how many boosting steps to wait before searching for interactions. If for example 800, then the algorithm will be forced to only fit main effects in the first 800 boosting steps, after which it is allowed to search for interactions (given that other hyperparameters that control interactions also allow this). The motivation for fitting main effects first may be 1) to get a cleaner looking model that puts more emphasis on main effects and 2) to speed up the algorithm since looking for interactions is computationally more demanding.
+
 
 ## Method: fit(X:npt.ArrayLike, y:List[str], sample_weight:npt.ArrayLike = np.empty(0), X_names:List[str]=[], validation_set_indexes:List[int]=[], prioritized_predictors_indexes:List[int]=[], monotonic_constraints:List[int]=[], interaction_constraints:List[List[int]]=[])
 

API_REFERENCE_FOR_REGRESSION.md

@@ -1,6 +1,6 @@
 # APLRRegressor
 
-## class aplr.APLRRegressor(m:int=1000, v:float=0.1, random_state:int=0, loss_function:str="mse", link_function:str="identity", n_jobs:int=0, validation_ratio:float=0.2, bins:int=300, max_interaction_level:int=1, max_interactions:int=100000, min_observations_in_split:int=20, ineligible_boosting_steps_added:int=10, max_eligible_terms:int=5, verbosity:int=0, dispersion_parameter:float=1.5, validation_tuning_metric:str="default", quantile:float=0.5, calculate_custom_validation_error_function:Optional[Callable[[npt.ArrayLike, npt.ArrayLike, npt.ArrayLike, npt.ArrayLike, npt.ArrayLike], float]]=None, calculate_custom_loss_function:Optional[Callable[[npt.ArrayLike, npt.ArrayLike, npt.ArrayLike, npt.ArrayLike, npt.ArrayLike], float]]=None, calculate_custom_negative_gradient_function:Optional[Callable[[npt.ArrayLike, npt.ArrayLike, npt.ArrayLike, npt.ArrayLike], npt.ArrayLike]]=None, calculate_custom_transform_linear_predictor_to_predictions_function:Optional[Callable[[npt.ArrayLike], npt.ArrayLike]]=None, calculate_custom_differentiate_predictions_wrt_linear_predictor_function:Optional[Callable[[npt.ArrayLike], npt.ArrayLike]]=None, boosting_steps_before_pruning_is_done: int = 0)
+## class aplr.APLRRegressor(m:int=1000, v:float=0.1, random_state:int=0, loss_function:str="mse", link_function:str="identity", n_jobs:int=0, validation_ratio:float=0.2, bins:int=300, max_interaction_level:int=1, max_interactions:int=100000, min_observations_in_split:int=20, ineligible_boosting_steps_added:int=10, max_eligible_terms:int=5, verbosity:int=0, dispersion_parameter:float=1.5, validation_tuning_metric:str="default", quantile:float=0.5, calculate_custom_validation_error_function:Optional[Callable[[npt.ArrayLike, npt.ArrayLike, npt.ArrayLike, npt.ArrayLike, npt.ArrayLike], float]]=None, calculate_custom_loss_function:Optional[Callable[[npt.ArrayLike, npt.ArrayLike, npt.ArrayLike, npt.ArrayLike, npt.ArrayLike], float]]=None, calculate_custom_negative_gradient_function:Optional[Callable[[npt.ArrayLike, npt.ArrayLike, npt.ArrayLike, npt.ArrayLike], npt.ArrayLike]]=None, calculate_custom_transform_linear_predictor_to_predictions_function:Optional[Callable[[npt.ArrayLike], npt.ArrayLike]]=None, calculate_custom_differentiate_predictions_wrt_linear_predictor_function:Optional[Callable[[npt.ArrayLike], npt.ArrayLike]]=None, boosting_steps_before_pruning_is_done: int = 0, boosting_steps_before_interactions_are_allowed: int = 0)
 
 ### Constructor parameters
 
@@ -105,6 +105,9 @@ def calculate_custom_differentiate_predictions_wrt_linear_predictor(linear_predi
 #### boosting_steps_before_pruning_is_done (default = 0)
 Specifies how many boosting steps to wait before pruning the model. If 0 (default) then pruning is not done. If for example 500 then the model will be pruned in boosting steps 500, 1000, and so on. When pruning, terms are removed as long as this reduces the training error. This can be a computationally costly operation especially if the model gets many terms. Pruning may improve predictiveness.
 
+#### boosting_steps_before_interactions_are_allowed (default = 0)
+Specifies how many boosting steps to wait before searching for interactions. If for example 800, then the algorithm will be forced to only fit main effects in the first 800 boosting steps, after which it is allowed to search for interactions (given that other hyperparameters that control interactions also allow this). The motivation for fitting main effects first may be 1) to get a cleaner looking model that puts more emphasis on main effects and 2) to speed up the algorithm since looking for interactions is computationally more demanding.
+
 ## Method: fit(X:npt.ArrayLike, y:npt.ArrayLike, sample_weight:npt.ArrayLike = np.empty(0), X_names:List[str]=[], validation_set_indexes:List[int]=[], prioritized_predictors_indexes:List[int]=[], monotonic_constraints:List[int]=[], group:npt.ArrayLike = np.empty(0), interaction_constraints:List[List[int]]=[], other_data: npt.ArrayLike = np.empty([0, 0]))
 
 ***This method fits the model to data.***

aplr/aplr.py

@@ -61,6 +61,7 @@ def __init__(
             Callable[[npt.ArrayLike], npt.ArrayLike]
         ] = None,
         boosting_steps_before_pruning_is_done: int = 0,
+        boosting_steps_before_interactions_are_allowed: int = 0,
     ):
         self.m = m
         self.v = v
@@ -95,6 +96,9 @@ def __init__(
         self.boosting_steps_before_pruning_is_done = (
             boosting_steps_before_pruning_is_done
         )
+        self.boosting_steps_before_interactions_are_allowed = (
+            boosting_steps_before_interactions_are_allowed
+        )
 
         # Creating aplr_cpp and setting parameters
         self.APLRRegressor = aplr_cpp.APLRRegressor()
@@ -139,6 +143,9 @@ def __set_params_cpp(self):
         self.APLRRegressor.boosting_steps_before_pruning_is_done = (
             self.boosting_steps_before_pruning_is_done
         )
+        self.APLRRegressor.boosting_steps_before_interactions_are_allowed = (
+            self.boosting_steps_before_interactions_are_allowed
+        )
 
     def fit(
         self,
@@ -246,6 +253,7 @@ def get_params(self, deep=True):
             "calculate_custom_transform_linear_predictor_to_predictions_function": self.calculate_custom_transform_linear_predictor_to_predictions_function,
             "calculate_custom_differentiate_predictions_wrt_linear_predictor_function": self.calculate_custom_differentiate_predictions_wrt_linear_predictor_function,
             "boosting_steps_before_pruning_is_done": self.boosting_steps_before_pruning_is_done,
+            "boosting_steps_before_interactions_are_allowed": self.boosting_steps_before_interactions_are_allowed,
         }
 
     # For sklearn
@@ -272,6 +280,7 @@ def __init__(
         ineligible_boosting_steps_added: int = 10,
         max_eligible_terms: int = 5,
         boosting_steps_before_pruning_is_done: int = 0,
+        boosting_steps_before_interactions_are_allowed: int = 0,
     ):
         self.m = m
         self.v = v
@@ -288,6 +297,9 @@ def __init__(
         self.boosting_steps_before_pruning_is_done = (
             boosting_steps_before_pruning_is_done
         )
+        self.boosting_steps_before_interactions_are_allowed = (
+            boosting_steps_before_interactions_are_allowed
+        )
 
         # Creating aplr_cpp and setting parameters
         self.APLRClassifier = aplr_cpp.APLRClassifier()
@@ -312,6 +324,9 @@ def __set_params_cpp(self):
         self.APLRClassifier.boosting_steps_before_pruning_is_done = (
             self.boosting_steps_before_pruning_is_done
         )
+        self.APLRClassifier.boosting_steps_before_interactions_are_allowed = (
+            self.boosting_steps_before_interactions_are_allowed
+        )
 
     def fit(
         self,
@@ -385,6 +400,7 @@ def get_params(self, deep=True):
             "ineligible_boosting_steps_added": self.ineligible_boosting_steps_added,
             "max_eligible_terms": self.max_eligible_terms,
             "boosting_steps_before_pruning_is_done": self.boosting_steps_before_pruning_is_done,
+            "boosting_steps_before_interactions_are_allowed": self.boosting_steps_before_interactions_are_allowed,
         }
 
     # For sklearn

cpp/APLRClassifier.h

@@ -43,11 +43,12 @@ class APLRClassifier
     std::vector<std::string> categories;
     std::map<std::string, APLRRegressor> logit_models; // Key is category and value is logit model
     size_t boosting_steps_before_pruning_is_done;
+    size_t boosting_steps_before_interactions_are_allowed;
 
     APLRClassifier(size_t m = 9000, double v = 0.1, uint_fast32_t random_state = std::numeric_limits<uint_fast32_t>::lowest(), size_t n_jobs = 0,
                    double validation_ratio = 0.2, size_t reserved_terms_times_num_x = 100, size_t bins = 300, size_t verbosity = 0, size_t max_interaction_level = 1,
                    size_t max_interactions = 100000, size_t min_observations_in_split = 20, size_t ineligible_boosting_steps_added = 10, size_t max_eligible_terms = 5,
-                   size_t boosting_steps_before_pruning_is_done = 0);
+                   size_t boosting_steps_before_pruning_is_done = 0, size_t boosting_steps_before_interactions_are_allowed = 0);
     APLRClassifier(const APLRClassifier &other);
     ~APLRClassifier();
     void fit(const MatrixXd &X, const std::vector<std::string> &y, const VectorXd &sample_weight = VectorXd(0),
@@ -68,21 +69,25 @@ class APLRClassifier
 APLRClassifier::APLRClassifier(size_t m, double v, uint_fast32_t random_state, size_t n_jobs, double validation_ratio,
                                size_t reserved_terms_times_num_x, size_t bins, size_t verbosity, size_t max_interaction_level, size_t max_interactions,
                                size_t min_observations_in_split, size_t ineligible_boosting_steps_added, size_t max_eligible_terms,
-                               size_t boosting_steps_before_pruning_is_done) : m{m}, v{v}, random_state{random_state}, n_jobs{n_jobs}, validation_ratio{validation_ratio},
-                                                                               reserved_terms_times_num_x{reserved_terms_times_num_x}, bins{bins}, verbosity{verbosity}, max_interaction_level{max_interaction_level},
-                                                                               max_interactions{max_interactions}, min_observations_in_split{min_observations_in_split},
-                                                                               ineligible_boosting_steps_added{ineligible_boosting_steps_added}, max_eligible_terms{max_eligible_terms},
-                                                                               boosting_steps_before_pruning_is_done{boosting_steps_before_pruning_is_done}
+                               size_t boosting_steps_before_pruning_is_done, size_t boosting_steps_before_interactions_are_allowed)
+    : m{m}, v{v}, random_state{random_state}, n_jobs{n_jobs}, validation_ratio{validation_ratio},
+      reserved_terms_times_num_x{reserved_terms_times_num_x}, bins{bins}, verbosity{verbosity}, max_interaction_level{max_interaction_level},
+      max_interactions{max_interactions}, min_observations_in_split{min_observations_in_split},
+      ineligible_boosting_steps_added{ineligible_boosting_steps_added}, max_eligible_terms{max_eligible_terms},
+      boosting_steps_before_pruning_is_done{boosting_steps_before_pruning_is_done},
+      boosting_steps_before_interactions_are_allowed{boosting_steps_before_interactions_are_allowed}
 {
 }
 
-APLRClassifier::APLRClassifier(const APLRClassifier &other) : m{other.m}, v{other.v}, random_state{other.random_state}, n_jobs{other.n_jobs}, validation_ratio{other.validation_ratio},
-                                                              reserved_terms_times_num_x{other.reserved_terms_times_num_x}, bins{other.bins}, verbosity{other.verbosity},
-                                                              max_interaction_level{other.max_interaction_level}, max_interactions{other.max_interactions},
-                                                              min_observations_in_split{other.min_observations_in_split}, ineligible_boosting_steps_added{other.ineligible_boosting_steps_added},
-                                                              max_eligible_terms{other.max_eligible_terms}, logit_models{other.logit_models}, categories{other.categories},
-                                                              validation_indexes{other.validation_indexes}, validation_error_steps{other.validation_error_steps}, validation_error{other.validation_error},
-                                                              feature_importance{other.feature_importance}, boosting_steps_before_pruning_is_done{other.boosting_steps_before_pruning_is_done}
+APLRClassifier::APLRClassifier(const APLRClassifier &other)
+    : m{other.m}, v{other.v}, random_state{other.random_state}, n_jobs{other.n_jobs}, validation_ratio{other.validation_ratio},
+      reserved_terms_times_num_x{other.reserved_terms_times_num_x}, bins{other.bins}, verbosity{other.verbosity},
+      max_interaction_level{other.max_interaction_level}, max_interactions{other.max_interactions},
+      min_observations_in_split{other.min_observations_in_split}, ineligible_boosting_steps_added{other.ineligible_boosting_steps_added},
+      max_eligible_terms{other.max_eligible_terms}, logit_models{other.logit_models}, categories{other.categories},
+      validation_indexes{other.validation_indexes}, validation_error_steps{other.validation_error_steps}, validation_error{other.validation_error},
+      feature_importance{other.feature_importance}, boosting_steps_before_pruning_is_done{other.boosting_steps_before_pruning_is_done},
+      boosting_steps_before_interactions_are_allowed{other.boosting_steps_before_interactions_are_allowed}
 {
 }
 
@@ -106,6 +111,7 @@ void APLRClassifier::fit(const MatrixXd &X, const std::vector<std::string> &y, c
                                                     bins, verbosity, max_interaction_level, max_interactions, min_observations_in_split, ineligible_boosting_steps_added,
                                                     max_eligible_terms, 1.5, "default", 0.5);
         logit_models[categories[0]].boosting_steps_before_pruning_is_done = boosting_steps_before_pruning_is_done;
+        logit_models[categories[0]].boosting_steps_before_interactions_are_allowed = boosting_steps_before_interactions_are_allowed;
         logit_models[categories[0]].fit(X, response_values[categories[0]], sample_weight, X_names, validation_indexes, prioritized_predictors_indexes,
                                         monotonic_constraints, VectorXi(0), interaction_constraints);
 
@@ -120,6 +126,7 @@ void APLRClassifier::fit(const MatrixXd &X, const std::vector<std::string> &y, c
                                                    bins, verbosity, max_interaction_level, max_interactions, min_observations_in_split, ineligible_boosting_steps_added,
                                                    max_eligible_terms, 1.5, "default", 0.5);
             logit_models[category].boosting_steps_before_pruning_is_done = boosting_steps_before_pruning_is_done;
+            logit_models[category].boosting_steps_before_interactions_are_allowed = boosting_steps_before_interactions_are_allowed;
             logit_models[category].fit(X, response_values[category], sample_weight, X_names, validation_indexes, prioritized_predictors_indexes,
                                        monotonic_constraints, VectorXi(0), interaction_constraints);
         }