pruning

Author: mathias-von-ottenbreit

API_REFERENCE_FOR_CLASSIFICATION.md

@@ -1,6 +1,6 @@
 # APLRClassifier
 
-## class aplr.APLRClassifier(m:int=9000, v:float=0.1, random_state:int=0, n_jobs:int=0, validation_ratio:float=0.2, bins:int=300, verbosity:int=0, max_interaction_level:int=1, max_interactions:int=100000, min_observations_in_split:int=20, ineligible_boosting_steps_added:int=10, max_eligible_terms:int=5)
+## class aplr.APLRClassifier(m:int=9000, v:float=0.1, random_state:int=0, n_jobs:int=0, validation_ratio:float=0.2, bins:int=300, verbosity:int=0, max_interaction_level:int=1, max_interactions:int=100000, min_observations_in_split:int=20, ineligible_boosting_steps_added:int=10, max_eligible_terms:int=5, boosting_steps_before_pruning_is_done:int = 500)
 
 ### Constructor parameters
 
@@ -40,8 +40,11 @@ Controls how many boosting steps a term that becomes ineligible has to remain in
 #### max_eligible_terms (default = 5)
 Limits 1) the number of terms already in the model that can be considered as interaction partners in a boosting step and 2) how many terms remain eligible in the next boosting step. The default value works well according to empirical results. This hyperparameter is intended for reducing computational costs.
 
+#### boosting_steps_before_pruning_is_done (default = 500)
+Specifies how many boosting steps to wait before pruning the model. With the default value, this means that in boosting steps 500, 1000, and so on, the model will be pruned. When pruning, terms are removed as long as this reduces the training error. This can be a computationally costly operation especially if the model gets many terms. To switch off pruning set ***boosting_steps_before_pruning_is_done*** to a value higher than ***m***.
 
-## Method: fit(X:npt.ArrayLike, y:List[str], sample_weight:npt.ArrayLike = np.empty(0), X_names:List[str]=[], validation_set_indexes:List[int]=[], prioritized_predictors_indexes:List[int]=[], monotonic_constraints:List[int]=[], interaction_constraints:List[int]=[])
+
+## Method: fit(X:npt.ArrayLike, y:List[str], sample_weight:npt.ArrayLike = np.empty(0), X_names:List[str]=[], validation_set_indexes:List[int]=[], prioritized_predictors_indexes:List[int]=[], monotonic_constraints:List[int]=[], interaction_constraints:List[List[int]]=[])
 
 ***This method fits the model to data.***
 
@@ -69,7 +72,7 @@ An optional list of integers specifying the indexes of predictors (columns) in *
 An optional list of integers specifying monotonic constraints on model terms. For example, if there are three predictors in ***X***, then monotonic_constraints = [1,0,-1] means that 1) the first predictor in ***X*** cannot be used in interaction terms as a secondary effect and all terms using the first predictor in ***X*** as a main effect must have positive regression coefficients, 2) there are no monotonic constraints on terms using the second predictor in ***X***, and 3) the third predictor in ***X*** cannot be used in interaction terms as a secondary effect and all terms using the third predictor in ***X*** as a main effect must have negative regression coefficients.
 
 #### interaction_constraints
-An optional list of integers specifying interaction constraints on model terms. For example, if there are three predictors in ***X***, then interaction_constraints = [1,0,2] means that 1) the first predictor in ***X*** cannot be used in interaction terms as a secondary effect, 2) there are no interaction constraints on terms using the second predictor in ***X***, and 3) the third predictor in ***X*** cannot be used in any interaction terms.
+An optional list containing lists of integers. Specifies interaction constraints on model terms. For example, interaction_constraints = [[0,1], [1,2,3]] means that 1) the first and second predictors may interact with each other, and that 2) the second, third and fourth predictors may interact with each other. There are no interaction constraints on predictors not mentioned in interaction_constraints.
 
 
 ## Method: predict_class_probabilities(X:npt.ArrayLike, cap_predictions_to_minmax_in_training:bool=False)

API_REFERENCE_FOR_REGRESSION.md

@@ -1,6 +1,6 @@
 # APLRRegressor
 
-## class aplr.APLRRegressor(m:int=1000, v:float=0.1, random_state:int=0, loss_function:str="mse", link_function:str="identity", n_jobs:int=0, validation_ratio:float=0.2, bins:int=300, max_interaction_level:int=1, max_interactions:int=100000, min_observations_in_split:int=20, ineligible_boosting_steps_added:int=10, max_eligible_terms:int=5, verbosity:int=0, dispersion_parameter:float=1.5, validation_tuning_metric:str="default", quantile:float=0.5, calculate_custom_validation_error_function:Optional[Callable[[npt.ArrayLike, npt.ArrayLike, npt.ArrayLike, npt.ArrayLike], float]]=None, calculate_custom_loss_function:Optional[Callable[[npt.ArrayLike, npt.ArrayLike, npt.ArrayLike, npt.ArrayLike], float]]=None, calculate_custom_negative_gradient_function:Optional[Callable[[npt.ArrayLike, npt.ArrayLike, npt.ArrayLike], npt.ArrayLike]]=None, calculate_custom_transform_linear_predictor_to_predictions_function:Optional[Callable[[npt.ArrayLike], npt.ArrayLike]]=None, calculate_custom_differentiate_predictions_wrt_linear_predictor_function:Optional[Callable[[npt.ArrayLike], npt.ArrayLike]]=None)
+## class aplr.APLRRegressor(m:int=1000, v:float=0.1, random_state:int=0, loss_function:str="mse", link_function:str="identity", n_jobs:int=0, validation_ratio:float=0.2, bins:int=300, max_interaction_level:int=1, max_interactions:int=100000, min_observations_in_split:int=20, ineligible_boosting_steps_added:int=10, max_eligible_terms:int=5, verbosity:int=0, dispersion_parameter:float=1.5, validation_tuning_metric:str="default", quantile:float=0.5, calculate_custom_validation_error_function:Optional[Callable[[npt.ArrayLike, npt.ArrayLike, npt.ArrayLike, npt.ArrayLike], float]]=None, calculate_custom_loss_function:Optional[Callable[[npt.ArrayLike, npt.ArrayLike, npt.ArrayLike, npt.ArrayLike], float]]=None, calculate_custom_negative_gradient_function:Optional[Callable[[npt.ArrayLike, npt.ArrayLike, npt.ArrayLike], npt.ArrayLike]]=None, calculate_custom_transform_linear_predictor_to_predictions_function:Optional[Callable[[npt.ArrayLike], npt.ArrayLike]]=None, calculate_custom_differentiate_predictions_wrt_linear_predictor_function:Optional[Callable[[npt.ArrayLike], npt.ArrayLike]]=None, boosting_steps_before_pruning_is_done: int = 500)
 
 ### Constructor parameters
 
@@ -102,7 +102,10 @@ def calculate_custom_differentiate_predictions_wrt_linear_predictor(linear_predi
     return differentiated_predictions
 ```
 
-## Method: fit(X:npt.ArrayLike, y:npt.ArrayLike, sample_weight:npt.ArrayLike = np.empty(0), X_names:List[str]=[], validation_set_indexes:List[int]=[], prioritized_predictors_indexes:List[int]=[], monotonic_constraints:List[int]=[], group:npt.ArrayLike = np.empty(0), interaction_constraints:List[int]=[])
+#### boosting_steps_before_pruning_is_done (default = 500)
+Specifies how many boosting steps to wait before pruning the model. With the default value, this means that in boosting steps 500, 1000, and so on, the model will be pruned. When pruning, terms are removed as long as this reduces the training error. This can be a computationally costly operation especially if the model gets many terms. To switch off pruning set ***boosting_steps_before_pruning_is_done*** to a value higher than ***m***.
+
+## Method: fit(X:npt.ArrayLike, y:npt.ArrayLike, sample_weight:npt.ArrayLike = np.empty(0), X_names:List[str]=[], validation_set_indexes:List[int]=[], prioritized_predictors_indexes:List[int]=[], monotonic_constraints:List[int]=[], group:npt.ArrayLike = np.empty(0), interaction_constraints:List[List[int]]=[])
 
 ***This method fits the model to data.***
 
@@ -133,7 +136,7 @@ An optional list of integers specifying monotonic constraints on model terms. Fo
 A numpy vector of integers that is used when ***loss_function*** is "group_mse". For example, ***group*** may represent year (could be useful in a time series model).
 
 #### interaction_constraints
-An optional list of integers specifying interaction constraints on model terms. For example, if there are three predictors in ***X***, then interaction_constraints = [1,0,2] means that 1) the first predictor in ***X*** cannot be used in interaction terms as a secondary effect, 2) there are no interaction constraints on terms using the second predictor in ***X***, and 3) the third predictor in ***X*** cannot be used in any interaction terms.
+An optional list containing lists of integers. Specifies interaction constraints on model terms. For example, interaction_constraints = [[0,1], [1,2,3]] means that 1) the first and second predictors may interact with each other, and that 2) the second, third and fourth predictors may interact with each other. There are no interaction constraints on predictors not mentioned in interaction_constraints.
 
 
 ## Method: predict(X:npt.ArrayLike, cap_predictions_to_minmax_in_training:bool=True)

aplr/aplr.py

@@ -43,6 +43,7 @@ def __init__(
         calculate_custom_differentiate_predictions_wrt_linear_predictor_function: Optional[
             Callable[[npt.ArrayLike], npt.ArrayLike]
         ] = None,
+        boosting_steps_before_pruning_is_done: int = 500,
     ):
         self.m = m
         self.v = v
@@ -74,6 +75,9 @@ def __init__(
         self.calculate_custom_differentiate_predictions_wrt_linear_predictor_function = (
             calculate_custom_differentiate_predictions_wrt_linear_predictor_function
         )
+        self.boosting_steps_before_pruning_is_done = (
+            boosting_steps_before_pruning_is_done
+        )
 
         # Creating aplr_cpp and setting parameters
         self.APLRRegressor = aplr_cpp.APLRRegressor()
@@ -115,6 +119,9 @@ def __set_params_cpp(self):
         self.APLRRegressor.calculate_custom_differentiate_predictions_wrt_linear_predictor_function = (
             self.calculate_custom_differentiate_predictions_wrt_linear_predictor_function
         )
+        self.APLRRegressor.boosting_steps_before_pruning_is_done = (
+            self.boosting_steps_before_pruning_is_done
+        )
 
     def fit(
         self,
@@ -126,7 +133,7 @@ def fit(
         prioritized_predictors_indexes: List[int] = [],
         monotonic_constraints: List[int] = [],
         group: npt.ArrayLike = np.empty(0),
-        interaction_constraints: List[int] = [],
+        interaction_constraints: List[List[int]] = [],
     ):
         self.__set_params_cpp()
         self.APLRRegressor.fit(
@@ -219,6 +226,7 @@ def get_params(self, deep=True):
             "calculate_custom_negative_gradient_function": self.calculate_custom_negative_gradient_function,
             "calculate_custom_transform_linear_predictor_to_predictions_function": self.calculate_custom_transform_linear_predictor_to_predictions_function,
             "calculate_custom_differentiate_predictions_wrt_linear_predictor_function": self.calculate_custom_differentiate_predictions_wrt_linear_predictor_function,
+            "boosting_steps_before_pruning_is_done": self.boosting_steps_before_pruning_is_done,
         }
 
     # For sklearn
@@ -244,6 +252,7 @@ def __init__(
         min_observations_in_split: int = 20,
         ineligible_boosting_steps_added: int = 10,
         max_eligible_terms: int = 5,
+        boosting_steps_before_pruning_is_done: int = 500,
     ):
         self.m = m
         self.v = v
@@ -257,6 +266,9 @@ def __init__(
         self.min_observations_in_split = min_observations_in_split
         self.ineligible_boosting_steps_added = ineligible_boosting_steps_added
         self.max_eligible_terms = max_eligible_terms
+        self.boosting_steps_before_pruning_is_done = (
+            boosting_steps_before_pruning_is_done
+        )
 
         # Creating aplr_cpp and setting parameters
         self.APLRClassifier = aplr_cpp.APLRClassifier()
@@ -278,6 +290,9 @@ def __set_params_cpp(self):
             self.ineligible_boosting_steps_added
         )
         self.APLRClassifier.max_eligible_terms = self.max_eligible_terms
+        self.APLRClassifier.boosting_steps_before_pruning_is_done = (
+            self.boosting_steps_before_pruning_is_done
+        )
 
     def fit(
         self,
@@ -288,7 +303,7 @@ def fit(
         validation_set_indexes: List[int] = [],
         prioritized_predictors_indexes: List[int] = [],
         monotonic_constraints: List[int] = [],
-        interaction_constraints: List[int] = [],
+        interaction_constraints: List[List[int]] = [],
     ):
         self.__set_params_cpp()
         self.APLRClassifier.fit(
@@ -350,6 +365,7 @@ def get_params(self, deep=True):
             "min_observations_in_split": self.min_observations_in_split,
             "ineligible_boosting_steps_added": self.ineligible_boosting_steps_added,
             "max_eligible_terms": self.max_eligible_terms,
+            "boosting_steps_before_pruning_is_done": self.boosting_steps_before_pruning_is_done,
         }
 
     # For sklearn