Revert "bugfix and updated docs"

This reverts commit f93b10f.

Author: mathias-von-ottenbreit

API_REFERENCE_FOR_CLASSIFICATION.md

@@ -1,14 +1,14 @@
 # APLRClassifier
 
-## class aplr.APLRClassifier(m:int=3000, v:float=0.1, random_state:int=0, n_jobs:int=0, cv_folds:int=5, bins:int=300, verbosity:int=0, max_interaction_level:int=1, max_interactions:int=100000, min_observations_in_split:int=20, ineligible_boosting_steps_added:int=10, max_eligible_terms:int=5, boosting_steps_before_interactions_are_allowed: int = 0, monotonic_constraints_ignore_interactions: bool = False, early_stopping_rounds: int = 500)
+## class aplr.APLRClassifier(m:int=3000, v:float=0.3, random_state:int=0, n_jobs:int=0, cv_folds:int=5, bins:int=300, verbosity:int=0, max_interaction_level:int=1, max_interactions:int=100000, min_observations_in_split:int=20, ineligible_boosting_steps_added:int=10, max_eligible_terms:int=5, boosting_steps_before_interactions_are_allowed: int = 0, monotonic_constraints_ignore_interactions: bool = False, early_stopping_rounds: int = 500)
 
 ### Constructor parameters
 
 #### m (default = 3000)
 The maximum number of boosting steps. If validation error does not flatten out at the end of the ***m***th boosting step, then try increasing it (or alternatively increase the learning rate).
 
-#### v (default = 0.1)
-The learning rate. Must be greater than zero and not more than one. The higher the faster the algorithm learns and the lower ***m*** is required. However, empirical evidence suggests that ***v <= 0.1*** gives better results. If the algorithm learns too fast (requires few boosting steps to converge) then try lowering the learning rate. Computational costs can be reduced by increasing the learning rate while simultaneously decreasing ***m***, potentially at the expense of predictiveness.
+#### v (default = 0.3)
+The learning rate. Must be greater than zero and not more than one. The higher the faster the algorithm learns and the lower ***m*** is required. If the algorithm learns too fast (requires few boosting steps to converge) then try lowering the learning rate. Computational costs can be reduced by increasing the learning rate while simultaneously decreasing ***m***, potentially at the expense of predictiveness.
 
 #### random_state (default = 0)
 Used to randomly split training observations into cv_folds if ***cv_observations*** is not specified when fitting.

aplr/aplr.py

@@ -300,7 +300,7 @@ class APLRClassifier:
     def __init__(
         self,
         m: int = 3000,
-        v: float = 0.1,
+        v: float = 0.3,
         random_state: int = 0,
         n_jobs: int = 0,
         cv_folds: int = 5,

cpp/APLRClassifier.h

@@ -46,7 +46,7 @@ class APLRClassifier
     bool monotonic_constraints_ignore_interactions;
     size_t early_stopping_rounds;
 
-    APLRClassifier(size_t m = 3000, double v = 0.1, uint_fast32_t random_state = std::numeric_limits<uint_fast32_t>::lowest(), size_t n_jobs = 0,
+    APLRClassifier(size_t m = 3000, double v = 0.3, uint_fast32_t random_state = std::numeric_limits<uint_fast32_t>::lowest(), size_t n_jobs = 0,
                    size_t cv_folds = 5, size_t reserved_terms_times_num_x = 100, size_t bins = 300, size_t verbosity = 0, size_t max_interaction_level = 1,
                    size_t max_interactions = 100000, size_t min_observations_in_split = 20, size_t ineligible_boosting_steps_added = 10, size_t max_eligible_terms = 5,
                    size_t boosting_steps_before_interactions_are_allowed = 0, bool monotonic_constraints_ignore_interactions = false,

cpp/APLRRegressor.h

@@ -435,14 +435,14 @@ void APLRRegressor::throw_error_if_dispersion_parameter_is_invalid()
     if (loss_function == "tweedie")
     {
         bool dispersion_parameter_equals_invalid_poits{is_approximately_equal(dispersion_parameter, 1.0) || is_approximately_equal(dispersion_parameter, 2.0)};
-        bool dispersion_parameter_is_in_invalid_range{is_less(dispersion_parameter, 1.0)};
+        bool dispersion_parameter_is_in_invalid_range{std::isless(dispersion_parameter, 1.0)};
         bool dispersion_parameter_is_invalid{dispersion_parameter_equals_invalid_poits || dispersion_parameter_is_in_invalid_range};
         if (dispersion_parameter_is_invalid)
             throw std::runtime_error("Invalid dispersion_parameter (variance power). It must not equal 1.0 or 2.0 and cannot be below 1.0.");
     }
     else if (loss_function == "negative_binomial" || loss_function == "cauchy" || loss_function == "weibull")
     {
-        bool dispersion_parameter_is_in_invalid{is_less_equal(dispersion_parameter, 0.0)};
+        bool dispersion_parameter_is_in_invalid{std::islessequal(dispersion_parameter, 0.0)};
         if (dispersion_parameter_is_in_invalid)
             throw std::runtime_error("Invalid dispersion_parameter. It must be greater than zero.");
     }
@@ -551,7 +551,7 @@ void APLRRegressor::throw_error_if_response_contains_invalid_values(const Vector
         std::string error_message{"Response values for the logit link function or binomial loss_function cannot be less than zero or greater than one."};
         throw_error_if_response_is_not_between_0_and_1(y, error_message);
     }
-    else if (loss_function == "gamma" || (loss_function == "tweedie" && is_greater(dispersion_parameter, 2.0)))
+    else if (loss_function == "gamma" || (loss_function == "tweedie" && std::isgreater(dispersion_parameter, 2)))
     {
         std::string error_message;
         if (loss_function == "tweedie")
@@ -560,7 +560,7 @@ void APLRRegressor::throw_error_if_response_contains_invalid_values(const Vector
             error_message = "Response values for the " + loss_function + " loss_function must be greater than zero.";
         throw_error_if_vector_contains_non_positive_values(y, error_message);
     }
-    else if (link_function == "log" || loss_function == "poisson" || loss_function == "negative_binomial" || loss_function == "weibull" || (loss_function == "tweedie" && is_less(dispersion_parameter, 2.0) && is_greater(dispersion_parameter, 1.0)))
+    else if (link_function == "log" || loss_function == "poisson" || loss_function == "negative_binomial" || loss_function == "weibull" || (loss_function == "tweedie" && std::isless(dispersion_parameter, 2) && std::isgreater(dispersion_parameter, 1)))
     {
         std::string error_message{"Response values for the log link function or poisson loss_function or negative binomial loss function or weibull loss function or tweedie loss_function when dispersion_parameter<2 cannot be less than zero."};
         throw_error_if_vector_contains_negative_values(y, error_message);
@@ -1098,7 +1098,7 @@ void APLRRegressor::update_intercept(size_t boosting_step)
     else
         intercept_update = v * (neg_gradient_current.array() * sample_weight_train.array()).sum() / sample_weight_train.array().sum();
     if (model_has_changed_in_this_boosting_step == false)
-        model_has_changed_in_this_boosting_step = !is_approximately_zero(intercept_update);
+        model_has_changed_in_this_boosting_step = !is_approximately_equal(intercept_update, 0.0);
     linear_predictor_update = VectorXd::Constant(neg_gradient_current.size(), intercept_update);
     linear_predictor_update_validation = VectorXd::Constant(y_validation.size(), intercept_update);
     update_linear_predictor_and_predictions();
@@ -1159,7 +1159,7 @@ size_t APLRRegressor::find_best_term_index(std::vector<Term> &terms, std::vector
         bool term_is_eligible{terms[term_index].ineligible_boosting_steps == 0};
         if (term_is_eligible)
         {
-            if (is_less(terms[term_index].split_point_search_errors_sum, lowest_errors_sum))
+            if (std::isless(terms[term_index].split_point_search_errors_sum, lowest_errors_sum))
             {
                 best_term_index = term_index;
                 lowest_errors_sum = terms[term_index].split_point_search_errors_sum;
@@ -1365,9 +1365,9 @@ void APLRRegressor::add_promising_interactions_and_select_the_best_one()
         if (allowed_to_add_one_interaction)
         {
             if (best_term_before_interactions_was_not_selected)
-                error_is_less_than_for_best_term_before_interactions = is_less(interactions_to_consider[sorted_indexes_of_errors_for_interactions_to_consider[j]].split_point_search_errors_sum, neg_gradient_nullmodel_errors_sum);
+                error_is_less_than_for_best_term_before_interactions = std::isless(interactions_to_consider[sorted_indexes_of_errors_for_interactions_to_consider[j]].split_point_search_errors_sum, neg_gradient_nullmodel_errors_sum);
             else
-                error_is_less_than_for_best_term_before_interactions = is_less(interactions_to_consider[sorted_indexes_of_errors_for_interactions_to_consider[j]].split_point_search_errors_sum, terms_eligible_current[best_term_before_interactions].split_point_search_errors_sum);
+                error_is_less_than_for_best_term_before_interactions = std::isless(interactions_to_consider[sorted_indexes_of_errors_for_interactions_to_consider[j]].split_point_search_errors_sum, terms_eligible_current[best_term_before_interactions].split_point_search_errors_sum);
 
             if (error_is_less_than_for_best_term_before_interactions)
             {
@@ -1390,7 +1390,7 @@ void APLRRegressor::select_the_best_term_and_update_errors(size_t boosting_step)
         return;
 
     if (model_has_changed_in_this_boosting_step == false)
-        model_has_changed_in_this_boosting_step = !is_approximately_zero(terms_eligible_current[best_term_index].coefficient);
+        model_has_changed_in_this_boosting_step = !is_approximately_equal(terms_eligible_current[best_term_index].coefficient, 0.0);
     linear_predictor_update = terms_eligible_current[best_term_index].calculate_contribution_to_linear_predictor(X_train);
     linear_predictor_update_validation = terms_eligible_current[best_term_index].calculate_contribution_to_linear_predictor(X_validation);
     update_linear_predictor_and_predictions();
@@ -1578,7 +1578,7 @@ void APLRRegressor::print_summary_after_boosting_step(size_t boosting_step, Eige
 
 void APLRRegressor::abort_boosting_when_no_validation_error_improvement_in_the_last_early_stopping_rounds(size_t boosting_step)
 {
-    bool validation_error_is_better{is_less(validation_error_steps.col(0)[boosting_step], best_validation_error_so_far)};
+    bool validation_error_is_better{std::isless(validation_error_steps.col(0)[boosting_step], best_validation_error_so_far)};
     if (validation_error_is_better)
     {
         best_validation_error_so_far = validation_error_steps.col(0)[boosting_step];
@@ -1744,7 +1744,7 @@ std::string APLRRegressor::compute_raw_base_term_name(const Term &term, const st
     {
         double temp_split_point{term.split_point};
         std::string sign{"-"};
-        if (is_less(temp_split_point, 0.0))
+        if (std::isless(temp_split_point, 0))
         {
             temp_split_point = -temp_split_point;
             sign = "+";
@@ -1902,11 +1902,11 @@ void APLRRegressor::check_term_integrity()
                 bool given_term_has_incorrect_split_point;
                 if (term.direction_right)
                 {
-                    given_term_has_incorrect_split_point = is_less_equal(given_term.split_point, term.split_point);
+                    given_term_has_incorrect_split_point = std::islessequal(given_term.split_point, term.split_point);
                 }
                 else
                 {
-                    given_term_has_incorrect_split_point = is_greater_equal(given_term.split_point, term.split_point);
+                    given_term_has_incorrect_split_point = std::isgreaterequal(given_term.split_point, term.split_point);
                 }
                 if (given_term_has_no_split_point)
                     throw std::runtime_error("Bug: Interaction in term " + term.name + " has no split point.");
@@ -1992,7 +1992,7 @@ void APLRRegressor::sort_terms()
               { return a.estimated_term_importance > b.estimated_term_importance ||
                        (is_approximately_equal(a.estimated_term_importance, b.estimated_term_importance) && (a.base_term < b.base_term)) ||
                        (is_approximately_equal(a.estimated_term_importance, b.estimated_term_importance) && (a.base_term == b.base_term) &&
-                        is_less(a.coefficient, b.coefficient)); });
+                        std::isless(a.coefficient, b.coefficient)); });
 
     for (size_t i = 0; i < terms.size(); ++i)
     {
@@ -2104,9 +2104,9 @@ void APLRRegressor::cap_predictions_to_minmax_in_training(VectorXd &predictions)
 {
     for (Eigen::Index i = 0; i < predictions.rows(); ++i)
     {
-        if (is_greater(predictions[i], max_training_prediction_or_response))
+        if (std::isgreater(predictions[i], max_training_prediction_or_response))
             predictions[i] = max_training_prediction_or_response;
-        else if (is_less(predictions[i], min_training_prediction_or_response))
+        else if (std::isless(predictions[i], min_training_prediction_or_response))
             predictions[i] = min_training_prediction_or_response;
     }
 }
@@ -2265,13 +2265,13 @@ std::map<double, double> APLRRegressor::get_coefficient_shape_function(size_t pr
             {
                 for (auto &key : coefficient_shape_function)
                 {
-                    bool key_split_point_is_higher{is_greater(key.first, terms[relevant_term_indexes[i]].split_point)};
+                    bool key_split_point_is_higher{std::isgreater(key.first, terms[relevant_term_indexes[i]].split_point)};
                     bool key_split_point_is_not_too_high{true};
                     for (auto &given_term : terms[relevant_term_indexes[i]].given_terms)
                     {
                         if (given_term.direction_right != terms[relevant_term_indexes[i]].direction_right)
                         {
-                            if (is_greater(key.first, given_term.split_point))
+                            if (std::isgreater(key.first, given_term.split_point))
                             {
                                 key_split_point_is_not_too_high = false;
                                 break;
@@ -2288,13 +2288,13 @@ std::map<double, double> APLRRegressor::get_coefficient_shape_function(size_t pr
             {
                 for (auto &key : coefficient_shape_function)
                 {
-                    bool key_split_point_is_lower{is_less(key.first, terms[relevant_term_indexes[i]].split_point)};
+                    bool key_split_point_is_lower{std::isless(key.first, terms[relevant_term_indexes[i]].split_point)};
                     bool key_split_point_is_not_too_low{true};
                     for (auto &given_term : terms[relevant_term_indexes[i]].given_terms)
                     {
                         if (given_term.direction_right != terms[relevant_term_indexes[i]].direction_right)
                         {
-                            if (is_less(key.first, given_term.split_point))
+                            if (std::isless(key.first, given_term.split_point))
                             {
                                 key_split_point_is_not_too_low = false;
                                 break;

cpp/CMakeLists.txt

@@ -4,7 +4,6 @@ project(APLR VERSION 0.1.0)
 include(CTest)
 enable_testing()
 set(CMAKE_CXX_FLAGS -pthread)
-set(CMAKE_CXX_FLAGS -fopenmp)
 
 add_executable(APLR main.cpp)
 

cpp/functions.h

@@ -13,44 +13,28 @@
 
 using namespace Eigen;
 
-bool is_approximately_equal(double a, double b, double tolerance = std::numeric_limits<double>::epsilon())
+template <typename TReal>
+static bool is_approximately_equal(TReal a, TReal b, TReal tolerance = std::numeric_limits<TReal>::epsilon())
 {
-    if (std::isinf(a) && std::isinf(b))
-    {
-        if (std::signbit(a) == std::signbit(b))
-            return true;
-        else
-            return false;
-    }
-
-    double relative_tolerance{(fabs(a) < fabs(b) ? fabs(b) : fabs(a)) * tolerance};
-    double absolute_tolerance{std::fmax(relative_tolerance, tolerance)};
-    return fabs(a - b) <= absolute_tolerance;
-}
-
-bool is_approximately_zero(double a, double tolerance = std::numeric_limits<double>::epsilon())
-{
-    return is_approximately_equal(a, 0.0, tolerance);
-}
+    if (std::isinf(a) && std::isinf(b) && std::signbit(a) == std::signbit(b))
+        return true;
 
-bool is_greater(double a, double b, double tolerance = std::numeric_limits<double>::epsilon())
-{
-    return std::isgreater(a, b) && !is_approximately_equal(a, b, tolerance);
-}
+    TReal diff = std::fabs(a - b);
+    if (diff <= tolerance)
+        return true;
 
-bool is_greater_equal(double a, double b, double tolerance = std::numeric_limits<double>::epsilon())
-{
-    return std::isgreater(a, b) || is_approximately_equal(a, b, tolerance);
-}
+    if (diff < std::fmax(std::fabs(a), std::fabs(b)) * tolerance)
+        return true;
 
-bool is_less(double a, double b, double tolerance = std::numeric_limits<double>::epsilon())
-{
-    return std::isless(a, b) && !is_approximately_equal(a, b, tolerance);
+    return false;
 }
 
-bool is_less_equal(double a, double b, double tolerance = std::numeric_limits<double>::epsilon())
+template <typename TReal>
+static bool is_approximately_zero(TReal a, TReal tolerance = std::numeric_limits<TReal>::epsilon())
 {
-    return std::isless(a, b) || is_approximately_equal(a, b, tolerance);
+    if (std::fabs(a) <= tolerance)
+        return true;
+    return false;
 }
 
 std::set<std::string> get_unique_strings(const std::vector<std::string> &string_vector)
@@ -288,14 +272,14 @@ VectorXd calculate_exp_of_linear_predictor_adjusted_for_numerical_problems(const
     double max_exp_of_linear_predictor{std::exp(max_exponent)};
     for (Eigen::Index i = 0; i < linear_predictor.rows(); ++i)
     {
-        bool linear_predictor_is_too_small{is_less(linear_predictor[i], min_exponent)};
+        bool linear_predictor_is_too_small{std::isless(linear_predictor[i], min_exponent)};
         if (linear_predictor_is_too_small)
         {
             exp_of_linear_predictor[i] = min_exp_of_linear_predictor;
             continue;
         }
 
-        bool linear_predictor_is_too_large{is_greater(linear_predictor[i], max_exponent)};
+        bool linear_predictor_is_too_large{std::isgreater(linear_predictor[i], max_exponent)};
         if (linear_predictor_is_too_large)
         {
             exp_of_linear_predictor[i] = max_exp_of_linear_predictor;

cpp/main.cpp

@@ -46,10 +46,10 @@ int main()
     // Saving results
     save_as_csv_file("output.csv", predictions);
     std::cout << "min validation_error " << model.validation_error_steps.minCoeff() << "\n\n";
-    std::cout << is_approximately_equal(model.validation_error_steps.minCoeff(), 6.82063, 0.00001) << "\n";
+    std::cout << is_approximately_equal(model.validation_error_steps.minCoeff(), 4.97628, 0.00001) << "\n";
 
     std::cout << "mean prediction " << predictions.mean() << "\n\n";
-    std::cout << is_approximately_equal(predictions.mean(), 23.6126, 0.0001) << "\n";
+    std::cout << is_approximately_equal(predictions.mean(), 23.6053, 0.0001) << "\n";
 
     std::cout << "best_m: " << model.m << "\n";
 

cpp/pythonbinding.cpp

@@ -233,7 +233,7 @@ PYBIND11_MODULE(aplr_cpp, m)
 
     py::class_<APLRClassifier>(m, "APLRClassifier", py::module_local())
         .def(py::init<int &, double &, int &, int &, int &, int &, int &, int &, int &, int &, int &, int &, int &, int &, bool &, int &>(),
-             py::arg("m") = 3000, py::arg("v") = 0.1, py::arg("random_state") = 0, py::arg("n_jobs") = 0, py::arg("cv_folds") = 5,
+             py::arg("m") = 3000, py::arg("v") = 0.3, py::arg("random_state") = 0, py::arg("n_jobs") = 0, py::arg("cv_folds") = 5,
              py::arg("reserved_terms_times_num_x") = 100, py::arg("bins") = 300, py::arg("verbosity") = 0,
              py::arg("max_interaction_level") = 1, py::arg("max_interactions") = 100000, py::arg("min_observations_in_split") = 20,
              py::arg("ineligible_boosting_steps_added") = 10, py::arg("max_eligible_terms") = 5,