autogluon and scikit-optimize

Author: Sergey Feldman

02_lightgbm.py

@@ -6,21 +6,23 @@
 import pickle
 import numpy as np
 import lightgbm as lgb
-from sklearn.model_selection import RandomizedSearchCV
+from skopt import BayesSearchCV
+from skopt.space import Real, Categorical, Integer
 from sklearn.model_selection import cross_val_score, StratifiedKFold
 from utils import load_data
 
-N_JOBS = 4 * 4 * 9
+N_JOBS = 4 * 4
 N_ITER = 25  # budget for hyperparam search
 
 
 def evaluate_pipeline_helper(X, y, pipeline, param_grid, random_state=0):
     inner_cv = StratifiedKFold(n_splits=4, shuffle=True, random_state=random_state)
     outer_cv = StratifiedKFold(n_splits=4, shuffle=True, random_state=random_state)
-    clf = RandomizedSearchCV(
+    clf = BayesSearchCV(
         estimator=pipeline,
-        param_distributions=param_grid,
+        search_spaces=param_grid,
         n_iter=N_ITER,
+        n_points=3,
         cv=inner_cv,
         scoring="roc_auc_ovr_weighted",
         n_jobs=N_JOBS,
@@ -35,7 +37,7 @@ def define_and_evaluate_lightgbm_pipeline(X, y, random_state=0):
     if len(set(y)) == 2:
         pipeline = lgb.LGBMClassifier(
             objective="binary",
-            n_estimators=500,
+            n_estimators=1000,
             metric="auc",
             verbose=-1,
             tree_learner="feature",
@@ -45,23 +47,23 @@ def define_and_evaluate_lightgbm_pipeline(X, y, random_state=0):
     else:
         pipeline = lgb.LGBMClassifier(
             objective="multiclass",
-            n_estimators=500,
+            n_estimators=1000,
             metric="auc_mu",
             verbose=-1,
             tree_learner="feature",
             random_state=random_state,
             silent=True,
         )
     param_grid = {
-        "learning_rate": [1e-7, 1e-6, 1e-5, 1e-4, 1e-3, 1e-2, 1e-1, 1e0],
-        "num_leaves": [2, 4, 8, 16, 32, 64],
-        "colsample_bytree": [0.5, 0.6, 0.7, 0.8, 0.9, 1.0],
-        "subsample": [0.5, 0.6, 0.7, 0.8, 0.9, 1.0],
-        "min_child_samples": [2, 4, 8, 16, 32, 64, 128, 256],
-        "min_child_weight": [1e-7, 1e-6, 1e-5, 1e-4, 1e-3, 1e-2, 1e-1, 1e0],
-        "reg_alpha": [1e-7, 1e-6, 1e-5, 1e-4, 1e-3, 1e-2, 1e-1, 1e0],
-        "reg_lambda": [1e-7, 1e-6, 1e-5, 1e-4, 1e-3, 1e-2, 1e-1, 1e0],
-        "max_depth": [1, 2, 4, 8, 16, 32, -1],
+        "learning_rate": Real(1e-7, 1e+0, prior='log-uniform'), #[1e-7, 1e-6, 1e-5, 1e-4, 1e-3, 1e-2, 1e-1, 1e0],
+        "num_leaves": Categorical([1, 3, 15, 31, 63, 127]),  # 2**depth - 1
+        "colsample_bytree": Categorical([0.5, 0.6, 0.7, 0.8, 0.9, 1.0]),
+        "subsample": Categorical([0.5, 0.6, 0.7, 0.8, 0.9, 1.0]),
+        "min_child_samples": Categorical([1, 2, 4, 8, 16, 32, 64, 128, 256]),
+        "min_child_weight": Real(1e-7, 1e+0, prior='log-uniform'), # [1e-7, 1e-6, 1e-5, 1e-4, 1e-3, 1e-2, 1e-1, 1e0],
+        "reg_alpha": Real(1e-7, 1e+0, prior='log-uniform'), # [1e-7, 1e-6, 1e-5, 1e-4, 1e-3, 1e-2, 1e-1, 1e0],
+        "reg_lambda": Real(1e-7, 1e+0, prior='log-uniform'), # [1e-7, 1e-6, 1e-5, 1e-4, 1e-3, 1e-2, 1e-1, 1e0],
+        "max_depth": [1, 2, 4, 8, 16, -1],
     }
     nested_scores = evaluate_pipeline_helper(X, y, pipeline, param_grid, random_state=random_state)
     return nested_scores

03_autogluon.py

@@ -9,8 +9,7 @@
 from utils import load_data
 
 
-SEC = 60
-
+SEC = 120
 
 def define_and_evaluate_autogluon_pipeline(X, y, random_state=0):
     # autogluon dataframes
@@ -23,15 +22,18 @@ def define_and_evaluate_autogluon_pipeline(X, y, random_state=0):
         data_df_test = data_df.iloc[test_inds, :]
         if len((set(y))) == 2:
             eval_metric = 'roc_auc'
+            problem_type = 'binary'
         else:
             eval_metric = 'f1_weighted'  # no multiclass auroc in autogluon
+            problem_type = 'multiclass'
         predictor = task.fit(
             data_df_train,
             "y",
             time_limits=SEC,
             auto_stack=True,
             output_directory=".autogluon_temp",
             eval_metric=eval_metric,
+            problem_type=problem_type,
             verbosity=0,
         )
         y_pred = predictor.predict_proba(data_df.iloc[test_inds, :])
@@ -69,5 +71,5 @@ def define_and_evaluate_autogluon_pipeline(X, y, random_state=0):
 times = np.array(times)
 
 # save everything to disk so we can make plots elsewhere
-with open(f"results/03_autoglun_sec_{SEC}.pickle", "wb") as f:
-    pickle.dump((results, times), f)
+with open(f"results/03_autogluon_NN_sec_{SEC}.pickle", "wb") as f:
+    pickle.dump((results, times), f)

make_figures.ipynb

@@ -2,7 +2,9 @@ numpy
 scipy
 pandas
 seaborn
-mxnet_cu102  # insert your own cuda number OR 'mxnet' if don't have a gpu
+mxnet_cu102  # insert your own cuda 3 digit version OR just 'mxnet' if don't have a gpu
 autogluon
-scikit-learn>=0.24
-lightgbm>=3.1.1
+scikit-learn==0.23.2
+lightgbm>=3.1.1
+optuna
+scikit-optimize