update LR to be less bad

Author: Sergey Feldman

01_compare_baseline_models.py

@@ -12,7 +12,7 @@
 from sklearn.model_selection import GridSearchCV, cross_val_score, StratifiedKFold
 from sklearn.preprocessing import MinMaxScaler
 from sklearn.pipeline import Pipeline
-from sklearn.svm import SVC, LinearSVC
+from sklearn.svm import SVC
 from utils import load_data
 
 
@@ -53,7 +53,12 @@ def define_and_evaluate_pipelines(X, y, random_state=0):
     pipeline2 = Pipeline(
         [
             ("scaler", MinMaxScaler()),
-            ("logistic", LogisticRegression(solver="saga", max_iter=10000, random_state=random_state)),
+            (
+                "logistic",
+                LogisticRegression(
+                    solver="saga", max_iter=10000, penalty="elasticnet", l1_ratio=0.1, random_state=random_state
+                ),  # without elasticnet penalty, LR can get pretty bad sometimes
+            ),
         ]
     )
     param_grid2 = {

02_lightgbm.py

@@ -15,8 +15,8 @@
 from utils import load_data
 
 N_JOBS = 4 * 4 * 9
-N_ITER = 25  # budget for hyperparam search
-
+N_ITER = 50  # budget for hyperparam search
+N_STARTUP_JOBS = 20  # hyperopt does a bunch of random jobs first
 
 HYPEROPT_SPACE = {
     "learning_rate": hp.choice("learning_rate", [0.1, 0.05, 0.01, 0.005, 0.001]),
@@ -71,7 +71,7 @@ def obj(params):
         _ = fmin(
             fn=obj,
             space=HYPEROPT_SPACE,
-            algo=partial(tpe.suggest, n_startup_jobs=5),
+            algo=partial(tpe.suggest, n_startup_jobs=N_STARTUP_JOBS),
             max_evals=N_ITER,
             trials=trials,
             rstate=np.random.RandomState(random_state),