ver

Author: microprediction

examples_ensemble_pycaret/README.md

@@ -0,0 +1,11 @@
+
+pycaret example
+
+# WARNING
+
+If you 
+
+    pip install pycaret
+    
+in order to run these examples you will force scikit-learn==0.23.2 
+

examples_ensemble_pycaret/playingaround.py

@@ -1,6 +1,13 @@
+import numpy as np
+import pandas as pd
 
+# TODO: Illustrate the use of a portfolio manager applied to model residuals
+# We use pycaret to generate some model predictions for various models
 # Example based on https://www.analyticsvidhya.com/blog/2021/07/automl-using-pycaret-with-a-regression-use-case-ii/
 
+
+
+
 if __name__=='__main__':
     try:
         import pycaret
@@ -11,35 +18,63 @@
 
     all_data = get_data('diamond', profile=False)
 
-    holdout = all_data[-500:]
+    n_train = 100
+    n_test = 100
+    all_data = all_data[:n_train+n_test]
+    data = all_data[:n_train]
+    holdout_data = all_data[n_train:]
 
-    data = all_data[:-500]
 
-    exp_reg102 = setup(data=data, target='Price', session_id=123,
-                       normalize=True, transformation=True, transform_target=True,
-                       combine_rare_levels=True, rare_level_threshold=0.05,
-                       remove_multicollinearity=True, multicollinearity_threshold=0.95,
-                       bin_numeric_features=['Carat Weight'],
-                       log_experiment=True, experiment_name='diamond1',html=False)
-    print('done')
+    if True:
+        # Is this needed?
+        exp_reg102 = setup(data=all_data, target='Price', session_id=123,
+                           normalize=True, transformation=True, transform_target=True,
+                           combine_rare_levels=True, rare_level_threshold=0.05,
+                           remove_multicollinearity=True, multicollinearity_threshold=0.95,
+                           bin_numeric_features=['Carat Weight'],
+                           log_experiment=True, experiment_name='diamond1',html=False)
+        print('done')
 
-
-    shortlist = ['catboost','xgboost','lightgbm','rf']
-    print('Creating')
+    shortlist = ['catboost','xgboost','lightgbm','rf','et','ada','mlp','knn','huber','tr','llar','lar','ridge','lasso']
+    print('Creating models and turning them')
     workin = dict()
     for nm in shortlist:
         try:
             model = create_model(nm)
             workin[nm]=model
+            tune_model(workin[nm])
         except Exception as e:
             print(str(e))
             print('sorry no dice for '+nm)
 
-    tuned = dict( [ (nm, tune_model(w)) for n,w in workin.items() ])
 
-    y_hats = list()
-    for nm, tuned_model in tuned:
-        y_hat = predict_model(estimator=tuned_model, data=holdout)
-        y_hats.append(y_hat)
+    stuff = dict()
+    for partition, the_data in zip(['train','holdout'],[data,holdout_data]):
+        df = pd.DataFrame(columns=list(workin.keys()))
+        for nm, tuned_model in workin.items():
+            y_hat = predict_model(estimator=tuned_model, data=the_data)['Label']
+            df[nm] = y_hat
+        df.to_csv(partition+'.csv')
+        stuff[partition] = y_hat
+
+    # Use a portfolio manager to combine models ?
+    ys = stuff['train'].values
+    from precise.skaters.managers.schurmanagers import schur_weak_pm_t0_d0_r050_n25_g100_long_manager as mgr
+    s = {}
+    for y in ys:
+        w, s = mgr(s=s,y=y)
+
+    X = stuff['holdout'].values
+    y_hat = np.dot(w, X)
+    stuff['holdout']['blend'] = y_hat
+
+    all_names = list(workin.keys())+['blend']
+    holdout_error_df = pd.DataFrame(columns=all_names)
+    for nm in all_names:
+        holdout_error_df[nm] = (stuff['holdout'][nm]-holdout_data['Price'])**2
+
+    print(holdout_error_df.describe())
+
+
 
 

examples_ensembles_lazypredict/__init__.py

@@ -0,0 +1,95 @@
+
+from sklearn import datasets
+from sklearn.utils import shuffle
+import numpy as np
+import pandas as pd
+from pprint import pprint
+
+if __name__=='__main__':
+    try:
+        from lazypredict.Supervised import LazyRegressor
+    except ImportError:
+        raise Exception('pip install lazypredict')
+
+    boston = datasets.load_boston()
+    X, y = shuffle(boston.data, boston.target)
+    X = X.astype(np.float32)
+    n_train = 100
+    n_test = 50
+    X_train, y_train = X[:n_train], y[:n_train]
+    X_test, y_test = X[n_train:(n_train+n_test)], y[n_train:(n_train+n_test)]
+    X_val, y_val = X[(n_train+n_test):], y[(n_train+n_test):]
+    X_train_and_test = X[:(n_train+n_test)]
+    y_train_and_test = y[:(n_train+n_test)]
+
+    # Train on some
+    reg1 = LazyRegressor(verbose=0, ignore_warnings=False, custom_metric=None, predictions=True)
+    models1, predictions1 = reg1.fit(np.copy(X_train), np.copy(X_test), np.copy(y_train), np.copy(y_test))
+    print(models1[:5])
+
+    # Train on some, predict validation
+    reg2 = LazyRegressor(verbose=0, ignore_warnings=False, custom_metric=None, predictions=True)
+    X_train_and_test_copy = np.copy(X_train_and_test)
+    X_val_copy = np.copy(X_val)
+    models2, predictions2 = reg2.fit(X_train_and_test_copy, X_val_copy, np.copy(y_train_and_test), np.copy(y_val))
+    yhat_val = predictions2.values
+    print(models2[:5])
+
+    # In-sample performance on train
+    reg3 = LazyRegressor(verbose=0, ignore_warnings=False, custom_metric=None, predictions=True)
+    models3, predictions3 = reg3.fit(np.copy(X_train), np.copy(X_train), np.copy(y_train), np.copy(y_train))
+
+    # In-sample performance on train + test
+    reg4 = LazyRegressor(verbose=0, ignore_warnings=False, custom_metric=None, predictions=True)
+    models4, predictions4 = reg4.fit(np.copy(X_train_and_test), np.copy(X_train_and_test), np.copy(y_train_and_test), np.copy(y_train_and_test))
+
+    best_model_1 = models1.index[0]  # <-- Best out of sample on test
+    best_model_2 = models3.index[0]  # <-- Best in sample on train
+    best_model_3 = models4.index[0]  # <-- Best in sample on train+test
+
+    if True:
+        # Train cov on out of sample prediction errors
+        print('Creating portfolio ...')
+        from precise.skaters.managers.ppomanagers import ppo_sk_glcv_pcov_d0_n100_t0_vol_long_manager as mgr
+        s = {}
+        yhat_train = np.copy(predictions1.values)
+        n_train = len(yhat_train)
+        es = [-1]*(n_train-1)+[1]
+        for y, y_target,e in zip(yhat_train, y_train,es):
+            y_error = np.copy(y-y_target)
+            w, s = mgr(s=s, y=y_error, e=e)
+
+    else:
+        n_models = len(models1)
+        w = np.ones(n_models)/n_models
+
+    w_dict = sorted(zip(w, models1.index), reverse=True)
+    pprint(w_dict)
+
+    # Refit models using all the train+test data, and combine
+
+    sum_w = sum(w)
+    yhat_weighted = np.dot( yhat_val, w )
+    predictions2['weighted'] = yhat_weighted
+    predictions2['best 1 (' + best_model_1 + ')'] = predictions2[best_model_1]
+    predictions2['best 2 (' + best_model_2 + ')'] = predictions2[best_model_2]
+    predictions2['best 3 (' + best_model_3 + ')'] = predictions2[best_model_3]
+
+    val_errors = predictions2.copy()
+    for col in predictions2.columns:
+        val_errors[col] = predictions2[col] - y_val
+
+    sq_errors = val_errors**2
+    print(sq_errors.mean().sort_values())
+    print('done')
+
+
+
+
+
+
+
+
+
+
+