Extended output result, new metrics and minor fixes (#81)

* minor fixes

* pep8

* random state

* size of datasets

* apply comments

* dbscan eps fix

* extend output result

* codefactor.io

* pep8

* fix ci

* fix

* whitespace

* return to stock functions

* remove debug putput

* apply comments

* metrics & metric_type

* pep8

* pca

* roc_auc details section

* pep8

* finally solve roc_auc trouble

* add kmeans.iter_ & done metrics in bench

* n_iter_

* stay columnwise_score because of xgb

* roc_auc_score binary case

* add n_sv in svms

* apply comments

Co-authored-by: denis.kulandin <[email protected]>

Author: OnlyDeniko

bench.py

@@ -338,20 +338,47 @@ def columnwise_score(y, yp, score_func):
     return score_func(y, yp)
 
 
-def accuracy_score(y, yp):
-    return columnwise_score(y, yp, lambda y1, y2: np.mean(y1 == y2))
+def accuracy_score(y_true, y_pred):
+    return columnwise_score(y_true, y_pred, lambda y1, y2: np.mean(y1 == y2))
 
 
-def log_loss(y, yp):
+def log_loss(y_true, y_pred):
     from sklearn.metrics import log_loss as sklearn_log_loss
-    y = convert_to_numpy(y)
-    yp = convert_to_numpy(yp)
-    return sklearn_log_loss(y, yp)
+    y_true = convert_to_numpy(y_true)
+    y_pred = convert_to_numpy(y_pred)
+    return sklearn_log_loss(y_true, y_pred)
+
 
+def roc_auc_score(y_true, y_pred, multi_class='ovr'):
+    from sklearn.metrics import roc_auc_score as sklearn_roc_auc
+    y_true = convert_to_numpy(y_true)
+    y_pred = convert_to_numpy(y_pred)
+    if y_pred.shape[1] == 2:  # binary case
+        y_pred = y_pred[:, 1]
+    return sklearn_roc_auc(y_true, y_pred, multi_class=multi_class)
 
-def rmse_score(y, yp):
+
+def rmse_score(y_true, y_pred):
     return columnwise_score(
-        y, yp, lambda y1, y2: float(np.sqrt(np.mean((y1 - y2)**2))))
+        y_true, y_pred, lambda y1, y2: float(np.sqrt(np.mean((y1 - y2)**2))))
+
+
+def r2_score(y_true, y_pred):
+    from sklearn.metrics import r2_score as sklearn_r2_score
+    y_true = convert_to_numpy(y_true)
+    y_pred = convert_to_numpy(y_pred)
+    return sklearn_r2_score(y_true, y_pred)
+
+
+def davies_bouldin_score(X, labels):
+    from sklearn.metrics.cluster import davies_bouldin_score as sklearn_dbs
+    X = convert_to_numpy(X)
+    labels = convert_to_numpy(labels)
+    try:
+        res = sklearn_dbs(X, labels)
+    except ValueError as ex:
+        res = ex
+    return res
 
 
 def convert_data(data, dtype, data_order, data_format):
@@ -488,16 +515,21 @@ def gen_basic_dict(library, algorithm, stage, params, data, alg_instance=None,
 
 
 def print_output(library, algorithm, stages, params, functions,
-                 times, accuracy_type, accuracies, data, alg_instance=None,
+                 times, metric_type, metrics, data, alg_instance=None,
                  alg_params=None):
     if params.output_format == 'json':
         output = []
         for i in range(len(stages)):
             result = gen_basic_dict(library, algorithm, stages[i], params,
                                     data[i], alg_instance, alg_params)
             result.update({'time[s]': times[i]})
-            if accuracy_type is not None:
-                result.update({f'{accuracy_type}': accuracies[i]})
+            if metric_type is not None:
+                if isinstance(metric_type, str):
+                    result.update({f'{metric_type}': metrics[i]})
+                elif isinstance(metric_type, list):
+                    for ind, val in enumerate(metric_type):
+                        if metrics[ind][i] is not None:
+                            result.update({f'{val}': metrics[ind][i]})
             if hasattr(params, 'n_classes'):
                 result['input_data'].update({'classes': params.n_classes})
             if hasattr(params, 'n_clusters'):

configs/blogs/skl_2021_3.json

@@ -307,7 +307,7 @@
                 }
             ],
             "nu": [0.25],
-            "kernel": ["sigmoid"]
+            "kernel": ["poly"]
         },
         {
             "algorithm": "svr",

cuml_bench/dbscan.py

@@ -48,5 +48,5 @@
 
 bench.print_output(library='cuml', algorithm='dbscan', stages=['training'],
                    params=params, functions=['DBSCAN'], times=[time],
-                   accuracies=[acc], accuracy_type='davies_bouldin_score', data=[X],
+                   metrics=[acc], metric_type='davies_bouldin_score', data=[X],
                    alg_instance=dbscan)

cuml_bench/df_clsf.py

@@ -97,6 +97,6 @@ def predict(X):
 bench.print_output(library='cuml', algorithm='decision_forest_classification',
                    stages=['training', 'prediction'],
                    params=params, functions=['df_clsf.fit', 'df_clsf.predict'],
-                   times=[fit_time, predict_time], accuracy_type='accuracy[%]',
-                   accuracies=[train_acc, test_acc], data=[X_train, X_test],
+                   times=[fit_time, predict_time], metric_type='accuracy[%]',
+                   metrics=[train_acc, test_acc], data=[X_train, X_test],
                    alg_instance=clf)

cuml_bench/df_regr.py

@@ -93,6 +93,6 @@ def predict(X):
 bench.print_output(library='cuml', algorithm='decision_forest_regression',
                    stages=['training', 'prediction'], params=params,
                    functions=['df_regr.fit', 'df_regr.predict'],
-                   times=[fit_time, predict_time], accuracy_type='rmse',
-                   accuracies=[train_rmse, test_rmse], data=[X_train, X_test],
+                   times=[fit_time, predict_time], metric_type='rmse',
+                   metrics=[train_rmse, test_rmse], data=[X_train, X_test],
                    alg_instance=regr)

cuml_bench/elasticnet.py

@@ -56,6 +56,6 @@
 bench.print_output(library='cuml', algorithm='elastic-net',
                    stages=['training', 'prediction'], params=params,
                    functions=['ElasticNet.fit', 'ElasticNet.predict'],
-                   times=[fit_time, predict_time], accuracy_type='rmse',
-                   accuracies=[train_rmse, test_rmse], data=[X_train, X_train],
+                   times=[fit_time, predict_time], metric_type='rmse',
+                   metrics=[train_rmse, test_rmse], data=[X_train, X_train],
                    alg_instance=regr)

cuml_bench/kmeans.py

@@ -88,6 +88,6 @@ def kmeans_fit(X):
 bench.print_output(library='cuml', algorithm='kmeans',
                    stages=['training', 'prediction'], params=params,
                    functions=['KMeans.fit', 'KMeans.predict'],
-                   times=[fit_time, predict_time], accuracy_type='davies_bouldin_score',
-                   accuracies=[acc_train, acc_test], data=[X_train, X_test],
+                   times=[fit_time, predict_time], metric_type='davies_bouldin_score',
+                   metrics=[acc_train, acc_test], data=[X_train, X_test],
                    alg_instance=kmeans)

cuml_bench/knn_clsf.py

@@ -68,13 +68,13 @@
                        stages=['training', 'prediction'], params=params,
                        functions=['knn_clsf.fit', 'knn_clsf.predict'],
                        times=[train_time, predict_time],
-                       accuracies=[train_acc, test_acc], accuracy_type='accuracy[%]',
+                       metrics=[train_acc, test_acc], metric_type='accuracy[%]',
                        data=[X_train, X_test], alg_instance=knn_clsf)
 else:
     bench.print_output(library='cuml',
                        algorithm=knn_clsf.algorithm + '_knn_search',
                        stages=['training', 'search'], params=params,
                        functions=['knn_clsf.fit', 'knn_clsf.kneighbors'],
                        times=[train_time, predict_time],
-                       accuracies=[], accuracy_type=None,
+                       metrics=[], metric_type=None,
                        data=[X_train, X_test], alg_instance=knn_clsf)

cuml_bench/lasso.py

@@ -53,6 +53,6 @@
 bench.print_output(library='sklearn', algorithm='lasso',
                    stages=['training', 'prediction'],
                    params=params, functions=['Lasso.fit', 'Lasso.predict'],
-                   times=[fit_time, predict_time], accuracy_type='rmse',
-                   accuracies=[train_rmse, test_rmse], data=[X_train, X_test],
+                   times=[fit_time, predict_time], metric_type='rmse',
+                   metrics=[train_rmse, test_rmse], data=[X_train, X_test],
                    alg_instance=regr)

cuml_bench/linear.py

@@ -50,6 +50,6 @@
 bench.print_output(library='cuml', algorithm='linear_regression',
                    stages=['training', 'prediction'], params=params,
                    functions=['Linear.fit', 'Linear.predict'],
-                   times=[fit_time, predict_time], accuracy_type='rmse',
-                   accuracies=[train_rmse, test_rmse], data=[X_train, X_test],
+                   times=[fit_time, predict_time], metric_type='rmse',
+                   metrics=[train_rmse, test_rmse], data=[X_train, X_test],
                    alg_instance=regr)