Update of parameters for sklearn & cuml benchmarks (#35)

Author: PetrovKP

.gitignore

@@ -0,0 +1,13 @@
+# Logs
+*.log
+
+# Release and work directories
+__pycache__*
+__work*
+
+# Visual Studio related files, e.g., ".vscode"
+.vs*
+
+# Datasets
+dataset
+*.csv

cuml/dbscan.py

@@ -3,8 +3,9 @@
 # SPDX-License-Identifier: MIT
 
 import argparse
-from bench import parse_args, measure_function_time, load_data, print_output
+from bench import parse_args, measure_function_time, load_data, print_output, convert_to_numpy
 from cuml import DBSCAN
+from sklearn.metrics.cluster import davies_bouldin_score
 
 parser = argparse.ArgumentParser(description='cuML DBSCAN benchmark')
 parser.add_argument('-e', '--eps', '--epsilon', type=float, default=10.,
@@ -27,9 +28,14 @@
 # Time fit
 time, _ = measure_function_time(dbscan.fit, X, params=params)
 labels = dbscan.labels_
-params.n_clusters = len(set(labels)) - (1 if -1 in labels else 0)
+
+X_host = convert_to_numpy(X)
+labels_host = convert_to_numpy(labels)
+
+acc = davies_bouldin_score(X_host, labels_host)
+params.n_clusters = len(set(labels_host)) - (1 if -1 in labels_host else 0)
 
 print_output(library='cuml', algorithm='dbscan', stages=['training'],
              columns=columns, params=params, functions=['DBSCAN'],
-             times=[time], accuracies=[None], accuracy_type=None, data=[X],
+             times=[time], accuracies=[acc], accuracy_type='davies_bouldin_score', data=[X],
              alg_instance=dbscan)

cuml/elasticnet.py

@@ -0,0 +1,51 @@
+# Copyright (C) 2020 Intel Corporation
+#
+# SPDX-License-Identifier: MIT
+
+import argparse
+from bench import (
+    parse_args, measure_function_time, load_data, print_output, rmse_score
+)
+from cuml.linear_model import ElasticNet
+
+parser = argparse.ArgumentParser(description='scikit-learn elastic-net regression '
+                                             'benchmark')
+parser.add_argument('--no-fit-intercept', dest='fit_intercept', default=True,
+                    action='store_false',
+                    help="Don't fit intercept (assume data already centered)")
+parser.add_argument('--alpha', dest='alpha', type=float, default=1.0,
+                    help='Regularization parameter')
+parser.add_argument('--maxiter', type=int, default=1000,
+                    help='Maximum iterations for the iterative solver')
+parser.add_argument('--l1_ratio', dest='l1_ratio', type=float, default=0.5,
+                    help='Regularization parameter')
+parser.add_argument('--tol', type=float, default=0.0,
+                    help='Tolerance for solver.')
+params = parse_args(parser)
+
+# Load data
+X_train, X_test, y_train, y_test = load_data(params)
+
+# Create our regression object
+regr = ElasticNet(fit_intercept=params.fit_intercept, l1_ratio=params.l1_ratio, alpha=params.alpha,
+                        tol=params.tol, max_iter=params.maxiter)
+
+columns = ('batch', 'arch', 'prefix', 'function', 'threads', 'dtype', 'size',
+           'time')
+
+# Time fit
+fit_time, _ = measure_function_time(regr.fit, X_train, y_train, params=params)
+
+# Time predict
+predict_time, pred_train = measure_function_time(regr.predict, X_train, params=params)
+
+train_rmse = rmse_score(pred_train, y_train)
+pred_test = regr.predict(X_test)
+test_rmse = rmse_score(pred_test, y_test)
+
+print_output(library='cuml', algorithm='elastic-net',
+             stages=['training', 'prediction'], columns=columns,
+             params=params, functions=['ElasticNet.fit', 'ElasticNet.predict'],
+             times=[fit_time, predict_time], accuracy_type='rmse',
+             accuracies=[train_rmse, test_rmse], data=[X_train, X_train],
+             alg_instance=regr)

cuml/kmeans.py

@@ -4,12 +4,12 @@
 
 import argparse
 from bench import (
-    parse_args, measure_function_time, load_data, print_output
+    parse_args, measure_function_time, load_data, print_output, convert_to_numpy
 )
 import numpy as np
 from cuml import KMeans
 import warnings
-
+from sklearn.metrics.cluster import davies_bouldin_score
 
 warnings.filterwarnings('ignore', category=FutureWarning)
 parser = argparse.ArgumentParser(description='cuML K-means benchmark')
@@ -27,8 +27,10 @@
 # Load and convert generated data
 X_train, X_test, _, _ = load_data(params)
 
+if params.filei == 'k-means++':
+    X_init = 'k-means++'
 # Load initial centroids from specified path
-if params.filei is not None:
+elif params.filei is not None:
     X_init = np.load(params.filei).astype(params.dtype)
     params.n_clusters = X_init.shape[0]
 # or choose random centroids from training data
@@ -57,15 +59,24 @@ def kmeans_fit(X):
 
 # Time fit
 fit_time, kmeans = measure_function_time(kmeans_fit, X_train, params=params)
-train_inertia = float(kmeans.inertia_)
+train_predict = kmeans.predict(X_train)
 
 # Time predict
-predict_time, _ = measure_function_time(kmeans.predict, X_test, params=params)
-test_inertia = float(kmeans.inertia_)
+predict_time, test_predict = measure_function_time(kmeans.predict, X_test, params=params)
+
+X_train_host = convert_to_numpy(X_train)
+train_predict_host = convert_to_numpy(train_predict)
+acc_train = davies_bouldin_score(X_train_host, train_predict_host)
+
+X_test_host = convert_to_numpy(X_test)
+test_predict_host = convert_to_numpy(test_predict)
+
+acc_test = davies_bouldin_score(X_test_host, test_predict_host)
 
 print_output(library='cuml', algorithm='kmeans',
              stages=['training', 'prediction'], columns=columns,
              params=params, functions=['KMeans.fit', 'KMeans.predict'],
-             times=[fit_time, predict_time], accuracy_type='inertia',
-             accuracies=[train_inertia, test_inertia], data=[X_train, X_test],
+             times=[fit_time, predict_time], accuracy_type='davies_bouldin_score',
+             accuracies=[acc_train, acc_test], data=[X_train, X_test],
              alg_instance=kmeans)
+

cuml/lasso.py

@@ -0,0 +1,49 @@
+# Copyright (C) 2020 Intel Corporation
+#
+# SPDX-License-Identifier: MIT
+
+import argparse
+from bench import (
+    parse_args, measure_function_time, load_data, print_output, rmse_score
+)
+from cuml.linear_model import Lasso
+
+parser = argparse.ArgumentParser(description='scikit-learn lasso regression '
+                                             'benchmark')
+parser.add_argument('--no-fit-intercept', dest='fit_intercept', default=False,
+                    action='store_false',
+                    help="Don't fit intercept (assume data already centered)")
+parser.add_argument('--alpha', dest='alpha', type=float, default=1.0,
+                    help='Regularization parameter')
+parser.add_argument('--maxiter', type=int, default=1000,
+                    help='Maximum iterations for the iterative solver')
+parser.add_argument('--tol', type=float, default=0.0,
+                    help='Tolerance for solver.')
+params = parse_args(parser)
+
+# Load data
+X_train, X_test, y_train, y_test = load_data(params)
+
+# Create our regression object
+regr = Lasso(fit_intercept=params.fit_intercept, alpha=params.alpha,
+                        tol=params.tol, max_iter=params.maxiter)
+
+columns = ('batch', 'arch', 'prefix', 'function', 'threads', 'dtype', 'size',
+           'time')
+
+# Time fit
+fit_time, _ = measure_function_time(regr.fit, X_train, y_train, params=params)
+
+# Time predict
+predict_time, pred_train = measure_function_time(regr.predict, X_train, params=params)
+
+train_rmse = rmse_score(pred_train, y_train)
+pred_test = regr.predict(X_test)
+test_rmse = rmse_score(pred_test, y_test)
+
+print_output(library='sklearn', algorithm='lasso',
+             stages=['training', 'prediction'], columns=columns,
+             params=params, functions=['Lasso.fit', 'Lasso.predict'],
+             times=[fit_time, predict_time], accuracy_type='rmse',
+             accuracies=[train_rmse, test_rmse], data=[X_train, X_test],
+             alg_instance=regr)

cuml/log_reg.py

@@ -33,8 +33,8 @@
 # Create our classifier object
 clf = LogisticRegression(penalty='l2', C=params.C,
                          linesearch_max_iter=params.linesearch_max_iter,
-                         fit_intercept=params.fit_intercept,
-                         verbose=params.verbose, tol=params.tol,
+                         fit_intercept=params.fit_intercept, verbose=params.verbose,
+                         tol=params.tol,
                          max_iter=params.maxiter, solver=params.solver)
 
 columns = ('batch', 'arch', 'prefix', 'function', 'threads', 'dtype', 'size',

cuml/svm.py

@@ -74,16 +74,17 @@ def get_optimal_cache_size(n_rows, dtype=np.double, max_cache=64):
 # Time fit and predict
 fit_time, _ = measure_function_time(clf.fit, X_train, y_train, params=params)
 params.sv_len = clf.support_.shape[0]
-y_pred = clf.predict(X_train)
-train_acc = 100 * accuracy_score(y_pred, y_train)
 
 predict_time, y_pred = measure_function_time(
-    clf.predict, X_test, params=params)
+    clf.predict, X_train, params=params)
+train_acc = 100 * accuracy_score(y_pred, y_train)
+
+y_pred = clf.predict(X_test)
 test_acc = 100 * accuracy_score(y_pred, y_test)
 
 print_output(library='cuml', algorithm='svc',
              stages=['training', 'prediction'], columns=columns,
              params=params, functions=['SVM.fit', 'SVM.predict'],
              times=[fit_time, predict_time], accuracy_type='accuracy[%]',
-             accuracies=[train_acc, test_acc], data=[X_train, X_test],
+             accuracies=[train_acc, test_acc], data=[X_train, X_train],
              alg_instance=clf)

daal4py/svm.py

@@ -140,11 +140,8 @@ def construct_dual_coefs(model, num_classes, X, y):
         del tmp
 
         support_ = two_class_sv_ind_[perm]
-        # support_vectors_ = X[support_]
-
         dual_coef_ = model.ClassificationCoefficients.T
         dual_coef_ = dual_coef_[:, perm]
-        # intercept_ = np.array([model.Bias])
 
     else:
         # multi-class
@@ -170,10 +167,6 @@ def construct_dual_coefs(model, num_classes, X, y):
                                  two_class_sv_ind_.ravel())
                 sv_ind_by_clf.append(sv_ind)
 
-                # svs_ = getArrayFromNumericTable(
-                #     svm_model.getSupportVectors())
-                # assert np.array_equal(svs_, X[sv_ind])
-
                 intercepts.append(-svm_model.Bias)
                 coefs.append(-svm_model.ClassificationCoefficients)
                 model_id += 1
@@ -190,8 +183,6 @@ def construct_dual_coefs(model, num_classes, X, y):
             sv_coef_by_clf,  # classification coeffs by two-class classifiers
             y.squeeze().astype(np.intp, copy=False)   # integer labels
         )
-        # support_vectors_ = X[support_]
-        # intercept_ = np.array(intercepts)
 
     return support_
 
@@ -210,12 +201,8 @@ def test_fit(X, y, params):
     fptype = getFPType(X)
     kf = daal_kernel(params.kernel, fptype, gamma=params.gamma)
 
-    if params.n_classes == 2:
-        y[y == 0] = -1
-    else:
-        y[y == -1] = 0
-
     svm_train = svm_training(
+            method='thunder',
             fptype=fptype,
             C=params.C,
             maxIterations=params.maxiter,
@@ -269,7 +256,6 @@ def test_predict(X, training_result, params):
     else:
         prdct = multi_class_classifier_prediction(
                 nClasses=params.n_classes,
-                method='thunder',
                 fptype=fptype,
                 maxIterations=params.maxiter,
                 accuracyThreshold=params.tol,
@@ -323,9 +309,6 @@ def main():
     params.cache_size_bytes = cache_size_bytes
     params.n_classes = np.unique(y_train).size
 
-    # This is necessary for daal
-    y_train[y_train == 0] = -1
-
     columns = ('batch', 'arch', 'prefix', 'function', 'threads', 'dtype',
                'size', 'kernel', 'cache_size_mb', 'C', 'sv_len', 'n_classes',
                'accuracy', 'time')

make_datasets.py

@@ -2,7 +2,6 @@
 #
 # SPDX-License-Identifier: MIT
 
-
 import argparse
 import sys
 
@@ -14,7 +13,7 @@
 def gen_blobs(args):
     X, y = make_blobs(n_samples=args.samples + args.test_samples,
                       n_features=args.features,
-                      centers=None,
+                      centers=args.clusters,
                       center_box=(-32, 32),
                       shuffle=True,
                       random_state=args.seed)
@@ -55,19 +54,16 @@ def gen_classification(args):
         np.save(args.fileytest, y[args.samples:])
     return 0
 
-
 def _ch_size(n):
     return n * (n + 1) // 2
 
-
 def _get_cluster_centers(clusters, features):
     import numpy.random_intel as nri
     rs = nri.RandomState(1234, brng='SFMT19937')
     cluster_centers = rs.randn(clusters, features)
     cluster_centers *= np.double(clusters)
     return cluster_centers
 
-
 def gen_kmeans(args):
     try:
         import numpy.random_intel as nri

runner.py

@@ -188,6 +188,7 @@ def is_ht_enabled():
     generate_cases(params)
     verbose_print(f'{algorithm} algorithm: {len(libs) * len(cases)} case(s),'
                   f' {len(params_set["dataset"])} dataset(s)\n')
+
     for dataset in params_set['dataset']:
         if dataset['source'] in ['csv', 'npy']:
             paths = f'--file-X-train {dataset["training"]["x"]}'
@@ -212,6 +213,8 @@ class GenerationArgs:
             else:
                 gen_args.seed = 777
 
+            # default values
+            gen_args.clusters = 10
             gen_args.type = dataset['type']
             gen_args.samples = dataset['training']['n_samples']
             gen_args.features = dataset['n_features']