Fix of getFPType import for newer daal4py, distances benchs change

Alexsandruss · Alexsandruss · commit 03fd5888928d · 2020-03-22T19:44:48.000+03:00
diff --git a/config_example.json b/config_example.json
@@ -92,14 +92,36 @@
             "algorithm": "svm",
             "dataset": [
                 {
-                    "training": "synth_clsf_100000_100_2"
+                    "training": "synth_clsf_20000_100_2"
                 },
                 {
-                    "training": "synth_clsf_100000_100_5"
+                    "training": "synth_clsf_20000_100_5"
                 }
             ],
-            "max-cache-size": [96],
+            "max-cache-size": [4],
             "kernel": ["rbf"]
+        },
+        {
+            "lib": ["xgboost"],
+            "algorithm": "gbt",
+            "dataset": [
+                {
+                    "training": "synth_clsf_10000_100_2"
+                }
+            ],
+            "tree-method": ["hist"],
+            "objective": ["binary:logistic"]
+        },
+        {
+            "lib": ["xgboost"],
+            "algorithm": "gbt",
+            "dataset": [
+                {
+                    "training": "synth_reg_10000_100"
+                }
+            ],
+            "tree-method": ["hist"],
+            "objective": ["reg:squarederror"]
         }
     ]
 }
diff --git a/cuml/bench.py b/cuml/bench.py
@@ -106,7 +106,7 @@ def parse_args(parser, size=None, loop_types=(),
     parser.add_argument('--time-method', type=str, default='mean_min',
                         choices=('box_filter', 'mean_min'),
                         help='Method used for time mesurements')
-    parser.add_argument('--n-meas', type=int, default=100,
+    parser.add_argument('--box-filter-measurements', type=int, default=100,
                         help='Maximum number of measurements in box filter')
     parser.add_argument('--inner-loops', default=100, type=int,
                         help='Maximum inner loop iterations '
@@ -226,10 +226,10 @@ def measure_function_time(func, *args, params, **kwargs):
                              inner_loops=params.inner_loops,
                              goal_outer_loops=params.goal,
                              time_limit=params.time_limit,
-                             verbose=params.verbose)
+                             verbose=params.verbose, **kwargs)
     else:
         return time_box_filter(func, *args, n_meas=params.n_meas,
-                               time_limit=params.time_limit)
+                               time_limit=params.time_limit, **kwargs)
 
 
 def time_box_filter(func, *args, n_meas, time_limit, **kwargs):
@@ -559,6 +559,7 @@ def print_output(library, algorithm, stages, columns, params, functions,
     if params.output_format == 'csv':
         output_csv(columns, params, functions, times, accuracies)
     elif params.output_format == 'json':
+        output = []
         for i in range(len(stages)):
             result = gen_basic_dict(library, algorithm, stages[i], params,
                                     data[i], alg_instance, alg_params)
@@ -580,5 +581,13 @@ def print_output(library, algorithm, stages, columns, params, functions,
                         result['algorithm_parameters']['init'] = 'random'
                 if 'handle' in result['algorithm_parameters'].keys():
                     del result['algorithm_parameters']['handle']
-            print(json.dumps(result, indent=4),
-                  end=',\n' if i != len(stages) - 1 else '\n')
+            output.append(result)
+        print(json.dumps(output, indent=4))
+
+
+def import_fptype_getter():
+    try:
+        from daal4py.sklearn._utils import getFPType
+    except:
+        from daal4py.sklearn.utils import getFPType
+    return getFPType
diff --git a/daal4py/bench.py b/daal4py/bench.py
@@ -106,7 +106,7 @@ def parse_args(parser, size=None, loop_types=(),
     parser.add_argument('--time-method', type=str, default='mean_min',
                         choices=('box_filter', 'mean_min'),
                         help='Method used for time mesurements')
-    parser.add_argument('--n-meas', type=int, default=100,
+    parser.add_argument('--box-filter-measurements', type=int, default=100,
                         help='Maximum number of measurements in box filter')
     parser.add_argument('--inner-loops', default=100, type=int,
                         help='Maximum inner loop iterations '
@@ -226,10 +226,10 @@ def measure_function_time(func, *args, params, **kwargs):
                              inner_loops=params.inner_loops,
                              goal_outer_loops=params.goal,
                              time_limit=params.time_limit,
-                             verbose=params.verbose)
+                             verbose=params.verbose, **kwargs)
     else:
         return time_box_filter(func, *args, n_meas=params.n_meas,
-                               time_limit=params.time_limit)
+                               time_limit=params.time_limit, **kwargs)
 
 
 def time_box_filter(func, *args, n_meas, time_limit, **kwargs):
@@ -559,6 +559,7 @@ def print_output(library, algorithm, stages, columns, params, functions,
     if params.output_format == 'csv':
         output_csv(columns, params, functions, times, accuracies)
     elif params.output_format == 'json':
+        output = []
         for i in range(len(stages)):
             result = gen_basic_dict(library, algorithm, stages[i], params,
                                     data[i], alg_instance, alg_params)
@@ -580,5 +581,13 @@ def print_output(library, algorithm, stages, columns, params, functions,
                         result['algorithm_parameters']['init'] = 'random'
                 if 'handle' in result['algorithm_parameters'].keys():
                     del result['algorithm_parameters']['handle']
-            print(json.dumps(result, indent=4),
-                  end=',\n' if i != len(stages) - 1 else '\n')
+            output.append(result)
+        print(json.dumps(output, indent=4))
+
+
+def import_fptype_getter():
+    try:
+        from daal4py.sklearn._utils import getFPType
+    except:
+        from daal4py.sklearn.utils import getFPType
+    return getFPType
diff --git a/daal4py/dbscan.py b/daal4py/dbscan.py
@@ -3,9 +3,13 @@
 # SPDX-License-Identifier: MIT
 
 import argparse
-from bench import parse_args, measure_function_time, load_data, print_output
+from bench import (
+    parse_args, measure_function_time, load_data, print_output,
+    import_fptype_getter
+)
 from daal4py import dbscan
-from daal4py.sklearn.utils import getFPType
+getFPType = import_fptype_getter()
+
 
 parser = argparse.ArgumentParser(description='daal4py DBSCAN clustering '
                                              'benchmark')
diff --git a/daal4py/df_clsf.py b/daal4py/df_clsf.py
@@ -5,14 +5,14 @@
 import argparse
 from bench import (
     parse_args, measure_function_time, load_data, print_output, accuracy_score,
-    float_or_int
+    float_or_int, import_fptype_getter
 )
 import numpy as np
 from daal4py import (
     decision_forest_classification_training,
     decision_forest_classification_prediction, engines_mt2203
 )
-from daal4py.sklearn.utils import getFPType
+getFPType = import_fptype_getter()
 
 
 def df_clsf_fit(X, y, n_classes, n_trees=100, seed=12345,
diff --git a/daal4py/df_regr.py b/daal4py/df_regr.py
@@ -6,12 +6,14 @@
 import argparse
 from bench import (
     parse_args, measure_function_time, load_data, print_output, rmse_score,
-    float_or_int
+    float_or_int, import_fptype_getter
 )
-from daal4py import decision_forest_regression_training, \
-                    decision_forest_regression_prediction, \
-                    engines_mt2203
-from daal4py.sklearn.utils import getFPType
+from daal4py import (
+    decision_forest_regression_training,
+    decision_forest_regression_prediction,
+    engines_mt2203
+)
+getFPType = import_fptype_getter()
 
 
 def df_regr_fit(X, y, n_trees=100, seed=12345, n_features_per_node=0,
diff --git a/daal4py/distances.py b/daal4py/distances.py
@@ -4,50 +4,38 @@
 
 import argparse
 from bench import (
-    parse_args, measure_function_time, print_header, print_row, load_data,
-    gen_basic_dict
+    parse_args, measure_function_time, print_output, load_data,
+    import_fptype_getter
 )
 import daal4py
-from daal4py.sklearn.utils import getFPType
+getFPType = import_fptype_getter()
 
-parser = argparse.ArgumentParser(description='daal4py pairwise distances '
-                                             'benchmark')
-parser.add_argument('--metrics', nargs='*', default=['cosine', 'correlation'],
-                    choices=('cosine', 'correlation'),
-                    help='Metrics to test for pairwise_distances')
-params = parse_args(parser, size=(1000, 150000), prefix='daal4py')
-
-# Generate random data
-X, _, _, _ = load_data(params, generated_data=['X_train'], add_dtype=True)
-
-columns = ('batch', 'arch', 'prefix', 'function', 'threads', 'dtype', 'size',
-           'time')
 
-if params.output_format == 'csv':
-    print_header(columns, params)
+def compute_distances(pairwise_distances, X):
+    algorithm = pairwise_distances(fptype=getFPType(X))
+    return algorithm.compute(X)
 
-for metric in params.metrics:
-    pairwise_distances = getattr(daal4py, f'{metric}_distance')
 
-    def test_distances(pairwise_distances, X):
-        algorithm = pairwise_distances(fptype=getFPType(X))
-        return algorithm.compute(X)
+parser = argparse.ArgumentParser(description='daal4py pairwise distances '
+                                             'benchmark')
+parser.add_argument('--metric', default='cosine',
+                    choices=['cosine', 'correlation'],
+                    help='Metric to test for pairwise distances')
+params = parse_args(parser, size=(1000, 150000))
 
-    time, _ = measure_function_time(
-        test_distances, pairwise_distances, X, params=params)
+# Load data
+X, _, _, _ = load_data(params, generated_data=['X_train'], add_dtype=True)
 
-    if params.output_format == 'csv':
-        print_row(columns, params, function=metric.capitalize(), time=time)
-    elif params.output_format == 'json':
-        import json
+pairwise_distances = getattr(daal4py, f'{params.metric}_distance')
 
-        result = gen_basic_dict('daal4py', 'distances', 'computation',
-                                params, X)
+time, _ = measure_function_time(
+    compute_distances, pairwise_distances, X, params=params)
 
-        result.update({
-            'metric': metric,
-            'time[s]': time
-        })
+columns = ('batch', 'arch', 'prefix', 'function', 'threads', 'dtype', 'size',
+           'time')
 
-        print(json.dumps(result, indent=4),
-              end=',\n' if metric != params.metrics[-1] else '\n')
+print_output(library='daal4py', algorithm='distances', stages=['computation'],
+             columns=columns, params=params,
+             functions=[params.metric.capitalize()], times=[time],
+             accuracy_type=None, accuracies=[None], data=[X],
+             alg_params={'metric': params.metric})
diff --git a/daal4py/kmeans.py b/daal4py/kmeans.py
@@ -4,11 +4,13 @@
 
 import argparse
 from bench import (
-    parse_args, measure_function_time, load_data, print_output
+    parse_args, measure_function_time, load_data, print_output,
+    import_fptype_getter
 )
-from daal4py import kmeans
-from daal4py.sklearn.utils import getFPType
 import numpy as np
+from daal4py import kmeans
+getFPType = import_fptype_getter()
+
 
 parser = argparse.ArgumentParser(description='daal4py K-Means clustering '
                                              'benchmark')
diff --git a/daal4py/linear.py b/daal4py/linear.py
@@ -4,10 +4,11 @@
 
 import argparse
 from bench import (
-    parse_args, measure_function_time, load_data, print_output, rmse_score
+    parse_args, measure_function_time, load_data, print_output, rmse_score,
+    import_fptype_getter
 )
 from daal4py import linear_regression_training, linear_regression_prediction
-from daal4py.sklearn.utils import getFPType
+getFPType = import_fptype_getter()
 
 
 parser = argparse.ArgumentParser(description='daal4py linear regression '
diff --git a/daal4py/log_reg.py b/daal4py/log_reg.py
@@ -4,13 +4,15 @@
 
 import argparse
 from bench import (
-    parse_args, measure_function_time, load_data, print_output, accuracy_score
+    parse_args, measure_function_time, load_data, print_output, accuracy_score,
+    import_fptype_getter
 )
 import numpy as np
 import daal4py
 from daal4py import math_logistic, math_softmax
-from daal4py.sklearn.utils import getFPType, make2d
+from daal4py.sklearn.utils import make2d
 import scipy.optimize
+getFPType = import_fptype_getter()
 
 _logistic_loss = daal4py.optimization_solver_logistic_loss
 _cross_entropy_loss = daal4py.optimization_solver_cross_entropy_loss
diff --git a/daal4py/pca.py b/daal4py/pca.py
@@ -4,12 +4,13 @@
 
 import argparse
 from bench import (
-    parse_args, measure_function_time, load_data, print_output
+    parse_args, measure_function_time, load_data, print_output,
+    import_fptype_getter
 )
 import numpy as np
 from daal4py import pca, pca_transform, normalization_zscore
-from daal4py.sklearn.utils import getFPType
 from sklearn.utils.extmath import svd_flip
+getFPType = import_fptype_getter()
 
 parser = argparse.ArgumentParser(description='daal4py PCA benchmark')
 parser.add_argument('--svd-solver', type=str,
diff --git a/daal4py/ridge.py b/daal4py/ridge.py
@@ -4,11 +4,13 @@
 
 import argparse
 from bench import (
-    parse_args, measure_function_time, load_data, print_output, rmse_score
+    parse_args, measure_function_time, load_data, print_output, rmse_score,
+    import_fptype_getter
 )
-from daal4py import ridge_regression_training, ridge_regression_prediction
-from daal4py.sklearn.utils import getFPType
 import numpy as np
+from daal4py import ridge_regression_training, ridge_regression_prediction
+getFPType = import_fptype_getter()
+
 
 parser = argparse.ArgumentParser(description='daal4py ridge regression '
                                              'benchmark')
diff --git a/daal4py/svm.py b/daal4py/svm.py
@@ -4,14 +4,15 @@
 
 import argparse
 from bench import (
-    parse_args, measure_function_time, load_data, print_output, accuracy_score
+    parse_args, measure_function_time, load_data, print_output, accuracy_score,
+    import_fptype_getter
 )
 import numpy as np
-from daal4py import svm_training, svm_prediction, \
-                    kernel_function_linear, kernel_function_rbf, \
-                    multi_class_classifier_training, \
-                    multi_class_classifier_prediction
-from daal4py.sklearn.utils import getFPType
+from daal4py import (
+    svm_training, svm_prediction, kernel_function_linear, kernel_function_rbf,
+    multi_class_classifier_training, multi_class_classifier_prediction
+)
+getFPType = import_fptype_getter()
 
 
 def get_optimal_cache_size(n_rows, dtype=np.double, max_cache=64):
diff --git a/runner.py b/runner.py
@@ -274,7 +274,7 @@ class GenerationArgs:
                     stdout, stderr = read_output_from_command(command)
                     stderr = filter_stderr(stderr)
                     if args.output_format == 'json':
-                        json_result['results'] += json.loads(f'[{stdout}]')
+                        json_result['results'].extend(json.loads(stdout))
                     elif args.output_format == 'csv':
                         csv_result += stdout + '\n'
                     if stderr != '':
diff --git a/sklearn/bench.py b/sklearn/bench.py
diff --git a/sklearn/distances.py b/sklearn/distances.py
diff --git a/xgboost/bench.py b/xgboost/bench.py
