Skip to content

Commit 2bb768e

Browse files
bibikarbibikar
authored
Add daal4py benchmarks and update native, python benchmarks (#3)
- Universally read from .npy files as described in numpy's NEP-0001. note: The current reader `npyfile.h` used in native code does not really understand dtypes and will simply read the entire array into memory, relying on the caller's interpretation for this data. - Factor out all dataset generation into `make_datasets.py` - Move sklearn benches to `sklearn/` for clarity - Add daal4py benches for correlation/cosine distances, linear/ridge regression, kmeans, SVM - Add native/sklearn benchmarks for logistic regression with L_BFGS solver and RandomForest regression and classification - Move kmeans.predict benchmarks into kmeans bench files - Update thread setting to use daal4py API in sklearn/daal4py benches - Update license headers
1 parent dbabd89 commit 2bb768e

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

44 files changed

+2938
-762
lines changed

‎LICENSE

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
MIT License
22

3-
Copyright (c) 2017-2018 Intel Corporation
3+
Copyright (c) 2017-2019 Intel Corporation
44

55
Permission is hereby granted, free of charge, to any person obtaining a copy
66
of this software and associated documentation files (the "Software"), to deal

‎Makefile

Lines changed: 100 additions & 37 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,12 @@
11
# Sizes
22
DISTANCES_SIZE = 1000x15000
33
REGRESSION_SIZE = 1000000x50
4-
KMEANS_SIZE = $(REGRESSION_SIZE)
5-
SVM_VECTORS = 10000
4+
KMEANS_SAMPLES = 1000000
5+
KMEANS_FEATURES = 50
6+
KMEANS_SIZE = $(KMEANS_SAMPLES)x$(KMEANS_FEATURES)
7+
SVM_SAMPLES = 10000
68
SVM_FEATURES = 1000
7-
ITERATIONS = ?
9+
ITERATIONS = 10
810

911
# Bookkeeping options
1012
BATCH = $(shell date -Iseconds)
@@ -15,18 +17,21 @@ NUM_THREADS = -1
1517
SVM_NUM_THREADS = 0
1618
MULTIPLIER = 100
1719
DATA_DIR = data/
18-
KMEANS_DATA = $(addsuffix .csv,$(addprefix data/kmeans_,$(KMEANS_SIZE)))
20+
KMEANS_DATA = data/kmeans_$(KMEANS_SIZE).npy
1921

2022
comma = ,
2123

2224
ifneq ($(CONDA_PREFIX),)
23-
LD_LIBRARY_PATH := $(CONDA_PREFIX)/lib
25+
LD_LIBRARY_PATH := $(LD_LIBRARY_PATH):$(CONDA_PREFIX)/lib
2426
export LD_LIBRARY_PATH
2527
endif
2628

29+
export I_MPI_ROOT
2730

2831
all: native python
2932

33+
python: sklearn daal4py
34+
3035
native: data
3136
git submodule init && git submodule update
3237
@echo "# Compiling native benchmarks"
@@ -40,56 +45,114 @@ native: data
4045
$(NUM_THREADS) double $(REGRESSION_SIZE)
4146
native/bin/linear $(BATCH) $(HOST) native linear \
4247
$(NUM_THREADS) double $(REGRESSION_SIZE)
43-
native/bin/kmeans $(BATCH) $(HOST) native kmeans.fit \
44-
$(NUM_THREADS) double $(REGRESSION_SIZE) $(DATA_DIR)
45-
native/bin/kmeans_predict $(BATCH) $(HOST) native kmeans.predict \
46-
$(NUM_THREADS) double $(REGRESSION_SIZE) $(DATA_DIR) $(MULTIPLIER)
48+
native/bin/kmeans $(BATCH) $(HOST) native kmeans \
49+
$(NUM_THREADS) double $(KMEANS_SIZE) $(DATA_DIR) $(MULTIPLIER)
4750
native/bin/two_class_svm \
48-
--fileX data/two/X-$(SVM_VECTORS)x$(SVM_FEATURES).npy.csv \
49-
--fileY data/two/y-$(SVM_VECTORS)x$(SVM_FEATURES).npy.csv \
50-
--num-threads $(SVM_NUM_THREADS)
51+
--fileX data/two/X-$(SVM_SAMPLES)x$(SVM_FEATURES).npy \
52+
--fileY data/two/y-$(SVM_SAMPLES)x$(SVM_FEATURES).npy \
53+
--num-threads $(SVM_NUM_THREADS) --header
5154
native/bin/multi_class_svm \
52-
--fileX data/multi/X-$(SVM_VECTORS)x$(SVM_FEATURES).npy.csv \
53-
--fileY data/multi/y-$(SVM_VECTORS)x$(SVM_FEATURES).npy.csv \
54-
--num-threads $(SVM_NUM_THREADS)
55+
--fileX data/multi/X-$(SVM_SAMPLES)x$(SVM_FEATURES).npy \
56+
--fileY data/multi/y-$(SVM_SAMPLES)x$(SVM_FEATURES).npy \
57+
--num-threads $(SVM_NUM_THREADS) --header
58+
native/bin/log_reg_lbfgs \
59+
--fileX data/two/X-$(SVM_SAMPLES)x$(SVM_FEATURES).npy \
60+
--fileY data/two/y-$(SVM_SAMPLES)x$(SVM_FEATURES).npy \
61+
--num-threads $(SVM_NUM_THREADS) --header
62+
native/bin/log_reg_lbfgs \
63+
--fileX data/multi/X-$(SVM_SAMPLES)x$(SVM_FEATURES).npy \
64+
--fileY data/multi/y-$(SVM_SAMPLES)x$(SVM_FEATURES).npy \
65+
--num-threads $(SVM_NUM_THREADS) --header
66+
native/bin/decision_forest_clsf \
67+
--fileX data/two/X-$(SVM_SAMPLES)x$(SVM_FEATURES).npy \
68+
--fileY data/two/y-$(SVM_SAMPLES)x$(SVM_FEATURES).npy \
69+
--num-threads $(SVM_NUM_THREADS) --header
70+
native/bin/decision_forest_clsf \
71+
--fileX data/multi/X-$(SVM_SAMPLES)x$(SVM_FEATURES).npy \
72+
--fileY data/multi/y-$(SVM_SAMPLES)x$(SVM_FEATURES).npy \
73+
--num-threads $(SVM_NUM_THREADS) --header
5574

56-
python: data
57-
@echo "# Running python benchmarks"
58-
python python/distances.py --batchID $(BATCH) --arch $(HOST) \
75+
sklearn: data
76+
@echo "# Running scikit-learn benchmarks"
77+
python sklearn/distances.py --batchID $(BATCH) --arch $(HOST) \
5978
--prefix python --core-number $(NUM_THREADS) \
6079
--size $(subst x,$(comma),$(DISTANCES_SIZE)) --iteration $(ITERATIONS)
61-
python python/ridge.py --batchID $(BATCH) --arch $(HOST) \
80+
python sklearn/ridge.py --batchID $(BATCH) --arch $(HOST) \
6281
--prefix python --core-number $(NUM_THREADS) \
6382
--size $(subst x,$(comma),$(REGRESSION_SIZE)) --iteration $(ITERATIONS)
64-
python python/linear.py --batchID $(BATCH) --arch $(HOST) \
83+
python sklearn/linear.py --batchID $(BATCH) --arch $(HOST) \
6584
--prefix python --core-number $(NUM_THREADS) \
6685
--size $(subst x,$(comma),$(REGRESSION_SIZE)) --iteration $(ITERATIONS)
67-
python python/kmeans.py --batchID $(BATCH) --arch $(HOST) \
86+
python sklearn/kmeans.py --batchID $(BATCH) --arch $(HOST) \
6887
--prefix python --core-number $(NUM_THREADS) \
6988
--size $(subst x,$(comma),$(KMEANS_SIZE)) --iteration $(ITERATIONS) \
70-
--input $(DATA_DIR)
71-
python python/kmeans_predict.py --batchID $(BATCH) --arch $(HOST) \
89+
-x $(KMEANS_DATA) -i $(basename $(KMEANS_DATA)).init.npy
90+
python sklearn/svm_bench.py --core-number $(NUM_THREADS) \
91+
--fileX data/two/X-$(SVM_SAMPLES)x$(SVM_FEATURES).npy \
92+
--fileY data/two/y-$(SVM_SAMPLES)x$(SVM_FEATURES).npy \
93+
--header
94+
python sklearn/svm_bench.py --core-number $(NUM_THREADS) \
95+
--fileX data/multi/X-$(SVM_SAMPLES)x$(SVM_FEATURES).npy \
96+
--fileY data/multi/y-$(SVM_SAMPLES)x$(SVM_FEATURES).npy \
97+
--header
98+
python sklearn/log_reg.py --num-threads $(NUM_THREADS) \
99+
--fileX data/two/X-$(SVM_SAMPLES)x$(SVM_FEATURES).npy \
100+
--fileY data/two/y-$(SVM_SAMPLES)x$(SVM_FEATURES).npy \
101+
--header
102+
python sklearn/log_reg.py --num-threads $(NUM_THREADS) \
103+
--fileX data/multi/X-$(SVM_SAMPLES)x$(SVM_FEATURES).npy \
104+
--fileY data/multi/y-$(SVM_SAMPLES)x$(SVM_FEATURES).npy \
105+
--header
106+
python sklearn/df_clsf.py --num-threads $(NUM_THREADS) \
107+
--fileX data/two/X-$(SVM_SAMPLES)x$(SVM_FEATURES).npy \
108+
--fileY data/two/y-$(SVM_SAMPLES)x$(SVM_FEATURES).npy \
109+
--header
110+
python sklearn/df_clsf.py --num-threads $(NUM_THREADS) \
111+
--fileX data/multi/X-$(SVM_SAMPLES)x$(SVM_FEATURES).npy \
112+
--fileY data/multi/y-$(SVM_SAMPLES)x$(SVM_FEATURES).npy \
113+
--header
114+
115+
daal4py: data
116+
@echo "# Running daal4py benchmarks"
117+
python daal4py/distances.py --batchID $(BATCH) --arch $(HOST) \
118+
--prefix python --core-number $(NUM_THREADS) \
119+
--size $(subst x,$(comma),$(DISTANCES_SIZE)) --iteration $(ITERATIONS)
120+
python daal4py/ridge.py --batchID $(BATCH) --arch $(HOST) \
121+
--prefix python --core-number $(NUM_THREADS) \
122+
--size $(subst x,$(comma),$(REGRESSION_SIZE)) --iteration $(ITERATIONS)
123+
python daal4py/linear.py --batchID $(BATCH) --arch $(HOST) \
124+
--prefix python --core-number $(NUM_THREADS) \
125+
--size $(subst x,$(comma),$(REGRESSION_SIZE)) --iteration $(ITERATIONS)
126+
python daal4py/kmeans.py --batchID $(BATCH) --arch $(HOST) \
72127
--prefix python --core-number $(NUM_THREADS) \
73128
--size $(subst x,$(comma),$(KMEANS_SIZE)) --iteration $(ITERATIONS) \
74-
--input $(DATA_DIR) --data-multiplier $(MULTIPLIER)
75-
python python/svm_bench.py --core-number $(NUM_THREADS) \
76-
--fileX data/two/X-$(SVM_VECTORS)x$(SVM_FEATURES).npy \
77-
--fileY data/two/y-$(SVM_VECTORS)x$(SVM_FEATURES).npy
78-
python python/svm_bench.py --core-number $(NUM_THREADS) \
79-
--fileX data/multi/X-$(SVM_VECTORS)x$(SVM_FEATURES).npy \
80-
--fileY data/multi/y-$(SVM_VECTORS)x$(SVM_FEATURES).npy
129+
-x $(KMEANS_DATA) -i $(basename $(KMEANS_DATA)).init.npy
130+
python daal4py/svm_bench.py --core-number $(NUM_THREADS) \
131+
--fileX data/two/X-$(SVM_SAMPLES)x$(SVM_FEATURES).npy \
132+
--fileY data/two/y-$(SVM_SAMPLES)x$(SVM_FEATURES).npy \
133+
--header
134+
python daal4py/svm_bench.py --core-number $(NUM_THREADS) \
135+
--fileX data/multi/X-$(SVM_SAMPLES)x$(SVM_FEATURES).npy \
136+
--fileY data/multi/y-$(SVM_SAMPLES)x$(SVM_FEATURES).npy \
137+
--header
81138

82139
data: $(KMEANS_DATA) svm_data
83140

84141
$(KMEANS_DATA): | data/
85-
python python/kmeans_data.py --size \
86-
$(shell basename $@ .csv | cut -d _ -f 2) --fname $@ --clusters 10
142+
python make_datasets.py -f $(KMEANS_FEATURES) -s $(KMEANS_SAMPLES) \
143+
kmeans -c 10 -x $(basename $@) -i $(basename $@).init \
144+
-t $(basename $@).tol
145+
146+
svm_data: data/two/X-$(SVM_SAMPLES)x$(SVM_FEATURES).npy \
147+
data/multi/X-$(SVM_SAMPLES)x$(SVM_FEATURES).npy
87148

88-
svm_data: data/two/X-$(SVM_VECTORS)x$(SVM_FEATURES).npy.csv
149+
data/two/X-$(SVM_SAMPLES)x$(SVM_FEATURES).npy: | data/
150+
python make_datasets.py -f $(SVM_FEATURES) -s $(SVM_SAMPLES) \
151+
classification -c 2 -x $@ -y $(dir $@)/$(subst X-,y-,$(notdir $@))
89152

90-
data/two/X-$(SVM_VECTORS)x$(SVM_FEATURES).npy.csv: | data/
91-
python python/svm_data.py -v $(SVM_VECTORS) -f $(SVM_FEATURES)
92-
native/svm_native_data.sh
153+
data/multi/X-$(SVM_SAMPLES)x$(SVM_FEATURES).npy: | data/
154+
python make_datasets.py -f $(SVM_FEATURES) -s $(SVM_SAMPLES) \
155+
classification -c 5 -x $@ -y $(dir $@)/$(subst X-,y-,$(notdir $@))
93156

94157
data/:
95158
mkdir -p data/
@@ -100,4 +163,4 @@ clean:
100163
$(MAKE) -C native clean
101164
rm -rf data
102165

103-
.PHONY: native python all clean native_data data kmeans_data svm_data
166+
.PHONY: native python sklearn daal4py all clean native_data data kmeans_data svm_data

‎README.md

Lines changed: 1 addition & 74 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
# scikit-learn_bench
22

3-
Benchmark for optimizations to scikit-learn in the Intel Distribution for
3+
Benchmark for optimizations to scikit-learn in the Intel(R) Distribution for
44
Python*
55

66
## Prerequisites
@@ -13,76 +13,3 @@ Python*
1313
- To run only native benchmarks, use `make native`.
1414
- If you have activated a conda environment, the build will use daal from
1515
the conda environment, if available.
16-
17-
## Manually
18-
19-
### Build
20-
21-
- `git submodule init && git submodule update` (for native versions only)
22-
- To build native versions, run `make -C native`.
23-
- Prepare data for KMeans benchmarks by running
24-
`mkdir -p data && python python/kmeans_data.py --size 1000000x50 --fname data/kmeans_1000000x50.csv --clusters 10`
25-
- Size can be adjusted. Example sizes are `500000x5`, `500000x25`, `1000000x50`.
26-
- Prepare data for SVM benchmarks by running
27-
`python python/svm_data.py -v 10000 -f 1000`.
28-
- Number of vectors can be specified with `-v`, and number of features
29-
can be specified with `-f`. Data will by default go in `data/two` and `data/multi`.
30-
- Prepare data for native benchmarks by running `native/svm_native_data.sh`.
31-
32-
### Run
33-
- All benchmarks must be given the number of threads to run. If this value
34-
is `-1`, then the number of processing threads will equal the number of
35-
CPUs available on the system. Otherwise, the benchmark will use the given
36-
number of threads.
37-
- For KMeans benchmarks, an input directory must be specified.
38-
The slash must be included at the end of the directory name.
39-
- For all benchmarks, the `batch`, `hostname`, and `env_name`
40-
arguments are only for bookkeeping purposes and can be replaced with
41-
placeholders.
42-
- The KMeans predict benchmark has a multiplier argument. An example value
43-
is 100.
44-
45-
#### Python benchmarks
46-
- Python benchmarks are located in the `python` directory
47-
`python python/<benchmark>.py <args...>`
48-
- The following benchmarks are available:
49-
- `distances`: benchmark pairwise distances using `cosine` and `correlation`
50-
metrics
51-
- `ridge`: benchmark ridge regression fit and prediction
52-
- `linear`: benchmark linear regression fit and prediction
53-
- `kmeans`: benchmark KMeans fit
54-
- `kmeans_predict`: benchmark KMeans predict
55-
- `svm_bench`: benchmark two- and multi-class SVM
56-
- A size must be passed in the form `--size M,N` for all benchmarks except SVM
57-
- The number of threads to run must be passed in the form `--core-number T`
58-
for all benchmarks.
59-
- For KMeans benchmarks, the input directory must be passed in the form
60-
`--input INPUT_DIR`.
61-
- For SVM benchmarks, the input files must be passed in the form
62-
`--fileX FILE_X --fileY FILE_Y`.
63-
- For the KMeans predict benchmark, the multiplier must be passed in the form
64-
`--data-multiplier X`.
65-
66-
#### Native benchmarks
67-
- Binaries are located in the `native/bin` directory.
68-
- Sizes must be specified in `MxN` form.
69-
- The following benchmarks are available:
70-
- `cosine <batch> <hostname> <env_name> cosine <threads> double <size>`:
71-
benchmark pairwise distances using `cosine` metric
72-
- `correlation <batch> <hostname> <env_name> cosine <threads> double <size>`:
73-
benchmark pairwise distances using `correlation` metric
74-
- `ridge <batch> <hostname> <env_name> cosine <threads> double <size>`:
75-
benchmark ridge regression fit and prediction
76-
- `linear <batch> <hostname> <env_name> cosine <threads> double <size>`:
77-
benchmark linear regression fit and prediction
78-
- `kmeans <batch> <hostname> <env_name> cosine <threads> double <size> <input_dir>`:
79-
benchmark KMeans fit, finding pregenerated input files in `input_dir`
80-
- `kmeans_predict <batch> <hostname> <env_name> cosine <threads> double <size> <input_dir> <multiplier>`
81-
benchmark KMeans fit, finding pregenerated input files in `input_dir`.
82-
A possible value for `multiplier` is 100.
83-
- `{two,multi}_class_svm --fileX <feature-file> --fileY <label-file> --num-threads <threads>`:
84-
benchmark two/multi class SVM, using pregenerated feature and label file and using the given
85-
number of threads
86-
87-
## See also
88-
"[Accelerating Scientific Python with Intel Optimizations](http://conference.scipy.org/proceedings/scipy2017/pdfs/oleksandr_pavlyk.pdf)" by Oleksandr Pavlyk, Denis Nagorny, Andres Guzman-Ballen, Anton Malakhov, Hai Liu, Ehsan Totoni, Todd A. Anderson, Sergey Maidanov. Proceedings of the 16th Python in Science Conference (SciPy 2017), July 10 - July 16, Austin, Texas

‎daal4py/args.py

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
# Copyright (C) 2017-2019 Intel Corporation
2+
#
3+
# SPDX-License-Identifier: MIT
4+
5+
6+
def getArguments(argParser):
7+
argParser.add_argument('--iteration', default=10, type=int,
8+
help='Number of repetitions to run')
9+
argParser.add_argument('--num-threads', '--core-number', default=-1,
10+
type=int, help='Number of threads to use')
11+
argParser.add_argument('--arch', default='?',
12+
help='Machine architecture, for bookkeeping')
13+
argParser.add_argument('--batchID', default='?',
14+
help='Batch ID, for bookkeeping')
15+
argParser.add_argument('--prefix', default='?',
16+
help='Prefix string, for bookkeeping')
17+
argParser.add_argument('--place', default='?', help="prefix string")
18+
argParser.add_argument('--cache', default='?', help="cached/non-cached")
19+
argParser.add_argument('--size', default='?',
20+
help="array size, delimited by comma or 'x'")
21+
args = argParser.parse_args()
22+
23+
args.size = [int(n) for n in args.size.replace('x', ',').split(',')]
24+
return args
25+
26+
27+
def coreString(num):
28+
return 'Serial' if num == 1 else 'Threaded'

‎daal4py/bench.py

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
# Copyright (C) 2018-2019 Intel Corporation
2+
#
3+
# SPDX-License-Identifier: MIT
4+
5+
6+
def set_daal_num_threads(num_threads):
7+
try:
8+
import daal4py
9+
if num_threads:
10+
daal4py.daalinit(nthreads=num_threads)
11+
except ImportError:
12+
print("@ Package 'daal4py' was not found. Number of threads is being ignored")
13+
14+
15+
def prepare_benchmark(args):
16+
try:
17+
if args.num_threads > 0:
18+
set_daal_num_threads(args.num_threads)
19+
num_threads = args.num_threads
20+
import daal4py
21+
daal_version = daal4py.__daal_run_version__
22+
except ImportError:
23+
num_threads = 1
24+
daal_version = None
25+
26+
return num_threads, daal_version
27+

‎daal4py/distances.py

Lines changed: 52 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,52 @@
1+
# Copyright (C) 2017-2019 Intel Corporation
2+
#
3+
# SPDX-License-Identifier: MIT
4+
5+
from __future__ import print_function
6+
import numpy as np
7+
import timeit
8+
from numpy.random import rand
9+
from daal4py import correlation_distance, cosine_distance, daalinit
10+
from args import getArguments, coreString
11+
from bench import prepare_benchmark
12+
13+
import argparse
14+
argParser = argparse.ArgumentParser(prog="pairwise_distances.py",
15+
description="sklearn pairwise_distances benchmark",
16+
formatter_class=argparse.ArgumentDefaultsHelpFormatter)
17+
18+
args = getArguments(argParser)
19+
REP = args.iteration if args.iteration != '?' else 10
20+
core_number, daal_version = prepare_benchmark(args)
21+
22+
23+
def st_time(func):
24+
def st_func(*args, **keyArgs):
25+
times = []
26+
for n in range(REP):
27+
t1 = timeit.default_timer()
28+
r = func(*args, **keyArgs)
29+
t2 = timeit.default_timer()
30+
times.append(t2-t1)
31+
print(min(times))
32+
return r
33+
return st_func
34+
35+
p = args.size[0]
36+
n = args.size[1]
37+
38+
39+
X = rand(p,n)
40+
41+
42+
@st_time
43+
def cosine(X):
44+
cos_dist = cosine_distance().compute(X)
45+
@st_time
46+
def correlation(X):
47+
cor_dist = correlation_distance().compute(X)
48+
49+
print (','.join([args.batchID, args.arch, args.prefix, "Cosine", coreString(args.num_threads), "Double", "%sx%s" % (p,n)]), end=',')
50+
cosine(X)
51+
print (','.join([args.batchID, args.arch, args.prefix, "Correlation", coreString(args.num_threads), "Double", "%sx%s" % (p,n)]), end=',')
52+
correlation(X)

0 commit comments

Comments
 (0)