Skip to content

Commit de3ce51

Browse files
bibikarbibikar
authored
Implement "target time" for benchmarks (#17)
On certain benchmarks, the predict method completes orders of magnitude faster than the fit method. Instead of requiring the user to manually select number of repetitions to run, this PR implements aiming for a certain target total time instead. This should help with measurement stability in these benchmarks. The timing function first takes some "warmup" measurements and determines a good number of inner loops such that the total timing meets the overall time limit and includes about --goal-outer-loops outer loops. If the warmup measurement time exceeds the time limit, the timing function just returns the warmup time. This PR also includes some refactoring of native benchmarks to use common functions for counting classes, printing tables, and timing. One other change made here is to use tol=0 in all kmeans benchmarks unless otherwise specified on the command-line. This removes randomness in number of iterations in K-Means fit.
1 parent 57a4f21 commit de3ce51

31 files changed

+953
-950
lines changed

‎Makefile

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -77,8 +77,7 @@ ARGS_NATIVE_pca_full = --num-threads "$(NUM_THREADS)" --header \
7777
ARGS_NATIVE_kmeans = --num-threads "$(NUM_THREADS)" --header \
7878
--data-multiplier "$(MULTIPLIER)" \
7979
--filex data/kmeans_$(KMEANS_SIZE).npy \
80-
--filei data/kmeans_$(KMEANS_SIZE).init.npy \
81-
--filet data/kmeans_$(KMEANS_SIZE).tol.npy
80+
--filei data/kmeans_$(KMEANS_SIZE).init.npy
8281
ARGS_NATIVE_svm2 = --fileX data/two/X-$(SVM_SIZE).npy \
8382
--fileY data/two/y-$(SVM_SIZE).npy \
8483
--num-threads $(SVM_NUM_THREADS) --header
@@ -159,8 +158,7 @@ ARGS_DAAL4PY_pca_daal = --size "$(REGRESSION_SIZE)" --svd-solver daal
159158
ARGS_DAAL4PY_pca_full = --size "$(REGRESSION_SIZE)" --svd-solver full
160159
ARGS_DAAL4PY_kmeans = --data-multiplier "$(MULTIPLIER)" \
161160
--filex data/kmeans_$(KMEANS_SIZE).npy \
162-
--filei data/kmeans_$(KMEANS_SIZE).init.npy \
163-
--filet data/kmeans_$(KMEANS_SIZE).tol.npy
161+
--filei data/kmeans_$(KMEANS_SIZE).init.npy
164162
ARGS_DAAL4PY_svm2 = --fileX data/two/X-$(SVM_SIZE).npy \
165163
--fileY data/two/y-$(SVM_SIZE).npy
166164
ARGS_DAAL4PY_svm5 = --fileX data/multi/X-$(SVM_SIZE).npy \

‎daal4py/bench.py

Lines changed: 82 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -93,12 +93,23 @@ def parse_args(parser, size=None, dtypes=None, loop_types=(),
9393
else:
9494
loop_dash = loop_for = ''
9595

96-
parser.add_argument(f'--{loop_dash}inner-loops', default=3, type=int,
97-
help=f'Number of inner loop iterations {loop_for}'
96+
parser.add_argument(f'--{loop_dash}inner-loops', default=100, type=int,
97+
help=f'Maximum inner loop iterations {loop_for}'
9898
f'(we take the mean over inner iterations)')
99-
parser.add_argument(f'--{loop_dash}outer-loops', default=1, type=int,
100-
help=f'Number of outer loop iterations {loop_for}'
99+
parser.add_argument(f'--{loop_dash}outer-loops', default=100, type=int,
100+
help=f'Maximum outer loop iterations {loop_for}'
101101
f'(we take the min over outer iterations)')
102+
parser.add_argument(f'--{loop_dash}time-limit', default=10.,
103+
type=float,
104+
help=f'Target time to spend to benchmark '
105+
f'{loop_for}')
106+
parser.add_argument(f'--{loop_dash}goal-outer-loops', default=10,
107+
type=int,
108+
dest=f'{loop_dash.replace("-", "_")}goal',
109+
help=f'Number of outer loops to aim {loop_for} '
110+
f'while automatically picking number of '
111+
f'inner loops. If zero, do not automatically '
112+
f'decide number of inner loops.')
102113

103114
params = parser.parse_args()
104115

@@ -186,7 +197,8 @@ def prepare_daal(num_threads=-1):
186197
return num_threads, daal_version
187198

188199

189-
def time_mean_min(func, *args, inner_loops=1, outer_loops=1, **kwargs):
200+
def time_mean_min(func, *args, inner_loops=1, outer_loops=1, time_limit=10.,
201+
goal_outer_loops=10, verbose=False, **kwargs):
190202
'''
191203
Time the given function (inner_loops * outer_loops) times, returning the
192204
min of the inner loop means.
@@ -196,9 +208,18 @@ def time_mean_min(func, *args, inner_loops=1, outer_loops=1, **kwargs):
196208
func : callable f(*args, **kwargs)
197209
The function to time.
198210
inner_loops : int
199-
Number of inner loop iterations to take the mean over.
211+
Maximum number of inner loop iterations to take the mean over.
200212
outer_loops : int
201-
Number of outer loop iterations to take the min over.
213+
Maximum number of outer loop iterations to take the min over.
214+
time_limit : double
215+
Number of seconds to aim for. If accumulated time exceeds time_limit
216+
in outer loops, exit without running more outer loops. If zero,
217+
disable time limit.
218+
goal_outer_loops : int
219+
Number of outer loop iterations to aim for by taking warmup rounds
220+
and tuning inner_loops automatically.
221+
verbose : boolean
222+
If True, print outer loop timings and miscellaneous information.
202223
203224
Returns
204225
-------
@@ -212,23 +233,71 @@ def time_mean_min(func, *args, inner_loops=1, outer_loops=1, **kwargs):
212233
'Must time the function at least once'
213234

214235
times = np.zeros(outer_loops, dtype='f8')
236+
total_time = 0.
215237

216-
for i in range(outer_loops):
217-
218-
t0 = timeit.default_timer()
238+
# Warm-up iterations to determine optimal inner_loops
239+
warmup = (goal_outer_loops > 0)
240+
warmup_time = 0.
241+
last_warmup = 0.
242+
if warmup:
219243
for _ in range(inner_loops):
244+
t0 = timeit.default_timer()
220245
val = func(*args, **kwargs)
221-
t1 = timeit.default_timer()
222-
223-
times[i] = t1 - t0
246+
t1 = timeit.default_timer()
247+
248+
last_warmup = t1 - t0
249+
warmup_time += last_warmup
250+
if warmup_time > time_limit / 10:
251+
break
252+
253+
inner_loops = max(1, int(time_limit / last_warmup / goal_outer_loops))
254+
logverbose(f'Optimal inner loops = {inner_loops}', verbose)
255+
256+
if last_warmup > time_limit:
257+
# If we took too much time in warm-up, just use those numbers
258+
logverbose(f'A single warmup iteration took {last_warmup:0.2f}s '
259+
f'> {time_limit:0.2f}s - not performing any more timings',
260+
verbose)
261+
outer_loops = 1
262+
inner_loops = 1
263+
times[0] = last_warmup
264+
times = times[:1]
265+
else:
266+
# Otherwise, actually take the timing
267+
for i in range(outer_loops):
268+
269+
t0 = timeit.default_timer()
270+
for _ in range(inner_loops):
271+
val = func(*args, **kwargs)
272+
t1 = timeit.default_timer()
273+
274+
times[i] = t1 - t0
275+
total_time += times[i]
276+
277+
if time_limit > 0 and total_time > time_limit:
278+
logverbose(f'TT={total_time:0.2f}s exceeding {time_limit}s '
279+
f'after iteration {i+1}', verbose)
280+
outer_loops = i + 1
281+
times = times[:outer_loops]
282+
break
224283

225284
# We take the mean of inner loop times
226285
times /= inner_loops
286+
logverbose('Mean times [s]', verbose)
287+
logverbose(f'{times}', verbose)
227288

228289
# We take the min of outer loop times
229290
return np.min(times), val
230291

231292

293+
def logverbose(msg, verbose):
294+
'''
295+
Print msg as a verbose logging message only if verbose is True
296+
'''
297+
if verbose:
298+
print('@', msg)
299+
300+
232301
def accuracy_score(y, yp):
233302
return np.mean(y == yp)
234303

‎daal4py/df_clsf.py

Lines changed: 8 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -103,16 +103,20 @@ def df_clsf_predict(X, training_result, n_classes, verbose=False):
103103
seed=params.seed,
104104
n_features_per_node=params.max_features,
105105
max_depth=params.max_depth,
106-
verbose=params.verbose,
107106
outer_loops=params.fit_outer_loops,
108-
inner_loops=params.fit_inner_loops)
107+
inner_loops=params.fit_inner_loops,
108+
goal_outer_loops=params.fit_goal,
109+
time_limit=params.fit_time_limit,
110+
verbose=params.verbose)
109111
print_row(columns, params, function='df_clsf.fit', time=fit_time)
110112

111113
predict_time, yp = time_mean_min(df_clsf_predict, X, res,
112114
params.n_classes,
113-
verbose=params.verbose,
114115
outer_loops=params.predict_outer_loops,
115-
inner_loops=params.predict_inner_loops)
116+
inner_loops=params.predict_inner_loops,
117+
goal_outer_loops=params.predict_goal,
118+
time_limit=params.predict_time_limit,
119+
verbose=params.verbose)
116120
acc = 100 * accuracy_score(yp, y)
117121
print_row(columns, params, function='df_clsf.predict', time=predict_time,
118122
accuracy=acc)

‎daal4py/df_regr.py

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -99,10 +99,16 @@ def df_regr_predict(X, training_result):
9999
n_features_per_node=params.max_features,
100100
max_depth=params.max_depth,
101101
outer_loops=params.fit_outer_loops,
102-
inner_loops=params.fit_inner_loops)
102+
inner_loops=params.fit_inner_loops,
103+
goal_outer_loops=params.fit_goal,
104+
time_limit=params.fit_time_limit,
105+
verbose=params.verbose)
103106
print_row(columns, params, function='df_regr.fit', time=fit_time)
104107

105108
predict_time, yp = time_mean_min(df_regr_predict, X, res,
106109
outer_loops=params.predict_outer_loops,
107-
inner_loops=params.predict_inner_loops)
110+
inner_loops=params.predict_inner_loops,
111+
goal_outer_loops=params.predict_goal,
112+
time_limit=params.predict_time_limit,
113+
verbose=params.verbose)
108114
print_row(columns, params, function='df_regr.predict', time=predict_time)

‎daal4py/distances.py

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -32,5 +32,8 @@ def test_distances(pairwise_distances, X):
3232

3333
time, _ = time_mean_min(test_distances, pairwise_distances, X,
3434
outer_loops=params.outer_loops,
35-
inner_loops=params.inner_loops)
35+
inner_loops=params.inner_loops,
36+
goal_outer_loops=params.goal,
37+
time_limit=params.time_limit,
38+
verbose=params.verbose)
3639
print_row(columns, params, function=metric.capitalize(), time=time)

‎daal4py/kmeans.py

Lines changed: 11 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -14,8 +14,8 @@
1414
type=str, help='Points to cluster')
1515
parser.add_argument('-i', '--filei', '--fileI', '--init', required=True,
1616
type=str, help='Initial clusters')
17-
parser.add_argument('-t', '--filet', '--fileT', '--tol', required=True,
18-
type=str, help='Absolute threshold')
17+
parser.add_argument('-t', '--tol', default=0., type=float,
18+
help='Absolute threshold')
1919
parser.add_argument('-m', '--data-multiplier', default=100,
2020
type=int, help='Data multiplier')
2121
parser.add_argument('--maxiter', type=int, default=100,
@@ -26,7 +26,6 @@
2626
X = np.load(params.filex)
2727
X_init = np.load(params.filei)
2828
X_mult = np.vstack((X,) * params.data_multiplier)
29-
tol = np.load(params.filet)
3029

3130
params.size = size_str(X.shape)
3231
params.n_clusters = X_init.shape[0]
@@ -40,7 +39,7 @@ def test_fit(X, X_init):
4039
nClusters=params.n_clusters,
4140
maxIterations=params.maxiter,
4241
assignFlag=True,
43-
accuracyThreshold=tol
42+
accuracyThreshold=params.tol
4443
)
4544
return algorithm.compute(X, X_init)
4645

@@ -63,11 +62,17 @@ def test_predict(X, X_init):
6362
# Time fit
6463
fit_time, _ = time_mean_min(test_fit, X, X_init,
6564
outer_loops=params.fit_outer_loops,
66-
inner_loops=params.fit_inner_loops)
65+
inner_loops=params.fit_inner_loops,
66+
goal_outer_loops=params.fit_goal,
67+
time_limit=params.fit_time_limit,
68+
verbose=params.verbose)
6769
print_row(columns, params, function='KMeans.fit', time=fit_time)
6870

6971
# Time predict
7072
predict_time, _ = time_mean_min(test_predict, X, X_init,
7173
outer_loops=params.predict_outer_loops,
72-
inner_loops=params.predict_inner_loops)
74+
inner_loops=params.predict_inner_loops,
75+
goal_outer_loops=params.predict_goal,
76+
time_limit=params.predict_time_limit,
77+
verbose=params.verbose)
7378
print_row(columns, params, function='KMeans.predict', time=predict_time)

‎daal4py/linear.py

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -47,11 +47,17 @@ def test_predict(Xp, model):
4747
# Time fit
4848
fit_time, res = time_mean_min(test_fit, X, y,
4949
outer_loops=params.fit_outer_loops,
50-
inner_loops=params.fit_inner_loops)
50+
inner_loops=params.fit_inner_loops,
51+
goal_outer_loops=params.fit_goal,
52+
time_limit=params.fit_time_limit,
53+
verbose=params.verbose)
5154
print_row(columns, params, function='Linear.fit', time=fit_time)
5255

5356
# Time predict
5457
predict_time, yp = time_mean_min(test_predict, Xp, res.model,
5558
outer_loops=params.predict_outer_loops,
56-
inner_loops=params.predict_inner_loops)
59+
inner_loops=params.predict_inner_loops,
60+
goal_outer_loops=params.predict_goal,
61+
time_limit=params.predict_time_limit,
62+
verbose=params.verbose)
5763
print_row(columns, params, function='Linear.predict', time=predict_time)

‎daal4py/log_reg.py

Lines changed: 8 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -222,13 +222,15 @@ def test_predict(X, beta, intercept=0, multi_class='ovr'):
222222

223223
# Time fit and predict
224224
fit_time, res = time_mean_min(test_fit, X, y, penalty='l2', C=params.C,
225-
verbose=params.verbose,
226225
fit_intercept=params.fit_intercept,
227226
tol=params.tol,
228227
max_iter=params.maxiter,
229228
solver=params.solver,
230229
outer_loops=params.fit_outer_loops,
231-
inner_loops=params.fit_inner_loops)
230+
inner_loops=params.fit_inner_loops,
231+
goal_outer_loops=params.fit_goal,
232+
time_limit=params.fit_time_limit,
233+
verbose=params.verbose)
232234

233235
beta, intercept, solver_result, params.multiclass = res
234236
print_row(columns, params, function='LogReg.fit', time=fit_time)
@@ -237,7 +239,10 @@ def test_predict(X, beta, intercept=0, multi_class='ovr'):
237239
intercept=intercept,
238240
multi_class=params.multiclass,
239241
outer_loops=params.predict_outer_loops,
240-
inner_loops=params.predict_inner_loops)
242+
inner_loops=params.predict_inner_loops,
243+
goal_outer_loops=params.predict_goal,
244+
time_limit=params.predict_time_limit,
245+
verbose=params.verbose)
241246
y_pred = np.argmax(yp, axis=1)
242247
acc = 100 * accuracy_score(y_pred, y)
243248
print_row(columns, params, function='LogReg.predict', time=predict_time,

‎daal4py/pca.py

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -123,13 +123,19 @@ def test_transform(Xp, pca_result, eigenvalues, eigenvectors):
123123
# Time fit
124124
fit_time, res = time_mean_min(test_fit, X,
125125
outer_loops=params.fit_outer_loops,
126-
inner_loops=params.fit_inner_loops)
126+
inner_loops=params.fit_inner_loops,
127+
goal_outer_loops=params.fit_goal,
128+
time_limit=params.fit_time_limit,
129+
verbose=params.verbose)
127130
print_row(columns, params, function='PCA.fit', time=fit_time)
128131

129132
# Time transform
130133
transform_time, tr = time_mean_min(test_transform, Xp, *res[:3],
131134
outer_loops=params.transform_outer_loops,
132-
inner_loops=params.transform_inner_loops)
135+
inner_loops=params.transform_inner_loops,
136+
goal_outer_loops=params.transform_goal,
137+
time_limit=params.transform_time_limit,
138+
verbose=params.verbose)
133139
print_row(columns, params, function='PCA.transform', time=transform_time)
134140

135141
if params.write_results:

‎daal4py/ridge.py

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -41,11 +41,17 @@ def test_predict(Xp, model):
4141
# Time fit
4242
fit_time, res = time_mean_min(test_fit, X, y,
4343
outer_loops=params.fit_outer_loops,
44-
inner_loops=params.fit_inner_loops)
44+
inner_loops=params.fit_inner_loops,
45+
goal_outer_loops=params.fit_goal,
46+
time_limit=params.fit_time_limit,
47+
verbose=params.verbose)
4548
print_row(columns, params, function='Ridge.fit', time=fit_time)
4649

4750
# Time predict
4851
predict_time, yp = time_mean_min(test_predict, Xp, res.model,
4952
outer_loops=params.predict_outer_loops,
50-
inner_loops=params.predict_inner_loops)
53+
inner_loops=params.predict_inner_loops,
54+
goal_outer_loops=params.predict_goal,
55+
time_limit=params.predict_time_limit,
56+
verbose=params.verbose)
5157
print_row(columns, params, function='Ridge.predict', time=predict_time)

0 commit comments

Comments
 (0)