Skip to content

Commit da7f425

Browse files
md-shafiul-alammd-shafiul-alam
committed
online vs spmd
1 parent 574ff2a commit da7f425

File tree

3 files changed

+210
-0
lines changed

3 files changed

+210
-0
lines changed

‎configs/spmd/large_scale/large_scale.json

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,24 @@
3636
"mpi_params": {"n": [1,2,3,4,5,6,7,8,9,10,11,12], "ppn": 12, "-hostfile": "", "-cpu-bind=list:0-7,104-111:8-15,112-119:16-23,120-127:24-31,128-135:32-39,136-143:40-47,144-151:52-59,156-163:60-67,164-171:68-75,172-179:76-83,180-187:84-91,188-195:92-99,196-203": "--envall gpu_tile_compact.sh" }
3737
}
3838
},
39+
"large scale full one node parameters": {
40+
"data": {
41+
"dtype": "float64",
42+
"distributed_split": "None"
43+
},
44+
"bench": {
45+
"mpi_params": {"n": [12], "ppn": 12, "-hostfile": "", "-cpu-bind=list:0-7,104-111:8-15,112-119:16-23,120-127:24-31,128-135:32-39,136-143:40-47,144-151:52-59,156-163:60-67,164-171:68-75,172-179:76-83,180-187:84-91,188-195:92-99,196-203": "--envall gpu_tile_compact.sh" }
46+
}
47+
},
48+
"large scale strong full one node parameters": {
49+
"data": {
50+
"dtype": "float64",
51+
"distributed_split": "rank_based"
52+
},
53+
"bench": {
54+
"mpi_params": {"n": [12], "ppn": 12, "-hostfile": "", "-cpu-bind=list:0-7,104-111:8-15,112-119:16-23,120-127:24-31,128-135:32-39,136-143:40-47,144-151:52-59,156-163:60-67,164-171:68-75,172-179:76-83,180-187:84-91,188-195:92-99,196-203": "--envall gpu_tile_compact.sh" }
55+
}
56+
},
3957
"large scale 2k parameters": {
4058
"data": {
4159
"dtype": "float64",

‎configs/spmd/large_scale/spmd_for_online.json

Lines changed: 96 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,96 @@
1+
{
2+
"INCLUDE": ["../../common/sklearn.json", "large_scale.json"],
3+
"PARAMETERS_SETS": {
4+
"covariance data": {
5+
"data": [
6+
{
7+
"source": "make_blobs",
8+
"generation_kwargs": {
9+
"centers": 1,
10+
"n_samples": 100000,
11+
"n_features": [10, 100]
12+
},
13+
"split_kwargs": {"ignore": true}
14+
}
15+
]
16+
},
17+
"basic_statistics data": {
18+
"data": {
19+
"source": "make_blobs",
20+
"generation_kwargs": {
21+
"centers": 1,
22+
"n_samples": 100000,
23+
"n_features": [10, 100]
24+
},
25+
"split_kwargs": {"ignore": true}
26+
}
27+
},
28+
"linear_regression data": {
29+
"data": {
30+
"source": "make_regression",
31+
"split_kwargs": {"train_size": 0.2, "test_size": 0.8},
32+
"generation_kwargs": {
33+
"n_samples": 100000,
34+
"n_features": [10, 100],
35+
"n_informative": 5,
36+
"noise": 2.0
37+
}
38+
}
39+
},
40+
"pca data": {
41+
"data": {
42+
"source": "make_blobs",
43+
"generation_kwargs": {
44+
"centers": 1,
45+
"n_samples": 100000,
46+
"n_features": [10, 100]
47+
},
48+
"split_kwargs": {"ignore": true}
49+
}
50+
},
51+
"covariance": {
52+
"algorithm": [
53+
{
54+
"estimator": "EmpiricalCovariance",
55+
"library": "sklearnex.covariance",
56+
"estimator_methods": {"training": "fit"}
57+
}
58+
]
59+
},
60+
"basic_statistics": {
61+
"algorithm": [
62+
{
63+
"estimator": "BasicStatistics",
64+
"library": "sklearnex.basic_statistics",
65+
"estimator_methods": {"training": "fit"}
66+
}
67+
]
68+
},
69+
"linear_regression": {
70+
"algorithm": [
71+
{
72+
"estimator": "LinearRegression",
73+
"library": "sklearnex.linear_model",
74+
"estimator_methods": {"training": "fit"}
75+
}
76+
]
77+
},
78+
"pca": {
79+
"algorithm": [
80+
{
81+
"estimator": "IncrementalPCA",
82+
"library": "sklearnex.decomposition",
83+
"estimator_methods": {"training": "fit"}
84+
}
85+
]
86+
}
87+
},
88+
"TEMPLATES": {
89+
"basic_statistics": {"SETS": ["basic_statistics", "basic_statistics data", "large scale strong full one node parameters", "sklearnex spmd implementation"]},
90+
"covariance": {"SETS": ["covariance", "covariance data", "large scale strong full one node parameters", "sklearnex spmd implementation"]},
91+
"linear_regression": {
92+
"SETS": ["linear_regression", "linear_regression data", "large scale strong full one node parameters", "sklearnex spmd implementation"]
93+
},
94+
"pca": {"SETS": ["pca", "pca data", "large scale strong full one node parameters", "sklearnex spmd implementation"]}
95+
}
96+
}

‎configs/spmd/large_scale/spmd_for_online_strong.json

Lines changed: 96 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,96 @@
1+
{
2+
"INCLUDE": ["../../common/sklearn.json", "large_scale.json"],
3+
"PARAMETERS_SETS": {
4+
"covariance data": {
5+
"data": [
6+
{
7+
"source": "make_blobs",
8+
"generation_kwargs": {
9+
"centers": 1,
10+
"n_samples": 1200000,
11+
"n_features": [10, 100]
12+
},
13+
"split_kwargs": {"ignore": true}
14+
}
15+
]
16+
},
17+
"basic_statistics data": {
18+
"data": {
19+
"source": "make_blobs",
20+
"generation_kwargs": {
21+
"centers": 1,
22+
"n_samples": 1200000,
23+
"n_features": [10, 100]
24+
},
25+
"split_kwargs": {"ignore": true}
26+
}
27+
},
28+
"linear_regression data": {
29+
"data": {
30+
"source": "make_regression",
31+
"split_kwargs": {"train_size": 0.2, "test_size": 0.8},
32+
"generation_kwargs": {
33+
"n_samples": 1200000,
34+
"n_features": [10, 100],
35+
"n_informative": 5,
36+
"noise": 2.0
37+
}
38+
}
39+
},
40+
"pca data": {
41+
"data": {
42+
"source": "make_blobs",
43+
"generation_kwargs": {
44+
"centers": 1,
45+
"n_samples": 1200000,
46+
"n_features": [10, 100]
47+
},
48+
"split_kwargs": {"ignore": true}
49+
}
50+
},
51+
"covariance": {
52+
"algorithm": [
53+
{
54+
"estimator": "EmpiricalCovariance",
55+
"library": "sklearnex.covariance",
56+
"estimator_methods": {"training": "fit"}
57+
}
58+
]
59+
},
60+
"basic_statistics": {
61+
"algorithm": [
62+
{
63+
"estimator": "BasicStatistics",
64+
"library": "sklearnex.basic_statistics",
65+
"estimator_methods": {"training": "fit"}
66+
}
67+
]
68+
},
69+
"linear_regression": {
70+
"algorithm": [
71+
{
72+
"estimator": "LinearRegression",
73+
"library": "sklearnex.linear_model",
74+
"estimator_methods": {"training": "fit"}
75+
}
76+
]
77+
},
78+
"pca": {
79+
"algorithm": [
80+
{
81+
"estimator": "IncrementalPCA",
82+
"library": "sklearnex.decomposition",
83+
"estimator_methods": {"training": "fit"}
84+
}
85+
]
86+
}
87+
},
88+
"TEMPLATES": {
89+
"basic_statistics": {"SETS": ["basic_statistics", "basic_statistics data", "large scale strong full one node parameters", "sklearnex spmd implementation"]},
90+
"covariance": {"SETS": ["covariance", "covariance data", "large scale strong full one node parameters", "sklearnex spmd implementation"]},
91+
"linear_regression": {
92+
"SETS": ["linear_regression", "linear_regression data", "large scale strong full one node parameters", "sklearnex spmd implementation"]
93+
},
94+
"pca": {"SETS": ["pca", "pca data", "large scale strong full one node parameters", "sklearnex spmd implementation"]}
95+
}
96+
}

0 commit comments

Comments
 (0)