Skip to content

Commit 56c7823

Browse files
mboehm7mboehm7
committed
[SYSTEMDS-3847] Fix perftest refactoring (datagen scripts)
1 parent 48e6f07 commit 56c7823

31 files changed

+2750
-1
lines changed

‎scripts/perftest/datagen/genALSData.sh‎

Lines changed: 68 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,68 @@
1+
#!/bin/bash
2+
#-------------------------------------------------------------
3+
#
4+
# Licensed to the Apache Software Foundation (ASF) under one
5+
# or more contributor license agreements. See the NOTICE file
6+
# distributed with this work for additional information
7+
# regarding copyright ownership. The ASF licenses this file
8+
# to you under the Apache License, Version 2.0 (the
9+
# "License"); you may not use this file except in compliance
10+
# with the License. You may obtain a copy of the License at
11+
#
12+
# http://www.apache.org/licenses/LICENSE-2.0
13+
#
14+
# Unless required by applicable law or agreed to in writing,
15+
# software distributed under the License is distributed on an
16+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
17+
# KIND, either express or implied. See the License for the
18+
# specific language governing permissions and limitations
19+
# under the License.
20+
#
21+
#-------------------------------------------------------------
22+
if [ "$(basename $PWD)" != "perftest" ];
23+
then
24+
echo "Please execute scripts from directory 'perftest'"
25+
exit 1;
26+
fi
27+
28+
CMD=$1
29+
DATADIR=$2/als
30+
MAXMEM=$3
31+
32+
FORMAT="text" # can be csv, mm, text, binary
33+
DENSE_SP=0.9
34+
SPARSE_SP=0.01
35+
36+
echo "-- Generating ALS data." >> results/times.txt;
37+
38+
#generate XS scenarios (80MB)
39+
if [ $MAXMEM -ge 80 ]; then
40+
${CMD} -f ../datagen/genRandData4ALS.dml --nvargs X=${DATADIR}/X10k_1k_dense rows=10000 cols=1000 rank=10 nnz=`echo "scale=0; 10000 * 1000 * $DENSE_SP" | bc` sigma=0.01 fmt=$FORMAT &
41+
${CMD} -f ../datagen/genRandData4ALS.dml --nvargs X=${DATADIR}/X10k_1k_sparse rows=10000 cols=1000 rank=10 nnz=`echo "scale=0; 10000 * 1000 * $SPARSE_SP" | bc` sigma=0.01 fmt=$FORMAT &
42+
fi
43+
44+
#generate S scenarios (800MB)
45+
if [ $MAXMEM -ge 800 ]; then
46+
${CMD} -f ../datagen/genRandData4ALS.dml --nvargs X=${DATADIR}/X100k_1k_dense rows=100000 cols=1000 rank=10 nnz=`echo "scale=0; 100000 * 1000 * $DENSE_SP" | bc` sigma=0.01 fmt=$FORMAT &
47+
${CMD} -f ../datagen/genRandData4ALS.dml --nvargs X=${DATADIR}/X100k_1k_sparse rows=100000 cols=1000 rank=10 nnz=`echo "scale=0; 100000 * 1000 * $SPARSE_SP" | bc` sigma=0.01 fmt=$FORMAT &
48+
fi
49+
50+
#generate M scenarios (8GB)
51+
if [ $MAXMEM -ge 8000 ]; then
52+
${CMD} -f ../datagen/genRandData4ALS.dml --nvargs X=${DATADIR}/X1M_1k_dense rows=1000000 cols=1000 rank=10 nnz=`echo "scale=0; 1000000 * 1000 * $DENSE_SP" | bc` sigma=0.01 fmt=$FORMAT &
53+
${CMD} -f ../datagen/genRandData4ALS.dml --nvargs X=${DATADIR}/X1M_1k_sparse rows=1000000 cols=1000 rank=10 nnz=`echo "scale=0; 1000000 * 1000 * $SPARSE_SP" | bc` sigma=0.01 fmt=$FORMAT &
54+
fi
55+
56+
#generate L scenarios (80GB)
57+
if [ $MAXMEM -ge 80000 ]; then
58+
${CMD} -f ../datagen/genRandData4ALS.dml --nvargs X=${DATADIR}/X10M_1k_dense rows=10000000 cols=1000 rank=10 nnz=`echo "scale=0; 10000000 * 1000 * $DENSE_SP" | bc` sigma=0.01 fmt=$FORMAT
59+
${CMD} -f ../datagen/genRandData4ALS.dml --nvargs X=${DATADIR}/X10M_1k_sparse rows=10000000 cols=1000 rank=10 nnz=`echo "scale=0; 10000000 * 1000 * $SPARSE_SP" | bc` sigma=0.01 fmt=$FORMAT
60+
fi
61+
62+
#generate XL scenarios (800GB)
63+
if [ $MAXMEM -ge 800000 ]; then
64+
${CMD} -f ../datagen/genRandData4ALS.dml --nvargs X=${DATADIR}/X100M_1k_dense rows=100000000 cols=1000 rank=10 nnz=`echo "scale=0; 100000000 * 1000 * $DENSE_SP" | bc` sigma=0.01 fmt=$FORMAT
65+
${CMD} -f ../datagen/genRandData4ALS.dml --nvargs X=${DATADIR}/X100M_1k_sparse rows=100000000 cols=1000 rank=10 nnz=`echo "scale=0; 100000000 * 1000 * $SPARSE_SP" | bc` sigma=0.01 fmt=$FORMAT
66+
fi
67+
68+
wait

‎scripts/perftest/datagen/genBinomialData.sh‎

Lines changed: 78 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,78 @@
1+
#!/bin/bash
2+
#-------------------------------------------------------------
3+
#
4+
# Licensed to the Apache Software Foundation (ASF) under one
5+
# or more contributor license agreements. See the NOTICE file
6+
# distributed with this work for additional information
7+
# regarding copyright ownership. The ASF licenses this file
8+
# to you under the Apache License, Version 2.0 (the
9+
# "License"); you may not use this file except in compliance
10+
# with the License. You may obtain a copy of the License at
11+
#
12+
# http://www.apache.org/licenses/LICENSE-2.0
13+
#
14+
# Unless required by applicable law or agreed to in writing,
15+
# software distributed under the License is distributed on an
16+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
17+
# KIND, either express or implied. See the License for the
18+
# specific language governing permissions and limitations
19+
# under the License.
20+
#
21+
#-------------------------------------------------------------
22+
if [ "$(basename $PWD)" != "perftest" ];
23+
then
24+
echo "Please execute scripts from directory 'perftest'"
25+
exit 1;
26+
fi
27+
28+
CMD=$1
29+
BASE=$2/binomial
30+
MAXMEM=$3
31+
32+
FORMAT="binary" # can be csv, mm, text, binary
33+
DENSE_SP=0.9
34+
SPARSE_SP=0.01
35+
36+
echo -e "\n\n-- Generating binomial data..." >> results/times.txt;
37+
38+
#generate XS scenarios (80MB)
39+
if [ $MAXMEM -ge 80 ]; then
40+
${CMD} -f datagen/genRandData4LogisticRegression.dml --args 10000 1000 5 5 ${BASE}/w10k_1k_dense ${BASE}/X10k_1k_dense ${BASE}/y10k_1k_dense 1 0 $DENSE_SP $FORMAT 1 & pidDense80=$!
41+
${CMD} -f datagen/genRandData4LogisticRegression.dml --args 10000 1000 5 5 ${BASE}/w10k_1k_sparse ${BASE}/X10k_1k_sparse ${BASE}/y10k_1k_sparse 1 0 $SPARSE_SP $FORMAT 1 & pidSparse80=$!
42+
wait $pidDense80; ${CMD} -f scripts/extractTestData.dml --args ${BASE}/X10k_1k_dense ${BASE}/y10k_1k_dense ${BASE}/X10k_1k_dense_test ${BASE}/y10k_1k_dense_test $FORMAT &
43+
wait $pidSparse80; ${CMD} -f scripts/extractTestData.dml --args ${BASE}/X10k_1k_sparse ${BASE}/y10k_1k_sparse ${BASE}/X10k_1k_sparse_test ${BASE}/y10k_1k_sparse_test $FORMAT &
44+
fi
45+
46+
##generate S scenarios (800MB)
47+
if [ $MAXMEM -ge 800 ]; then
48+
${CMD} -f datagen/genRandData4LogisticRegression.dml --args 100000 1000 5 5 ${BASE}/w100k_1k_dense ${BASE}/X100k_1k_dense ${BASE}/y100k_1k_dense 1 0 $DENSE_SP $FORMAT 1 & pidDense800=$!
49+
${CMD} -f datagen/genRandData4LogisticRegression.dml --args 100000 1000 5 5 ${BASE}/w100k_1k_sparse ${BASE}/X100k_1k_sparse ${BASE}/y100k_1k_sparse 1 0 $SPARSE_SP $FORMAT 1 & pidSparse800=$!
50+
wait $pidDense800; ${CMD} -f scripts/extractTestData.dml --args ${BASE}/X100k_1k_dense ${BASE}/y100k_1k_dense ${BASE}/X100k_1k_dense_test ${BASE}/y100k_1k_dense_test $FORMAT &
51+
wait $pidSparse800; ${CMD} -f scripts/extractTestData.dml --args ${BASE}/X100k_1k_sparse ${BASE}/y100k_1k_sparse ${BASE}/X100k_1k_sparse_test ${BASE}/y100k_1k_sparse_test $FORMAT &
52+
fi
53+
54+
#generate M scenarios (8GB)
55+
if [ $MAXMEM -ge 8000 ]; then
56+
${CMD} -f datagen/genRandData4LogisticRegression.dml --args 1000000 1000 5 5 ${BASE}/w1M_1k_dense ${BASE}/X1M_1k_dense ${BASE}/y1M_1k_dense 1 0 $DENSE_SP $FORMAT 1 & pidDense8000=$!
57+
${CMD} -f datagen/genRandData4LogisticRegression.dml --args 1000000 1000 5 5 ${BASE}/w1M_1k_sparse ${BASE}/X1M_1k_sparse ${BASE}/y1M_1k_sparse 1 0 $SPARSE_SP $FORMAT 1 & pidSparse8000=$!
58+
wait $pidDense8000; ${CMD} -f scripts/extractTestData.dml --args ${BASE}/X1M_1k_dense ${BASE}/y1M_1k_dense ${BASE}/X1M_1k_dense_test ${BASE}/y1M_1k_dense_test $FORMAT &
59+
wait $pidSparse8000; ${CMD} -f scripts/extractTestData.dml --args ${BASE}/X1M_1k_sparse ${BASE}/y1M_1k_sparse ${BASE}/X1M_1k_sparse_test ${BASE}/y1M_1k_sparse_test $FORMAT &
60+
fi
61+
62+
#generate L scenarios (80GB)
63+
if [ $MAXMEM -ge 80000 ]; then
64+
${CMD} -f datagen/genRandData4LogisticRegression.dml --args 10000000 1000 5 5 ${BASE}/w10M_1k_dense ${BASE}/X10M_1k_dense ${BASE}/y10M_1k_dense 1 0 $DENSE_SP $FORMAT 1
65+
${CMD} -f datagen/genRandData4LogisticRegression.dml --args 10000000 1000 5 5 ${BASE}/w10M_1k_sparse ${BASE}/X10M_1k_sparse ${BASE}/y10M_1k_sparse 1 0 $SPARSE_SP $FORMAT 1
66+
${CMD} -f scripts/extractTestData.dml --args ${BASE}/X10M_1k_dense ${BASE}/y10M_1k_dense ${BASE}/X10M_1k_dense_test ${BASE}/y10M_1k_dense_test $FORMAT
67+
${CMD} -f scripts/extractTestData.dml --args ${BASE}/X10M_1k_sparse ${BASE}/y10M_1k_sparse ${BASE}/X10M_1k_sparse_test ${BASE}/y10M_1k_sparse_test $FORMAT
68+
fi
69+
70+
##generate XL scenarios (800GB)
71+
if [ $MAXMEM -ge 800000 ]; then
72+
${CMD} -f datagen/genRandData4LogisticRegression.dml --args 100000000 1000 5 5 ${BASE}/w100M_1k_dense ${BASE}/X100M_1k_dense ${BASE}/y100M_1k_dense 1 0 $DENSE_SP $FORMAT 1
73+
${CMD} -f datagen/genRandData4LogisticRegression.dml --args 100000000 1000 5 5 ${BASE}/w100M_1k_sparse ${BASE}/X100M_1k_sparse ${BASE}/y100M_1k_sparse 1 0 $SPARSE_SP $FORMAT 1
74+
${CMD} -f scripts/extractTestData.dml --args ${BASE}/X100M_1k_dense ${BASE}/y100M_1k_dense ${BASE}/X100M_1k_dense_test ${BASE}/y100M_1k_dense_test $FORMAT
75+
${CMD} -f scripts/extractTestData.dml --args ${BASE}/X100M_1k_sparse ${BASE}/y100M_1k_sparse ${BASE}/X100M_1k_sparse_test ${BASE}/y100M_1k_sparse_test $FORMAT
76+
fi
77+
78+
wait

‎scripts/perftest/datagen/genClusteringData.sh‎

Lines changed: 68 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,68 @@
1+
#!/bin/bash
2+
#-------------------------------------------------------------
3+
#
4+
# Licensed to the Apache Software Foundation (ASF) under one
5+
# or more contributor license agreements. See the NOTICE file
6+
# distributed with this work for additional information
7+
# regarding copyright ownership. The ASF licenses this file
8+
# to you under the Apache License, Version 2.0 (the
9+
# "License"); you may not use this file except in compliance
10+
# with the License. You may obtain a copy of the License at
11+
#
12+
# http://www.apache.org/licenses/LICENSE-2.0
13+
#
14+
# Unless required by applicable law or agreed to in writing,
15+
# software distributed under the License is distributed on an
16+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
17+
# KIND, either express or implied. See the License for the
18+
# specific language governing permissions and limitations
19+
# under the License.
20+
#
21+
#-------------------------------------------------------------
22+
if [ "$(basename $PWD)" != "perftest" ];
23+
then
24+
echo "Please execute scripts from directory 'perftest'"
25+
exit 1;
26+
fi
27+
28+
CMD=${1:-systemds}
29+
BASE=${2:-"temp"}/clustering
30+
MAXMEM=${3:-80}
31+
32+
FORMAT="binary"
33+
DENSE_SP=0.9
34+
SPARSE_SP=0.01
35+
36+
echo "-- Generating clustering data..." >> results/times.txt;
37+
38+
#generate XS scenarios (80MB)
39+
if [ $MAXMEM -ge 80 ]; then
40+
${CMD} -f datagen/genRandData4Kmeans.dml --nvargs nr=10000 nf=1000 nc=5 dc=10.0 dr=1.0 fbf=100.0 cbf=100.0 X=$BASE/X10k_1k_dense C=$BASE/C10k_1k_dense Y=$BASE/y10k_1k_dense YbyC=$BASE/YbyC10k_1k_dense fmt=$FORMAT & pidDense80=$!
41+
wait $pidDense80; ${CMD} -f scripts/extractTestData.dml --args $BASE/X10k_1k_dense $BASE/y10k_1k_dense $BASE/X10k_1k_dense_test $BASE/y10k_1k_dense_test $FORMAT &
42+
fi
43+
44+
#generate S scenarios (800MB)
45+
if [ $MAXMEM -ge 800 ]; then
46+
${CMD} -f datagen/genRandData4Kmeans.dml --nvargs nr=100000 nf=1000 nc=5 dc=10.0 dr=1.0 fbf=100.0 cbf=100.0 X=$BASE/X100k_1k_dense C=$BASE/C100k_1k_dense Y=$BASE/y100k_1k_dense YbyC=$BASE/YbyC100k_1k_dense fmt=$FORMAT & pidDense800=$!
47+
wait $pidDense800; ${CMD} -f scripts/extractTestData.dml --args $BASE/X100k_1k_dense $BASE/y100k_1k_dense $BASE/X100k_1k_dense_test $BASE/y100k_1k_dense_test $FORMAT &
48+
fi
49+
50+
#generate M scenarios (8GB)
51+
if [ $MAXMEM -ge 8000 ]; then
52+
${CMD} -f datagen/genRandData4Kmeans.dml --nvargs nr=1000000 nf=1000 nc=5 dc=10.0 dr=1.0 fbf=100.0 cbf=100.0 X=$BASE/X1M_1k_dense C=$BASE/C1M_1k_dense Y=$BASE/y1M_1k_dense YbyC=$BASE/YbyC1M_1k_dense fmt=$FORMAT & pidDense8000=$!
53+
wait $pidDense8000; ${CMD} -f scripts/extractTestData.dml --args $BASE/X1M_1k_dense $BASE/y1M_1k_dense $BASE/X1M_1k_dense_test $BASE/y1M_1k_dense_test $FORMAT &
54+
fi
55+
56+
#generate L scenarios (80GB)
57+
if [ $MAXMEM -ge 80000 ]; then
58+
${CMD} -f datagen/genRandData4Kmeans.dml --nvargs nr=10000000 nf=1000 nc=5 dc=10.0 dr=1.0 fbf=100.0 cbf=100.0 X=$BASE/X10M_1k_dense C=$BASE/C10M_1k_dense Y=$BASE/y10M_1k_dense YbyC=$BASE/YbyC10M_1k_dense fmt=$FORMAT
59+
${CMD} -f scripts/extractTestData.dml --args $BASE/X10M_1k_dense $BASE/y10M_1k_dense $BASE/X10M_1k_dense_test $BASE/y10M_1k_dense_test $FORMAT
60+
fi
61+
62+
#generate LARGE scenarios (800GB)
63+
if [ $MAXMEM -ge 800000 ]; then
64+
${CMD} -f datagen/genRandData4Kmeans.dml --nvargs nr=100000000 nf=1000 nc=5 dc=10.0 dr=1.0 fbf=100.0 cbf=100.0 X=$BASE/X100M_1k_dense C=$BASE/C100M_1k_dense Y=$BASE/y100M_1k_dense YbyC=$BASE/YbyC100M_1k_dense fmt=$FORMAT
65+
${CMD} -f scripts/extractTestData.dml --args $BASE/X100M_1k_dense $BASE/y100M_1k_dense $BASE/X100M_1k_dense_test $BASE/y100M_1k_dense_test $FORMAT
66+
fi
67+
68+
wait

‎scripts/perftest/datagen/genDescriptiveStatisticsData.sh‎

Lines changed: 60 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,60 @@
1+
#!/bin/bash
2+
#-------------------------------------------------------------
3+
#
4+
# Licensed to the Apache Software Foundation (ASF) under one
5+
# or more contributor license agreements. See the NOTICE file
6+
# distributed with this work for additional information
7+
# regarding copyright ownership. The ASF licenses this file
8+
# to you under the Apache License, Version 2.0 (the
9+
# "License"); you may not use this file except in compliance
10+
# with the License. You may obtain a copy of the License at
11+
#
12+
# http://www.apache.org/licenses/LICENSE-2.0
13+
#
14+
# Unless required by applicable law or agreed to in writing,
15+
# software distributed under the License is distributed on an
16+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
17+
# KIND, either express or implied. See the License for the
18+
# specific language governing permissions and limitations
19+
# under the License.
20+
#
21+
#-------------------------------------------------------------
22+
if [ "$(basename $PWD)" != "perftest" ];
23+
then
24+
echo "Please execute scripts from directory 'perftest'"
25+
exit 1;
26+
fi
27+
28+
CMD=$1
29+
BASE=$2/bivar
30+
MAXMEM=$3
31+
32+
FORMAT="binary"
33+
34+
c=1000
35+
nc=100
36+
mdomain=1100
37+
set=20
38+
labelset=10
39+
40+
#XS data 10K rows
41+
if [ $MAXMEM -ge 80 ]; then
42+
${CMD} -f datagen/genRandData4DescriptiveStats.dml --explain --stats --nvargs R=10000 C=$c NC=$nc MAXDOMAIN=$mdomain DATA=${BASE}/A_10k/data TYPES=${BASE}/A_10k/types SETSIZE=$set LABELSETSIZE=$labelset TYPES1=${BASE}/A_10k/set1.types TYPES2=${BASE}/A_10k/set2.types INDEX1=${BASE}/A_10k/set1.indices INDEX2=${BASE}/A_10k/set2.indices FMT=$FORMAT &
43+
fi
44+
45+
#S data 100K rows
46+
if [ $MAXMEM -ge 800 ]; then
47+
${CMD} -f datagen/genRandData4DescriptiveStats.dml --explain --stats --nvargs R=100000 C=$c NC=$nc MAXDOMAIN=$mdomain DATA=${BASE}/A_100k/data TYPES=${BASE}/A_100k/types SETSIZE=$set LABELSETSIZE=$labelset TYPES1=${BASE}/A_100k/set1.types TYPES2=${BASE}/A_100k/set2.types INDEX1=${BASE}/A_100k/set1.indices INDEX2=${BASE}/A_100k/set2.indices FMT=$FORMAT &
48+
fi
49+
50+
#M data 1M rows
51+
if [ $MAXMEM -ge 8000 ]; then
52+
${CMD} -f datagen/genRandData4DescriptiveStats.dml --explain --stats --nvargs R=1000000 C=$c NC=$nc MAXDOMAIN=$mdomain DATA=${BASE}/A_1M/data TYPES=${BASE}/A_1M/types SETSIZE=$set LABELSETSIZE=$labelset TYPES1=${BASE}/A_1M/set1.types TYPES2=${BASE}/A_1M/set2.types INDEX1=${BASE}/A_1M/set1.indices INDEX2=${BASE}/A_1M/set2.indices FMT=$FORMAT &
53+
fi
54+
55+
#L data 10M rows
56+
if [ $MAXMEM -ge 80000 ]; then
57+
${CMD} -f datagen/genRandData4DescriptiveStats.dml --explain --stats --nvargs R=10000000 C=$c NC=$nc MAXDOMAIN=$mdomain DATA=${BASE}/A_10M/data TYPES=${BASE}/A_10M/types SETSIZE=$set LABELSETSIZE=$labelset TYPES1=${BASE}/A_10M/set1.types TYPES2=${BASE}/A_10M/set2.types INDEX1=${BASE}/A_10M/set1.indices INDEX2=${BASE}/A_10M/set2.indices FMT=$FORMAT
58+
fi
59+
60+
wait

‎scripts/perftest/datagen/genDimensionReductionData.sh‎

Lines changed: 61 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,61 @@
1+
#!/bin/bash
2+
#-------------------------------------------------------------
3+
#
4+
# Licensed to the Apache Software Foundation (ASF) under one
5+
# or more contributor license agreements. See the NOTICE file
6+
# distributed with this work for additional information
7+
# regarding copyright ownership. The ASF licenses this file
8+
# to you under the Apache License, Version 2.0 (the
9+
# "License"); you may not use this file except in compliance
10+
# with the License. You may obtain a copy of the License at
11+
#
12+
# http://www.apache.org/licenses/LICENSE-2.0
13+
#
14+
# Unless required by applicable law or agreed to in writing,
15+
# software distributed under the License is distributed on an
16+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
17+
# KIND, either express or implied. See the License for the
18+
# specific language governing permissions and limitations
19+
# under the License.
20+
#
21+
#-------------------------------------------------------------
22+
if [ "$(basename $PWD)" != "perftest" ];
23+
then
24+
echo "Please execute scripts from directory 'perftest'"
25+
exit 1;
26+
fi
27+
28+
CMD=${1:-systemds}
29+
BASE=${2:-"temp"}/dimensionreduction
30+
MAXMEM=${3:-80}
31+
32+
FORMAT="binary"
33+
34+
echo "-- Generating Dimension Reduction data." >> results/times.txt;
35+
36+
#generate XS scenarios (80MB)
37+
if [ $MAXMEM -ge 80 ]; then
38+
${CMD} -f datagen/genRandData4PCA.dml --nvargs R=5000 C=2000 OUT=$BASE/pcaData5k_2k_dense FMT=$FORMAT &
39+
fi
40+
41+
#generate S scenarios (800MB)
42+
if [ $MAXMEM -ge 800 ]; then
43+
${CMD} -f datagen/genRandData4PCA.dml --nvargs R=50000 C=2000 OUT=$BASE/pcaData50k_2k_dense FMT=$FORMAT &
44+
fi
45+
46+
#generate M scenarios (8GB)
47+
if [ $MAXMEM -ge 8000 ]; then
48+
${CMD} -f datagen/genRandData4PCA.dml --nvargs R=500000 C=2000 OUT=$BASE/pcaData500k_2k_dense FMT=$FORMAT &
49+
fi
50+
51+
#generate L scenarios (80GB)
52+
if [ $MAXMEM -ge 80000 ]; then
53+
${CMD} -f datagen/genRandData4PCA.dml --nvargs R=5000000 C=2000 OUT=$BASE/pcaData5M_2k_dense FMT=$FORMAT
54+
fi
55+
56+
#generate XL scenarios (800GB)
57+
if [ $MAXMEM -ge 800000 ]; then
58+
${CMD} -f datagen/genRandData4PCA.dml --nvargs R=50000000 C=2000 OUT=$BASE/pcaData50M_2k_dense FMT=$FORMAT
59+
fi
60+
61+
wait

0 commit comments

Comments
 (0)