Skip to content

Commit 6ce5648

Browse files
alexaverbuchalexaverbuch
committed
scripts
1 parent e26a97c commit 6ce5648

10 files changed

+175
-6
lines changed

‎params.ini

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -8,13 +8,13 @@ ldbc.snb.datagen.serializer.compressed:false
88

99
### Serialization
1010
# -- Regular --
11-
# ldbc.snb.datagen.serializer.personSerializer:ldbc.snb.datagen.serializer.snb.interactive.CSVPersonSerializer
12-
# ldbc.snb.datagen.serializer.invariantSerializer:ldbc.snb.datagen.serializer.snb.interactive.CSVInvariantSerializer
13-
# ldbc.snb.datagen.serializer.personActivitySerializer:ldbc.snb.datagen.serializer.snb.interactive.CSVPersonActivitySerializer
11+
ldbc.snb.datagen.serializer.personSerializer:ldbc.snb.datagen.serializer.snb.interactive.CSVPersonSerializer
12+
ldbc.snb.datagen.serializer.invariantSerializer:ldbc.snb.datagen.serializer.snb.interactive.CSVInvariantSerializer
13+
ldbc.snb.datagen.serializer.personActivitySerializer:ldbc.snb.datagen.serializer.snb.interactive.CSVPersonActivitySerializer
1414
### -- Merge --
15-
ldbc.snb.datagen.serializer.personSerializer:ldbc.snb.datagen.serializer.snb.interactive.CSVMergeForeignPersonSerializer
16-
ldbc.snb.datagen.serializer.invariantSerializer:ldbc.snb.datagen.serializer.snb.interactive.CSVMergeForeignInvariantSerializer
17-
ldbc.snb.datagen.serializer.personActivitySerializer:ldbc.snb.datagen.serializer.snb.interactive.CSVMergeForeignPersonActivitySerializer
15+
# ldbc.snb.datagen.serializer.personSerializer:ldbc.snb.datagen.serializer.snb.interactive.CSVMergeForeignPersonSerializer
16+
# ldbc.snb.datagen.serializer.invariantSerializer:ldbc.snb.datagen.serializer.snb.interactive.CSVMergeForeignInvariantSerializer
17+
# ldbc.snb.datagen.serializer.personActivitySerializer:ldbc.snb.datagen.serializer.snb.interactive.CSVMergeForeignPersonActivitySerializer
1818

1919
ldbc.snb.datagen.serializer.updateStreams:false
2020
ldbc.snb.datagen.serializer.numUpdatePartitions:4

‎scripts/copy_data_and_params_to_validation_set.sh

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
#!/bin/sh
2+
connector_dir=/Users/alexaverbuch/IdeaProjects/ldbc-load-generator/ldbc_snb_workload_interactive_neo4j
3+
validation_set_dir=${connector_dir}/data-import/src/test/resources/validation_sets/business_intelligence/neo4j
4+
data_gen_dir=/Users/alexaverbuch/IdeaProjects/ldbc_snb_datagen
5+
parameters_dir=${data_gen_dir}/substitution_parameters
6+
7+
# --- DELETE ---
8+
9+
rm ${validation_set_dir}/params.ini
10+
rm ${validation_set_dir}/validation_params.csv
11+
rm ${validation_set_dir}/substitution_parameters/*
12+
rm ${validation_set_dir}/social_network/*
13+
14+
# --- COPY ---
15+
16+
cp ${data_gen_dir}/params.ini ${validation_set_dir}/
17+
18+
cp ${parameters_dir}/q1* ${validation_set_dir}/substitution_parameters/
19+
cp ${parameters_dir}/q2* ${validation_set_dir}/substitution_parameters/
20+
cp ${parameters_dir}/q3* ${validation_set_dir}/substitution_parameters/
21+
cp ${parameters_dir}/q4* ${validation_set_dir}/substitution_parameters/
22+
cp ${parameters_dir}/q5* ${validation_set_dir}/substitution_parameters/
23+
cp ${parameters_dir}/q6* ${validation_set_dir}/substitution_parameters/
24+
cp ${parameters_dir}/q7* ${validation_set_dir}/substitution_parameters/
25+
cp ${parameters_dir}/q8* ${validation_set_dir}/substitution_parameters/
26+
cp ${parameters_dir}/q9* ${validation_set_dir}/substitution_parameters/
27+
28+
cp /Users/alexaverbuch/hadoopTempDir/output/social_network/* ${validation_set_dir}/social_network/

‎scripts/copy_merge_data_to_validation_set.sh

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
#!/bin/sh
2+
connector_dir=/Users/alexaverbuch/IdeaProjects/ldbc-load-generator/ldbc_snb_workload_interactive_neo4j
3+
validation_set_dir=${connector_dir}/data-import/src/test/resources/validation_sets/business_intelligence/neo4j
4+
5+
# --- DELETE ---
6+
7+
rm ${validation_set_dir}/merge/social_network/*
8+
9+
# --- COPY ---
10+
11+
cp /Users/alexaverbuch/hadoopTempDir/output/social_network/* ${validation_set_dir}/merge/social_network/

‎scripts/copy_params_ldbc_driver.sh

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
#!/bin/sh
2+
parameters_dir=/Users/alexaverbuch/IdeaProjects/ldbc_snb_datagen/substitution_parameters
3+
driver_dir=/Users/alexaverbuch/IdeaProjects/ldbc-load-generator/ldbc_snb_workload_interactive_neo4j/ldbc_driver/src/test/resources/snb/bi/
4+
5+
mv ${parameters_dir}/q1* ${driver_dir}
6+
mv ${parameters_dir}/q2* ${driver_dir}
7+
mv ${parameters_dir}/q3* ${driver_dir}
8+
mv ${parameters_dir}/q4* ${driver_dir}
9+
mv ${parameters_dir}/q5* ${driver_dir}
10+
mv ${parameters_dir}/q6* ${driver_dir}
11+
mv ${parameters_dir}/q7* ${driver_dir}
12+
mv ${parameters_dir}/q8* ${driver_dir}
13+
mv ${parameters_dir}/q9* ${driver_dir}

‎scripts/package_neo4j_validation_set.sh

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
#!/bin/sh
2+
validation_set_dir=/Users/alexaverbuch/IdeaProjects/ldbc_snb_bi_validation/neo4j
3+
connector_dir=/Users/alexaverbuch/IdeaProjects/ldbc-load-generator/ldbc_snb_workload_interactive_neo4j/data-import/src/test/resources/validation_sets/business_intelligence/neo4j
4+
5+
rm -rf neo4j--validation_set/
6+
mkdir neo4j--validation_set/
7+
rm -rf neo4j--validation_set/*
8+
9+
cp -r ${connector_dir}/* neo4j--validation_set/
10+
tar -czvf neo4j--validation_set.tar.gz neo4j--validation_set/
11+
rm ${validation_set_dir}/neo4j--validation_set.tar.gz
12+
cp neo4j--validation_set.tar.gz ${validation_set_dir}/
13+
14+
rm -rf neo4j--validation_set*

‎scripts/package_validation_set.sh

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
#!/bin/sh
2+
validation_set_dir=/Users/alexaverbuch/IdeaProjects/ldbc_snb_bi_validation/neo4j
3+
connector_dir=/Users/alexaverbuch/IdeaProjects/ldbc-load-generator/ldbc_snb_workload_interactive_neo4j/data-import/src/test/resources/validation_sets/business_intelligence/neo4j
4+
5+
rm -rf neo4j--validation_set/
6+
mkdir neo4j--validation_set/
7+
rm -rf neo4j--validation_set/*
8+
9+
cp -r ${connector_dir}/* neo4j--validation_set/
10+
cd neo4j--validation_set/
11+
tar -czvf neo4j--validation_set.tar.gz *
12+
cd ..
13+
rm ${validation_set_dir}/neo4j--validation_set.tar.gz
14+
cp neo4j--validation_set/neo4j--validation_set.tar.gz ${validation_set_dir}/
15+
16+
rm -rf neo4j--validation_set*

‎scripts/params_merge.ini

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
# ldbc.snb.datagen.generator.scaleFactor:snb.interactive.1
2+
3+
ldbc.snb.datagen.generator.numPersons:3000
4+
ldbc.snb.datagen.generator.numYears:1
5+
ldbc.snb.datagen.generator.startYear:2010
6+
7+
ldbc.snb.datagen.serializer.compressed:false
8+
9+
### Serialization
10+
# -- Regular --
11+
# ldbc.snb.datagen.serializer.personSerializer:ldbc.snb.datagen.serializer.snb.interactive.CSVPersonSerializer
12+
# ldbc.snb.datagen.serializer.invariantSerializer:ldbc.snb.datagen.serializer.snb.interactive.CSVInvariantSerializer
13+
# ldbc.snb.datagen.serializer.personActivitySerializer:ldbc.snb.datagen.serializer.snb.interactive.CSVPersonActivitySerializer
14+
### -- Merge --
15+
ldbc.snb.datagen.serializer.personSerializer:ldbc.snb.datagen.serializer.snb.interactive.CSVMergeForeignPersonSerializer
16+
ldbc.snb.datagen.serializer.invariantSerializer:ldbc.snb.datagen.serializer.snb.interactive.CSVMergeForeignInvariantSerializer
17+
ldbc.snb.datagen.serializer.personActivitySerializer:ldbc.snb.datagen.serializer.snb.interactive.CSVMergeForeignPersonActivitySerializer
18+
19+
ldbc.snb.datagen.serializer.updateStreams:false
20+
ldbc.snb.datagen.serializer.numUpdatePartitions:4
21+
ldbc.snb.datagen.serializer.outputDir:/Users/alexaverbuch/hadoopTempDir/output/
22+
23+
ldbc.snb.datagen.generator.numThreads:8
24+
ldbc.snb.datagen.generator.numPartitions:1
25+
26+
# https://github.com/ldbc-dev/ldbc_snb_datagen_0.2/wiki/Compilation_Execution
27+
# https://github.com/ldbc-dev/ldbc_snb_datagen_0.2/blob/master/src/main/resources/params.ini

‎scripts/params_regular.ini

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
# ldbc.snb.datagen.generator.scaleFactor:snb.interactive.1
2+
3+
ldbc.snb.datagen.generator.numPersons:3000
4+
ldbc.snb.datagen.generator.numYears:1
5+
ldbc.snb.datagen.generator.startYear:2010
6+
7+
ldbc.snb.datagen.serializer.compressed:false
8+
9+
### Serialization
10+
# -- Regular --
11+
ldbc.snb.datagen.serializer.personSerializer:ldbc.snb.datagen.serializer.snb.interactive.CSVPersonSerializer
12+
ldbc.snb.datagen.serializer.invariantSerializer:ldbc.snb.datagen.serializer.snb.interactive.CSVInvariantSerializer
13+
ldbc.snb.datagen.serializer.personActivitySerializer:ldbc.snb.datagen.serializer.snb.interactive.CSVPersonActivitySerializer
14+
### -- Merge --
15+
# ldbc.snb.datagen.serializer.personSerializer:ldbc.snb.datagen.serializer.snb.interactive.CSVMergeForeignPersonSerializer
16+
# ldbc.snb.datagen.serializer.invariantSerializer:ldbc.snb.datagen.serializer.snb.interactive.CSVMergeForeignInvariantSerializer
17+
# ldbc.snb.datagen.serializer.personActivitySerializer:ldbc.snb.datagen.serializer.snb.interactive.CSVMergeForeignPersonActivitySerializer
18+
19+
ldbc.snb.datagen.serializer.updateStreams:false
20+
ldbc.snb.datagen.serializer.numUpdatePartitions:4
21+
ldbc.snb.datagen.serializer.outputDir:/Users/alexaverbuch/hadoopTempDir/output/
22+
23+
ldbc.snb.datagen.generator.numThreads:8
24+
ldbc.snb.datagen.generator.numPartitions:1
25+
26+
# https://github.com/ldbc-dev/ldbc_snb_datagen_0.2/wiki/Compilation_Execution
27+
# https://github.com/ldbc-dev/ldbc_snb_datagen_0.2/blob/master/src/main/resources/params.ini

‎scripts/unpackage_sparksee_validation_sets.sh

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
#!/bin/sh
2+
validation_set_dir=/Users/alexaverbuch/IdeaProjects/ldbc_snb_bi_validation/sparksee/1_5
3+
connector_dir=/Users/alexaverbuch/IdeaProjects/ldbc-load-generator/ldbc_snb_workload_interactive_neo4j/data-import/src/test/resources/validation_sets/business_intelligence/sparksee
4+
5+
rm -rf ${connector_dir}/*
6+
7+
mkdir ${connector_dir}/merge/
8+
mkdir ${connector_dir}/merge/social_network/
9+
mkdir ${connector_dir}/social_network/
10+
mkdir ${connector_dir}/substitution_parameters/
11+
12+
tar -xzvf ${validation_set_dir}/readwrite_sparksee--validation_set.tar.gz
13+
mv validation_set/validation_params.csv ${connector_dir}/
14+
rm validation_set/query_*
15+
mv validation_set/q*.txt ${connector_dir}/substitution_parameters/
16+
mv validation_set/merge/*.csv ${connector_dir}/merge/social_network/
17+
mv validation_set/*.csv ${connector_dir}/social_network/
18+
19+
rm -rf validation_set

‎scripts/update_validation_set.sh

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
#!/bin/sh
2+
data_gen_dir=/Users/alexaverbuch/IdeaProjects/ldbc_snb_datagen
3+
scripts_dir=${data_gen_dir}/scripts
4+
5+
rm ${data_gen_dir}/params.ini
6+
cp ${scripts_dir}/params_regular.ini ${data_gen_dir}/params.ini
7+
bash ${data_gen_dir}/run.sh
8+
bash ${scripts_dir}/copy_data_and_params_to_validation_set.sh
9+
rm ${data_gen_dir}/params.ini
10+
cp ${scripts_dir}/params_merge.ini ${data_gen_dir}/params.ini
11+
bash ${data_gen_dir}/run.sh
12+
bash ${scripts_dir}/copy_merge_data_to_validation_set.sh
13+
rm ${data_gen_dir}/params.ini
14+
cp ${scripts_dir}/params_regular.ini ${data_gen_dir}/params.ini

0 commit comments

Comments
 (0)