Skip to content

Commit 6d893ca

Browse files
szarnyasgszarnyasg
committed
Generate and deploy factors in CI
1 parent 053c36e commit 6d893ca

File tree

1 file changed

+72
-25
lines changed

1 file changed

+72
-25
lines changed

‎.circleci/config.yml

Lines changed: 72 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -33,14 +33,16 @@ jobs:
3333
DATAGEN_VERSION: 0.5.0-SNAPSHOT
3434
steps:
3535
- checkout
36-
- run: |
37-
mkdir out/
38-
DEBIAN_FRONTEND=noninteractive
39-
sudo apt update
40-
sudo apt install -y openjdk-8-jdk zip
41-
sudo update-alternatives --install /usr/bin/java java /usr/lib/jvm/java-8-openjdk-amd64/bin/java 1
42-
sudo update-alternatives --set java /usr/lib/jvm/java-8-openjdk-amd64/bin/java
43-
java -version
36+
- run:
37+
name: Install dependencies
38+
command: |
39+
mkdir out/
40+
DEBIAN_FRONTEND=noninteractive
41+
sudo apt update
42+
sudo apt install -y openjdk-8-jdk zip
43+
sudo update-alternatives --install /usr/bin/java java /usr/lib/jvm/java-8-openjdk-amd64/bin/java 1
44+
sudo update-alternatives --set java /usr/lib/jvm/java-8-openjdk-amd64/bin/java
45+
java -version
4446
- run: docker build . -t ldbc/spark
4547
- restore_cache:
4648
keys:
@@ -55,24 +57,69 @@ jobs:
5557
- ~/.m2/repository # maven deps
5658
- run: mkdir social-network-sf0.003
5759
# BI
58-
- run: tools/docker-run.sh --mode bi --scale-factor 0.003 --explode-edges --explode-attrs && mv out social-network-sf0.003-bi-singular-projected-fk
59-
- run: tools/docker-run.sh --mode bi --scale-factor 0.003 --explode-attrs && mv out social-network-sf0.003-bi-singular-merged-fk
60-
- run: tools/docker-run.sh --mode bi --scale-factor 0.003 --explode-edges && mv out social-network-sf0.003-bi-composite-projected-fk
61-
- run: tools/docker-run.sh --mode bi --scale-factor 0.003 && mv out social-network-sf0.003-bi-composite-merged-fk
62-
- run: tools/docker-run.sh --mode bi --scale-factor 0.003 --format-options header=true && mv out social-network-sf0.003-bi-composite-merged-fk-postgres
63-
- run: tools/docker-run.sh --mode bi --scale-factor 0.003 --explode-edges --format-options header=false,quoteAll=true && mv out social-network-sf0.003-bi-composite-projected-fk-neo4j
60+
- run:
61+
name: Generate SF0.003 / BI / singular-projected CSVs
62+
command: |
63+
tools/docker-run.sh --mode bi --scale-factor 0.003 --explode-edges --explode-attrs
64+
mv out social-network-sf0.003-bi-singular-projected-fk
65+
- run:
66+
name: Generate SF0.003 / BI / singular-projected CSVs
67+
command: |
68+
tools/docker-run.sh --mode bi --scale-factor 0.003 --explode-attrs
69+
mv out social-network-sf0.003-bi-singular-merged-fk
70+
- run:
71+
name: Generate SF0.003 / BI / singular-projected CSVs
72+
command: |
73+
tools/docker-run.sh --mode bi --scale-factor 0.003 --explode-edges
74+
mv out social-network-sf0.003-bi-composite-projected-fk
75+
- run:
76+
name: Generate SF0.003 / BI / singular-projected CSVs
77+
command: |
78+
tools/docker-run.sh --mode bi --scale-factor 0.003 --generate-factors
79+
mv out social-network-sf0.003-bi-composite-merged-fk
80+
- run:
81+
name: Generate SF0.003 / BI / singular-projected CSVs
82+
command: |
83+
tools/docker-run.sh --mode bi --scale-factor 0.003 --format-options header=true
84+
mv out social-network-sf0.003-bi-composite-merged-fk-postgres
85+
- run:
86+
name: Generate SF0.003 / BI / singular-projected CSVs
87+
command: |
88+
tools/docker-run.sh --mode bi --scale-factor 0.003 --explode-edges --format-options header=false,quoteAll=true
89+
mv out social-network-sf0.003-bi-composite-projected-fk-neo4j
6490
# Interactive
65-
- run: tools/docker-run.sh --mode interactive --scale-factor 0.003 --explode-edges --explode-attrs && mv out social-network-sf0.003-interactive-singular-projected-fk
66-
- run: tools/docker-run.sh --mode interactive --scale-factor 0.003 --explode-attrs && mv out social-network-sf0.003-interactive-singular-merged-fk
67-
- run: tools/docker-run.sh --mode interactive --scale-factor 0.003 --explode-edges && mv out social-network-sf0.003-interactive-composite-projected-fk
68-
- run: tools/docker-run.sh --mode interactive --scale-factor 0.003 && mv out social-network-sf0.003-interactive-composite-merged-fk
69-
- run: |
70-
# deploy CircleCI configuration to provide the 'filters' instructions (and prevent failed builds on the gh-pages branch)
71-
mv .circleci dist/
72-
for d in social-network-sf0.003*; do
73-
echo "Generated with <https://github.com/ldbc/ldbc_snb_datagen_spark/commit/${CIRCLE_SHA1}>" > $d/README.md
74-
zip -r dist/$d.zip $d/
75-
done
91+
- run:
92+
name: Generate SF0.003 / BI / singular-projected CSVs
93+
command: |
94+
tools/docker-run.sh --mode interactive --scale-factor 0.003 --explode-edges --explode-attrs
95+
mv out social-network-sf0.003-interactive-singular-projected-fk
96+
- run:
97+
name: Generate SF0.003 / BI / singular-projected CSVs
98+
command: |
99+
tools/docker-run.sh --mode interactive --scale-factor 0.003 --explode-attrs
100+
mv out social-network-sf0.003-interactive-singular-merged-fk
101+
- run:
102+
name: Generate SF0.003 / BI / singular-projected CSVs
103+
command: |
104+
tools/docker-run.sh --mode interactive --scale-factor 0.003 --explode-edges
105+
mv out social-network-sf0.003-interactive-composite-projected-fk
106+
- run:
107+
name: Generate SF0.003 / BI / singular-projected CSVs
108+
command: |
109+
tools/docker-run.sh --mode interactive --scale-factor 0.003
110+
mv out social-network-sf0.003-interactive-composite-merged-fk
111+
- run:
112+
name: Compress directories and prepare for deployment
113+
command: |
114+
# include the CircleCI configuration in the deployed package to provide the 'filters' instructions (and prevent failed builds on the gh-pages branch)
115+
mv .circleci dist/
116+
# move factors to a separate directory
117+
mv social-network-sf0.003-bi-composite-merged-fk/factors social-network-sf0.003-bi-factors
118+
# compress each directory
119+
for d in social-network-sf0.003*; do
120+
echo "Generated with <https://github.com/ldbc/ldbc_snb_datagen_spark/commit/${CIRCLE_SHA1}>" > $d/README.md
121+
zip -r dist/$d.zip $d/
122+
done
76123
- persist_to_workspace:
77124
# Must be an absolute path, or relative path from working_directory. This is a directory on the container which is
78125
# taken to be the root directory of the workspace.

0 commit comments

Comments
 (0)