Merge branch 'main' into interactive-more-factor-tables

Author: szarnyasg

.circleci/config.yml

@@ -22,110 +22,77 @@ workflows:
 executors:
   my-executor:
     machine:
-      image: ubuntu-2004:202008-01
+      image: ubuntu-2204:2022.04.1
+      docker_layer_caching: true
     working_directory: ~/ldbc/ldbc_snb_datagen
 
 jobs:
   test:
     executor: my-executor
     environment:
-      PLATFORM_VERSION: 2.12_spark3.1
+      PLATFORM_VERSION: 2.12_spark3.2
       DATAGEN_VERSION: 0.5.0-SNAPSHOT
     steps:
       - checkout
-      - run:
-          name: Install dependencies
-          command: |
-            mkdir out/
-            DEBIAN_FRONTEND=noninteractive
-            sudo apt update
-            sudo apt install -y openjdk-8-jdk zip
-            sudo update-alternatives --install /usr/bin/java java /usr/lib/jvm/java-8-openjdk-amd64/bin/java 1
-            sudo update-alternatives --set java /usr/lib/jvm/java-8-openjdk-amd64/bin/java
       - run:
           name: Build Docker container
           command: |
-            docker build . -t ldbc/spark
-      - restore_cache:
-          keys:
-            - m2-dep-branch:{{ .Branch }}-pom:{{ checksum "pom.xml" }}-
-            - m2-dep-branch:dev-pom:{{ checksum "pom.xml" }}-
-            - m2-dep-branch:{{ .Branch }}-
-            - m2-dep-branch:dev-
-      - run:
-          name: Build JAR file
-          command: |
-            mvn -ntp clean test-compile assembly:assembly
-      - save_cache:
-          key: m2-dep-branch:{{ .Branch }}-pom:{{ checksum "pom.xml" }}-{{ epoch }}
-          paths:
-            - ~/.m2/repository   # maven deps
+            docker build . --target=standalone --tag ldbc/datagen-standalone:latest
       # BI
-      - run: 
+      - run:
           name: Generate SF0.003 / BI / singular-projected CSVs
           command: |
-            tools/docker-run.sh --mode bi --scale-factor 0.003 --explode-edges --explode-attrs
+            tools/docker-run.sh -- --mode bi --scale-factor 0.003 --explode-edges --explode-attrs
             mv out/ social-network-sf0.003-bi-singular-projected-fk/
       - run:
           name: Generate SF0.003 / BI / singular-merged CSVs
           command: |
-            tools/docker-run.sh --mode bi --scale-factor 0.003 --explode-attrs
+            tools/docker-run.sh -- --mode bi --scale-factor 0.003 --explode-attrs
             mv out/ social-network-sf0.003-bi-singular-merged-fk/
       - run:
           name: Generate SF0.003 / BI / composite-projected CSVs
           command: |
-            tools/docker-run.sh --mode bi --scale-factor 0.003 --explode-edges
+            tools/docker-run.sh -- --mode bi --scale-factor 0.003 --explode-edges
             mv out/ social-network-sf0.003-bi-composite-projected-fk/
       - run:
-          name: Generate SF0.003 / BI / composite-merged CSVs
+          name: Generate SF0.003 / BI / composite-merged CSVs, generate factors
           command: |
             # we generate factors here but they are moved to a separate archive (social-network-sf0.003-bi-factors.zip)
-            tools/docker-run.sh --mode bi --scale-factor 0.003 --generate-factors
+            tools/docker-run.sh -- --mode bi --scale-factor 0.003 --generate-factors
             mv out/ social-network-sf0.003-bi-composite-merged-fk/
       - run:
           name: Generate SF0.003 / BI / compressed composite-merged CSVs for Postgres
           command: |
-            tools/docker-run.sh --mode bi --scale-factor 0.003 --format-options compression=gzip
+            tools/docker-run.sh -- --mode bi --scale-factor 0.003 --format-options compression=gzip
             mv out/ social-network-sf0.003-bi-composite-merged-fk-postgres-compressed/
       - run:
           name: Generate SF0.003 / BI / composite-projected CSVs for Neo4j
           command: |
-            tools/docker-run.sh --mode bi --scale-factor 0.003 --explode-edges --format-options header=false,quoteAll=true
+            tools/docker-run.sh -- --mode bi --scale-factor 0.003 --explode-edges --format-options header=false,quoteAll=true
             mv out/ social-network-sf0.003-bi-composite-projected-fk-neo4j/
       - run:
           name: Generate SF0.003 / BI / compressed composite-projected CSVs for Neo4j
           command: |
-            tools/docker-run.sh --mode bi --scale-factor 0.003 --explode-edges --format-options header=false,quoteAll=true,compression=gzip
+            tools/docker-run.sh -- --mode bi --scale-factor 0.003 --explode-edges --format-options header=false,quoteAll=true,compression=gzip
             mv out/ social-network-sf0.003-bi-composite-projected-fk-neo4j-compressed/
-      # Interactive
-      - run:
-          name: Generate SF0.003 / Interactive / singular-projected CSVs
-          command: |
-            tools/docker-run.sh --mode interactive --scale-factor 0.003 --explode-edges --explode-attrs
-            mv out/ social-network-sf0.003-interactive-singular-projected-fk/
-      - run:
-          name: Generate SF0.003 / Interactive / singular-merged CSVs
-          command: |
-            tools/docker-run.sh --mode interactive --scale-factor 0.003 --explode-attrs
-            mv out/ social-network-sf0.003-interactive-singular-merged-fk/
       - run:
-          name: Generate SF0.003 / Interactive / composite-projected CSVs
+          name: Generate SF0.003 / BI / compressed composite-projected CSVs for Neo4j with epoch milli timestamps
           command: |
-            tools/docker-run.sh --mode interactive --scale-factor 0.003 --explode-edges                
-            mv out/ social-network-sf0.003-interactive-composite-projected-fk/
+            tools/docker-run.sh -- --mode bi --scale-factor 0.003 --explode-edges --epoch-millis --format-options header=false,quoteAll=true,compression=gzip
+            mv out/ social-network-sf0.003-bi-composite-projected-fk-neo4j-compressed-epoch-millis/
       - run:
-          name: Generate SF0.003 / Interactive / composite-merged CSVs
+          name: Generate SF0.003 / BI / compressed composite-projected CSVs for Neo4j with epoch milli timestamps
           command: |
-            tools/docker-run.sh --mode interactive --scale-factor 0.003                                
-            mv out/ social-network-sf0.003-interactive-composite-merged-fk/
+            tools/docker-run.sh -- --mode bi --scale-factor 0.003 --format parquet
+            mv out/ social-network-sf0.003-bi-parquet
       - run:
           name: Compress directories and prepare for deployment
           command: |
             # include the CircleCI configuration in the deployed package to provide the 'filters' instructions (and prevent failed builds on the gh-pages branch)
             mv .circleci dist/
             # move factors to a separate directory
             mkdir social-network-sf0.003-bi-factors
-            mv social-network-sf0.003-bi-composite-merged-fk/factors social-network-sf0.003-bi-factors/factors
+            cp -r social-network-sf0.003-bi-composite-merged-fk/factors social-network-sf0.003-bi-factors/factors
             # compress each directory
             for d in social-network-sf0.003*; do
               echo "Generated with <https://github.com/ldbc/ldbc_snb_datagen_spark/commit/${CIRCLE_SHA1}>" > $d/README.md

.dockerignore

@@ -1,15 +1,13 @@
-tools/*
-
-target/*
+**/*.egg-info
+**/target/*
 
 *.swp
 *.crc
 *.log
-
-*.iml
-.travis.yml
+**/*.iml
 .idea/*
-.gitignore
+**/.gitignore
 .git/*
-
+.circleci
+**/.bloop
 Dockerfile

.editorconfig

@@ -0,0 +1,20 @@
+root = true
+
+# Unix-style newlines with a newline ending every file
+[*]
+end_of_line = lf
+insert_final_newline = true
+indent_style = space
+indent_size = 2
+trim_trailing_whitespace = true
+charset = utf-8
+
+[.py]
+indent_size = 4
+
+
+[.java]
+indent_size = 4
+
+[Makefile,*.mk]
+indent_style = tab

.gitignore

@@ -18,7 +18,6 @@ target/
 test_data/
 substitution_parameters/
 *.pyc
-scripts/
 *.iml
 /hadoop/
 /social_network/
@@ -208,3 +207,4 @@ dmypy.json
 .prof
 
 # End of https://www.toptal.com/developers/gitignore/api/python
+.bsp/

Dockerfile

@@ -1,27 +1,39 @@
-FROM bde2020/spark-master:3.1.1-hadoop3.2
+FROM eclipse-temurin:8 as build-jar
+
+RUN apt-get update
+RUN apt-get install -y curl
+
+ARG SBT_VERSION=1.5.2
+RUN cd /opt && curl -fSsL https://github.com/sbt/sbt/releases/download/v${SBT_VERSION}/sbt-${SBT_VERSION}.tgz | tar xvz
+ENV PATH=/opt/sbt/bin:$PATH
+WORKDIR build
+COPY build.sbt build.sbt
+COPY project project
+RUN sbt update
+COPY src src
+RUN sbt assembly
+
+FROM scratch as jar
+COPY --from=build-jar /build/target/ldbc_snb_datagen_*-jar-with-dependencies.jar /jar
+
+FROM python:3.7-slim as build-tools
+RUN pip install --no-cache virtualenv && virtualenv -p python3.7 /tools
+COPY tools build
+WORKDIR build
+RUN . /tools/bin/activate && pip install .
+
+FROM python:3.7-slim as tools
+COPY --from=build-tools /tools /tools
+
+FROM bde2020/spark-master:3.2.1-hadoop3.2 as standalone
+COPY --from=jar /jar /jar
+COPY --from=tools /tools /tools
+RUN ln -sf /usr/bin/python3 /tools/bin/python
 
-ENV GOSU_VERSION 1.12
-
-RUN apk add --no-cache su-exec
-RUN apk add shadow
-RUN [ -d /var/mail ] || mkdir /var/mail
-
-VOLUME /mnt/datagen.jar /mnt/params.ini /mnt/data
-
-WORKDIR /mnt/data
-
-# adjust these environment variables
 ENV TEMP_DIR /tmp
-ENV EXECUTOR_MEMORY "1G"
-ENV DRIVER_MEMORY "5G"
-
-# the SPARK_* variables are used by submit.sh to configure the Spark job
 ENV SPARK_LOCAL_DIRS ${TEMP_DIR}
-ENV SPARK_SUBMIT_ARGS --executor-memory ${EXECUTOR_MEMORY} --driver-memory ${DRIVER_MEMORY}
-ENV SPARK_APPLICATION_MAIN_CLASS ldbc.snb.datagen.LdbcDatagen
-ENV SPARK_MASTER_URL local[*]
-ENV SPARK_APPLICATION_JAR_LOCATION /mnt/datagen.jar
-
-COPY submit.sh /
+ENV PATH=/tools/bin:/spark/bin:$PATH
+ENV LDBC_SNB_DATAGEN_JAR=/jar
 
-ENTRYPOINT ["/bin/bash", "/submit.sh"]
+WORKDIR /
+ENTRYPOINT ["run.py"]