unify build

Author: dszakallas

.dockerignore

@@ -4,11 +4,10 @@
 *.swp
 *.crc
 *.log
-
 **/*.iml
 .idea/*
 **/.gitignore
 .git/*
 .circleci
-
+**/.bloop
 Dockerfile

Dockerfile

@@ -1,12 +1,13 @@
 FROM eclipse-temurin:8 as build-jar
-ARG MAVEN_VERSION=3.8.6
-COPY pom.xml /build/pom.xml
+ARG SBT_VERSION=1.5.2
+RUN cd /opt && curl -fSsL https://github.com/sbt/sbt/releases/download/v${SBT_VERSION}/sbt-${SBT_VERSION}.tgz | tar xvz
+ENV PATH=/opt/sbt/bin:$PATH
 WORKDIR build
-RUN cd /opt && curl https://dlcdn.apache.org/maven/maven-3/${MAVEN_VERSION}/binaries/apache-maven-${MAVEN_VERSION}-bin.tar.gz | tar xvz
-ENV PATH=/opt/apache-maven-${MAVEN_VERSION}/bin:$PATH
-RUN mvn install
-COPY src /build/src
-RUN mvn assembly:assembly -DskipTests
+COPY build.sbt build.sbt
+COPY project project
+RUN sbt update
+COPY src src
+RUN sbt assembly
 
 FROM scratch as jar
 COPY --from=build-jar /build/target/ldbc_snb_datagen_*-jar-with-dependencies.jar /jar

README.md

@@ -68,7 +68,7 @@ Both Java 8 and Java 11 are supported.
 Once you have Spark in place and built the JAR file, run the generator as follows:
 
 ```bash
-export PLATFORM_VERSION=spark3.2_2.12
+export PLATFORM_VERSION=$(sbt -batch -error 'print platformVersion')
 export DATAGEN_VERSION=$(sbt -batch -error 'print version')
 export LDBC_SNB_DATAGEN_JAR=$(sbt -batch -error 'print assembly / assemblyOutputPath')
 ./tools/run.py <runtime configuration arguments> -- <generator configuration arguments>

build.sbt

@@ -14,9 +14,11 @@ ThisBuild / Test / fork := true
 
 val sparkVersion = settingKey[String]("The version of Spark used for building.")
 val sparkCompatVersion = taskKey[String]("The compatibility version of Spark")
+val platformVersion = taskKey[String]("The version of the target platform")
 
 sparkVersion := "3.2.1"
 sparkCompatVersion := { sparkVersion.value.split("\\.", 3).take(2).mkString(".") }
+platformVersion := { scalaBinaryVersion.value + "_spark" + sparkCompatVersion.value }
 
 resolvers += "TUDelft Repository" at "https://simulation.tudelft.nl/maven/"
 
@@ -27,7 +29,7 @@ libraryDependencies ++= Seq(
   "com.github.scopt" %%  "scopt" % "3.7.1",
   "org.javatuples" %  "javatuples" % "1.2",
   "ca.umontreal.iro" %  "ssj" % "2.5",
-  "xerces" %  "xercesImpl" % "2.12.0" % Runtime,
+  "xml-apis" %  "xml-apis" % "1.4.01",
   "org.specs2" %%  "specs2-core" % "4.2.0" % Test,
   "org.specs2" %%  "specs2-junit" % "4.2.0" % Test,
   "org.mockito" %  "mockito-core" % "3.3.3" % Test,
@@ -99,7 +101,10 @@ assembly / assemblyMergeStrategy := {
   case _ => MergeStrategy.first
 }
 
+// Override JAR name
 assembly / assemblyJarName := {
-  moduleName.value + "-spark" + sparkCompatVersion.value + "_" +
-    scalaBinaryVersion.value + "-" + version.value + ".assembly.jar"
-}
+  moduleName.value + "_" + platformVersion.value + "-" + version.value + "-jar-with-dependencies.jar"
+}
+
+// Put under target instead of target/<scala-binary-version>
+assembly / target := { target.value }

tools/build.sh

@@ -1,3 +1 @@
-platform_version = "2.12_spark3.2"
-version = "0.5.0-SNAPSHOT"
 main_class = 'ldbc.snb.datagen.LdbcDatagen'

tools/datagen/lib.py

@@ -28,12 +28,14 @@ Install the required libraries as described in the [main README](../../README.md
 
 ## Submitting a job
 
-1. Upload the JAR to S3. (We don't version the JARs yet, so you can only make sure that you run the intended code this way :( ) 
+1. Upload the JAR to S3. (We don't version the JARs yet, so you can only make sure that you run the intended code this way :( )
 
 ```bash
-PLATFORM_VERSION=2.12_spark3.1
-DATAGEN_VERSION=0.5.0-SNAPSHOT
-aws s3 cp target/ldbc_snb_datagen_${PLATFORM_VERSION}-${DATAGEN_VERSION}-jar-with-dependencies.jar s3://${BUCKET_NAME}/jars/ldbc_snb_datagen_${PLATFORM_VERSION}-${DATAGEN_VERSION}-jar-with-dependencies.jar
+export PLATFORM_VERSION=$(sbt -batch -error 'print platformVersion')
+export DATAGEN_VERSION=$(sbt -batch -error 'print version')
+export LDBC_SNB_DATAGEN_JAR=$(sbt -batch -error 'print assembly / assemblyOutputPath')
+export JAR_NAME=$(basename ${LDBC_SNB_DATAGEN_JAR})
+aws s3 cp ${LDBC_SNB_DATAGEN_JAR} s3://${BUCKET_NAME}/jars/$JAR_NAME
 ```
 
 1. Submit the job. Run with `--help` for customization options.
@@ -43,6 +45,7 @@ JOB_NAME=MyTest
 SCALE_FACTOR=10
 ./tools/emr/submit_datagen_job.py \
     --bucket ${BUCKET_NAME} \
+    --jar ${JAR_NAME} \
     ${JOB_NAME} \
     ${SCALE_FACTOR} \
     csv \
@@ -59,6 +62,7 @@ To use spot instances, add the `--use-spot` argument:
 ./tools/emr/submit_datagen_job.py \
     --use-spot \
     --bucket ${BUCKET_NAME} \
+    --jar ${JAR_NAME} \
     ${JOB_NAME} \
     ${SCALE_FACTOR} \
     csv \
@@ -78,6 +82,7 @@ Generate the BI data set with the following configuration:
 ./tools/emr/submit_datagen_job.py \
     --use-spot \
     --bucket ${BUCKET_NAME} \
+    --jar ${JAR_NAME} \
     --az us-east-2c \
     --copy-all \
     ${JOB_NAME} \
@@ -99,7 +104,7 @@ Make sure that you have uploaded the right JAR first.
 PLATFORM_VERSION=2.12_spark3.1
 ./tools/emr/submit_datagen_job.py \
     --bucket ${BUCKET_NAME} \
-    --platform-version ${PLATFORM_VERSION} \
+    --jar ${JAR_NAME} \
     --emr-release emr-6.2.0 \
     ${JOB_NAME} \
     ${SCALE_FACTOR} \

tools/emr/README.md

@@ -9,7 +9,9 @@
 import __main__
 
 from math import ceil
+from botocore.credentials import subprocess
 from datagen import lib, util
+import subprocess
 
 import argparse
 
@@ -24,8 +26,6 @@
     'master_instance_type': 'r6gd.2xlarge',
     'instance_type': 'r6gd.4xlarge',
     'sf_ratio': 100.0,  # ratio of SFs and machines. a ratio of 250.0 for SF1000 yields 4 machines
-    'platform_version': lib.platform_version,
-    'version': lib.version,
     'az': 'us-west-2c',
     'yes': False,
     'ec2_key': None,
@@ -73,22 +73,21 @@ def submit_datagen_job(name,
                        format,
                        mode,
                        bucket,
+                       jar,
                        use_spot,
                        instance_type,
                        sf_ratio,
                        master_instance_type,
                        az,
                        emr_release,
-                       platform_version,
-                       version,
                        yes,
                        ec2_key,
                        conf,
                        copy_filter,
                        copy_all,
                        passthrough_args, **kwargs
                        ):
-    
+
     is_interactive = (not yes) and hasattr(__main__, '__file__')
 
     build_dir = '/ldbc_snb_datagen/build'
@@ -107,7 +106,7 @@ def submit_datagen_job(name,
     ts = datetime.utcnow()
     ts_formatted = ts.strftime('%Y%m%d_%H%M%S')
 
-    jar_url = f's3://{bucket}/jars/ldbc_snb_datagen_{platform_version}-{version}-jar-with-dependencies.jar'
+    jar_url = f's3://{bucket}/jars/{jar}'
 
     results_url = f's3://{bucket}/results/{name}'
     run_url = f'{results_url}/runs/{ts_formatted}'
@@ -241,12 +240,9 @@ def submit_datagen_job(name,
     parser.add_argument('--ec2-key',
                         default=defaults['ec2_key'],
                         help='EC2 key name for SSH connection')
-    parser.add_argument('--platform-version',
-                        default=defaults['platform_version'],
-                        help='The spark platform the JAR is compiled for formatted like {scala.compat.version}_spark{spark.compat.version}, e.g. 2.12_spark3.2')
-    parser.add_argument('--version',
-                        default=defaults['version'],
-                        help='LDBC SNB Datagen library version')
+    parser.add_argument('--jar',
+                        required=True,
+                        help='LDBC SNB Datagen library JAR name')
     parser.add_argument('--emr-release',
                         default=defaults['emr_release'],
                         help='The EMR release to use. E.g. emr-6.6.0')