Merge branch 'main' into docker-image

Author: dszakallas

.circleci/config.yml

@@ -30,7 +30,7 @@ jobs:
   test:
     executor: my-executor
     environment:
-      PLATFORM_VERSION: 2.12_spark3.1
+      PLATFORM_VERSION: 2.12_spark3.2
       DATAGEN_VERSION: 0.5.0-SNAPSHOT
     steps:
       - checkout

Dockerfile

@@ -20,7 +20,7 @@ RUN . /tools/bin/activate && pip install .
 FROM python:3.7-slim as tools
 COPY --from=build-tools /tools /tools
 
-FROM bde2020/spark-master:3.1.1-hadoop3.2 as standalone
+FROM bde2020/spark-master:3.2.1-hadoop3.2 as standalone
 COPY --from=jar /jar /jar
 COPY --from=tools /tools /tools
 RUN ln -sf /usr/bin/python3 /tools/bin/python

README.md

@@ -53,23 +53,23 @@ pip install ./tools
 
 The `./tools/run.py` script is intended for **local runs**. To use it, download and extract Spark as follows.
 
-#### Spark 3.1.x
+#### Spark 3.2.x
 
-Spark 3.1.x is the recommended runtime to use. The rest of the instructions are provided assuming Spark 3.1.x.
+Spark 3.2.x is the recommended runtime to use. The rest of the instructions are provided assuming Spark 3.2.x.
 
 To place Spark under `/opt/`:
 
 ```bash
-curl https://downloads.apache.org/spark/spark-3.1.2/spark-3.1.2-bin-hadoop3.2.tgz | sudo tar -xz -C /opt/
-export SPARK_HOME=/opt/spark-3.1.2-bin-hadoop3.2
+curl https://downloads.apache.org/spark/spark-3.2.1/spark-3.2.1-bin-hadoop3.2.tgz | sudo tar -xz -C /opt/
+export SPARK_HOME="/opt/spark-3.2.1-bin-hadoop3.2"
 export PATH="$SPARK_HOME/bin":"$PATH"
 ```
 
 To place under `~/`:
 
 ```bash
-curl https://downloads.apache.org/spark/spark-3.1.2/spark-3.1.2-bin-hadoop3.2.tgz | tar -xz -C ~/
-export SPARK_HOME=~/spark-3.1.2-bin-hadoop3.2
+curl https://downloads.apache.org/spark/spark-3.2.1/spark-3.2.1-bin-hadoop3.2.tgz | tar -xz -C ~/
+export SPARK_HOME=~/spark-3.2.1-bin-hadoop3.2
 export PATH="$SPARK_HOME/bin":"$PATH"
 ```
 
@@ -78,7 +78,7 @@ Both Java 8 and Java 11 are supported.
 Once you have Spark in place and built the JAR file, run the generator as follows:
 
 ```bash
-export PLATFORM_VERSION=2.12_spark3.1
+export PLATFORM_VERSION=2.12_spark3.2
 export DATAGEN_VERSION=0.5.0-SNAPSHOT
 export LDBC_SNB_DATAGEN_JAR=./target/ldbc_snb_datagen_${PLATFORM_VERSION}-${DATAGEN_VERSION}.jar
 ./tools/run.py <runtime configuration arguments> -- <generator configuration arguments>

pom.xml

@@ -16,8 +16,8 @@
     <maven.compiler.target>${java.version}</maven.compiler.target>
     <scala.version>2.12.15</scala.version>
     <scala.compat.version>2.12</scala.compat.version>
-    <spark.version>3.1.2</spark.version>
-    <spark.compat.version>3.1</spark.compat.version>
+    <spark.version>3.2.0</spark.version>
+    <spark.compat.version>3.2</spark.compat.version>
     <spec2.version>4.2.0</spec2.version>
   </properties>
 
@@ -305,7 +305,7 @@
 
   <profiles>
     <profile>
-      <id>spark3.1</id>
+      <id>spark3.2</id>
       <properties>
         <!-- This is the default profile. -->
       </properties>

tools/datagen/lib.py

@@ -1,3 +1,3 @@
-platform_version = "2.12_spark3.1"
+platform_version = "2.12_spark3.2"
 version = "0.5.0-SNAPSHOT"
 main_class = 'ldbc.snb.datagen.LdbcDatagen'

tools/emr/submit_datagen_job.py

@@ -29,7 +29,7 @@
     'az': 'us-west-2c',
     'yes': False,
     'ec2_key': None,
-    'emr_release': 'emr-6.3.0'
+    'emr_release': 'emr-6.6.0'
 }
 
 pp = pprint.PrettyPrinter(indent=2)
@@ -243,13 +243,13 @@ def submit_datagen_job(name,
                         help='EC2 key name for SSH connection')
     parser.add_argument('--platform-version',
                         default=defaults['platform_version'],
-                        help='The spark platform the JAR is compiled for formatted like {scala.compat.version}_spark{spark.compat.version}, e.g. 2.12_spark3.1')
+                        help='The spark platform the JAR is compiled for formatted like {scala.compat.version}_spark{spark.compat.version}, e.g. 2.12_spark3.2')
     parser.add_argument('--version',
                         default=defaults['version'],
                         help='LDBC SNB Datagen library version')
     parser.add_argument('--emr-release',
                         default=defaults['emr_release'],
-                        help='The EMR release to use. E.g. emr-6.3.0')
+                        help='The EMR release to use. E.g. emr-6.6.0')
     parser.add_argument('-y', '--yes',
                         default=defaults['yes'],
                         action='store_true',