Merge pull request #357 from ldbc/spark3.2

Bump Spark version to 3.2

Author: szarnyasg

.circleci/config.yml

@@ -29,7 +29,7 @@ jobs:
   test:
     executor: my-executor
     environment:
-      PLATFORM_VERSION: 2.12_spark3.1
+      PLATFORM_VERSION: 2.12_spark3.2
       DATAGEN_VERSION: 0.5.0-SNAPSHOT
     steps:
       - checkout

README.md

@@ -46,30 +46,30 @@ E.g. with [pyenv](https://github.com/pyenv/pyenv) and [pyenv-virtualenv](https:/
 pyenv install 3.7.13
 pyenv virtualenv 3.7.13 ldbc_datagen_tools
 pyenv local ldbc_datagen_tools
-pip install -U pip 
+pip install -U pip
 pip install ./tools
 ```
 ### Running locally
 
 The `./tools/run.py` script is intended for **local runs**. To use it, download and extract Spark as follows.
 
-#### Spark 3.1.x
+#### Spark 3.2.x
 
-Spark 3.1.x is the recommended runtime to use. The rest of the instructions are provided assuming Spark 3.1.x.
+Spark 3.2.x is the recommended runtime to use. The rest of the instructions are provided assuming Spark 3.2.x.
 
 To place Spark under `/opt/`:
 
 ```bash
-curl https://downloads.apache.org/spark/spark-3.1.2/spark-3.1.2-bin-hadoop3.2.tgz | sudo tar -xz -C /opt/
-export SPARK_HOME=/opt/spark-3.1.2-bin-hadoop3.2
+curl https://downloads.apache.org/spark/spark-3.2.1/spark-3.2.1-bin-hadoop3.2.tgz | sudo tar -xz -C /opt/
+export SPARK_HOME="/opt/spark-3.2.1-bin-hadoop3.2"
 export PATH="$SPARK_HOME/bin":"$PATH"
 ```
 
 To place under `~/`:
 
 ```bash
-curl https://downloads.apache.org/spark/spark-3.1.2/spark-3.1.2-bin-hadoop3.2.tgz | tar -xz -C ~/
-export SPARK_HOME=~/spark-3.1.2-bin-hadoop3.2
+curl https://downloads.apache.org/spark/spark-3.2.1/spark-3.2.1-bin-hadoop3.2.tgz | tar -xz -C ~/
+export SPARK_HOME=~/spark-3.2.1-bin-hadoop3.2
 export PATH="$SPARK_HOME/bin":"$PATH"
 ```
 
@@ -78,7 +78,7 @@ Both Java 8 and Java 11 are supported.
 Once you have Spark in place and built the JAR file, run the generator as follows:
 
 ```bash
-export PLATFORM_VERSION=2.12_spark3.1
+export PLATFORM_VERSION=2.12_spark3.2
 export DATAGEN_VERSION=0.5.0-SNAPSHOT
 ./tools/run.py ./target/ldbc_snb_datagen_${PLATFORM_VERSION}-${DATAGEN_VERSION}.jar <runtime configuration arguments> -- <generator configuration arguments>
 ```

pom.xml

@@ -16,8 +16,8 @@
     <maven.compiler.target>${java.version}</maven.compiler.target>
     <scala.version>2.12.15</scala.version>
     <scala.compat.version>2.12</scala.compat.version>
-    <spark.version>3.1.2</spark.version>
-    <spark.compat.version>3.1</spark.compat.version>
+    <spark.version>3.2.0</spark.version>
+    <spark.compat.version>3.2</spark.compat.version>
     <spec2.version>4.2.0</spec2.version>
   </properties>
 
@@ -294,7 +294,7 @@
 
   <profiles>
     <profile>
-      <id>spark3.1</id>
+      <id>spark3.2</id>
       <properties>
         <!-- This is the default profile. -->
       </properties>

tools/emr/submit_datagen_job.py

@@ -29,7 +29,7 @@
     'az': 'us-west-2c',
     'yes': False,
     'ec2_key': None,
-    'emr_release': 'emr-6.3.0'
+    'emr_release': 'emr-6.6.0'
 }
 
 pp = pprint.PrettyPrinter(indent=2)
@@ -243,13 +243,13 @@ def submit_datagen_job(name,
                         help='EC2 key name for SSH connection')
     parser.add_argument('--platform-version',
                         default=defaults['platform_version'],
-                        help='The spark platform the JAR is compiled for formatted like {scala.compat.version}_spark{spark.compat.version}, e.g. 2.12_spark3.1')
+                        help='The spark platform the JAR is compiled for formatted like {scala.compat.version}_spark{spark.compat.version}, e.g. 2.12_spark3.2')
     parser.add_argument('--version',
                         default=defaults['version'],
                         help='LDBC SNB Datagen library version')
     parser.add_argument('--emr-release',
                         default=defaults['emr_release'],
-                        help='The EMR release to use. E.g. emr-6.3.0')
+                        help='The EMR release to use. E.g. emr-6.6.0')
     parser.add_argument('-y', '--yes',
                         default=defaults['yes'],
                         action='store_true',