Merge branch 'batching'

Author: szarnyasg

.travis.yml

@@ -13,10 +13,10 @@ before_install:
 - docker build . --tag ldbc/datagen
 install: true
 script:
-- mvn test
 - cp params-csv-basic.ini params.ini
 - docker run --rm --mount type=bind,source="$(pwd)/",target="/opt/ldbc_snb_datagen/out" --mount type=bind,source="$(pwd)/params.ini",target="/opt/ldbc_snb_datagen/params.ini" ldbc/datagen
-- bash check-md5sums-csv-basic.sh
+- md5sum social_network/*.csv | sort
+- "[[ `md5sum social_network/*.csv | sort | md5sum` == 'ee9e6dd99bf7c3459f4c6156d355f5ea  -' ]]"
 - mkdir out
 - cp -r substitution_parameters out/
 notifications:

README.md

@@ -42,11 +42,11 @@ To grab Hadoop, extract it, and set the environment values to sensible defaults,
 
 ```bash
 cp params-csv-basic.ini params.ini
-wget http://archive.apache.org/dist/hadoop/core/hadoop-2.9.2/hadoop-2.9.2.tar.gz
-tar xf hadoop-2.9.2.tar.gz
+wget http://archive.apache.org/dist/hadoop/core/hadoop-3.2.1/hadoop-3.2.1.tar.gz
+tar xf hadoop-3.2.1.tar.gz
 export HADOOP_CLIENT_OPTS="-Xmx2G"
-# set this to the Hadoop 2.9.2 directory
-export HADOOP_HOME=`pwd`/hadoop-2.9.2
+# set this to the Hadoop 3.2.1 directory
+export HADOOP_HOME=`pwd`/hadoop-3.2.1
 # set this to the repository's directory
 export LDBC_SNB_DATAGEN_HOME=`pwd`
 ./run.sh
@@ -66,10 +66,13 @@ docker build . --tag ldbc/datagen
 
 Set the `params.ini` in the repository as for the pseudo-distributed case. The file will be mounted in the container by the `--mount type=bind,source="$(pwd)/params.ini,target="/opt/ldbc_snb_datagen/params.ini"` option. If required, the source path can be set to a different path.
 
-The container outputs its results in the `/opt/ldbc_snb_datagen/out/` directory which contains two sub-directories, `social_network/` and `subsitution_parameters`. In order to save the results of the generation, a directory must be mounted in the container from the host. The driver requires the results be in the datagen repository directory. To generate the data, run the following command which includes changing the owner (`chown`) of the Docker-mounted volumes:
+The container outputs its results in the `/opt/ldbc_snb_datagen/out/` directory which contains two sub-directories, `social_network/` and `substitution_parameters`. In order to save the results of the generation, a directory must be mounted in the container from the host. The driver requires the results be in the datagen repository directory. To generate the data, run the following command which includes changing the owner (`chown`) of the Docker-mounted volumes.
+
+:warning: This removes the previously generated `social_network` directory:
 
 ```bash
-docker run --rm --mount type=bind,source="$(pwd)/",target="/opt/ldbc_snb_datagen/out" --mount type=bind,source="$(pwd)/params.ini",target="/opt/ldbc_snb_datagen/params.ini" ldbc/datagen && \
+rm -rf social_network/ substitution_parameters && \
+  docker run --rm --mount type=bind,source="$(pwd)/",target="/opt/ldbc_snb_datagen/out" --mount type=bind,source="$(pwd)/params.ini",target="/opt/ldbc_snb_datagen/params.ini" ldbc/datagen; \
   sudo chown -R $USER:$USER social_network/ substitution_parameters/
 ```
 

base-docker-image/Dockerfile

@@ -4,7 +4,7 @@ FROM openjdk:8-jdk-stretch
 WORKDIR /opt
 RUN apt-get update
 RUN apt-get install -y bash curl maven python
-RUN curl -L 'http://archive.apache.org/dist/hadoop/core/hadoop-2.9.2/hadoop-2.9.2.tar.gz' | tar -xz
+RUN curl -L 'http://archive.apache.org/dist/hadoop/core/hadoop-3.2.1/hadoop-3.2.1.tar.gz' | tar -xz
 RUN curl -L 'https://julialang-s3.julialang.org/bin/linux/x64/1.2/julia-1.2.0-linux-x86_64.tar.gz' | tar -xz
 
 # Copy the project

base-docker-image/pom.xml

@@ -40,7 +40,7 @@
     <dependency>
       <groupId>org.apache.hadoop</groupId>
       <artifactId>hadoop-client</artifactId>
-      <version>2.9.2</version>
+      <version>3.2.1</version>
     </dependency>
     <dependency>
       <groupId>ca.umontreal.iro</groupId>

docker_run.sh

@@ -8,7 +8,7 @@ if [ ! -f /opt/ldbc_snb_datagen/params.ini ]; then
 fi
 
 # Running the generator
-/opt/hadoop-2.9.2/bin/hadoop jar /opt/ldbc_snb_datagen/target/ldbc_snb_datagen-0.4.0-SNAPSHOT-jar-with-dependencies.jar /opt/ldbc_snb_datagen/params.ini
+/opt/hadoop-3.2.1/bin/hadoop jar /opt/ldbc_snb_datagen/target/ldbc_snb_datagen-0.4.0-SNAPSHOT-jar-with-dependencies.jar /opt/ldbc_snb_datagen/params.ini
 
 # Cleanup
 rm -f m*personFactors*

pom.xml

@@ -40,7 +40,7 @@
     <dependency>
       <groupId>org.apache.hadoop</groupId>
       <artifactId>hadoop-client</artifactId>
-      <version>2.9.2</version>
+      <version>3.2.1</version>
     </dependency>
     <dependency>
       <groupId>ca.umontreal.iro</groupId>

run.sh

@@ -5,7 +5,7 @@ if [ ! -f params.ini ]; then
   exit 1
 fi
 
-DEFAULT_HADOOP_HOME=/home/user/hadoop-2.9.2 #change to your hadoop folder
+DEFAULT_HADOOP_HOME=/home/user/hadoop-3.2.1 #change to your hadoop folder
 DEFAULT_LDBC_SNB_DATAGEN_HOME=`pwd` #change to your ldbc_snb_datagen folder
 
 # allow overriding configuration from outside via environment variables