Merge pull request #84 from ldbc/build-docker-in-travis

Build Docker image on Travis #83

Author: szarnyasg

.travis.yml

@@ -3,18 +3,17 @@ jdk:
 - openjdk8
 python:
 - '2.7'
+env:
+  global:
+  - GH_REF=github.com/ldbc/ldbc_snb_datagen.git
+  - secure: UG0OiJIFnm9mlQvV62cRwZzRFclw1Tu1QTFTUvEVcyskFV7fvIZjVZ1ozvIs8arfVulHXnijWYOr4X76HqB4bEMCpuk47nTk6zA+8omC+IVML1C8Mq0yT7L0k9iCShPOylAtvTqUMtrgewYXINqsQQxunqwWsIoeIdFh3wC+Km+0k2/NlxGlsCj4ajP1reGcYxj9b51mjskAC4EueJn0mgd7uFS3C92hJEaFovKUykpnVx9Vx54ssm2QkTtijdGiEUtps8b93dQQ0wQ/KShTQ6HiHr60Z0pt5PGThVT7EeO9RSHEJjIGhK+UKEf6ab0yxWjQxoIGjXEE19orVA+hLiHq9nnAQK293Y6+J6Qc6s+HI2PTrymI3kQGi3KcTN/K9ukW2wfulxX4R/nTFAtWJK4MMmMBahC+t2dSeKDKbq0yzFfFaSgFPLfDcfe/aQ/YBWFDhbvAqWmguMTcwreNeZMvMNLDdRCY3uXMOjqVEhxbJItbyhXCYTnPgdDDKcaJO2GaOadSzZdomwFhqpKjOca9ynVes1NX1Q+jv9E1862l9D8gkWGiMekMW2IhaVZTQvp1V77TCJ+az4jJVmTPdMAjiQvyisKFzC39xYUix11fYuZOTKQjPB1r4CUwVTTS649n6VwGukE0OdnzEFqbMHql6iV4ODQlHEO+syNbAOM=
+services:
+- docker
+before_install:
+- docker build . --tag ldbc/datagen
 install: true
 script:
-- mvn test | grep "^\\[" | tee mvn.log
-# test if the output of the mvn command contained an "[INFO] BUILD SUCCESS" entry
-- grep 'BUILD SUCCESS' mvn.log
-# generate SF1 data set
-- wget -q http://archive.apache.org/dist/hadoop/core/hadoop-2.6.0/hadoop-2.6.0.tar.gz
-- tar xf hadoop-2.6.0.tar.gz
-- export HADOOP_CLIENT_OPTS="-Xmx2G"
-- export HADOOP_HOME=`pwd`/hadoop-2.6.0
-- export LDBC_SNB_DATAGEN_HOME=`pwd`
-- ./run.sh
+- docker run --rm --mount type=bind,source="$(pwd)/",target="/opt/ldbc_snb_datagen/out" --mount type=bind,source="$(pwd)/params.ini",target="/opt/ldbc_snb_datagen/params.ini" ldbc/datagen
 - mkdir out
 - cp -r substitution_parameters out/
 notifications:

Dockerfile

@@ -1,9 +1,11 @@
-FROM openjdk:8-jdk-alpine
+FROM openjdk:8-jdk-stretch
 
 # Download hadoop
 WORKDIR /opt
-RUN apk add bash curl maven python
+RUN apt-get update
+RUN apt-get install -y bash curl maven python
 RUN curl -L 'http://archive.apache.org/dist/hadoop/core/hadoop-2.6.0/hadoop-2.6.0.tar.gz' | tar -xz
+RUN curl -L 'https://julialang-s3.julialang.org/bin/linux/x64/1.2/julia-1.2.0-linux-x86_64.tar.gz' | tar -xz
 
 # Copy the project
 COPY . /opt/ldbc_snb_datagen
@@ -14,4 +16,5 @@ RUN rm params*.ini
 RUN mvn -DskipTests clean assembly:assembly
 
 ENV HADOOP_CLIENT_OPTS '-Xmx8G'
+ENV PATH "/opt/julia-1.2.0/bin:${PATH}"
 CMD /opt/ldbc_snb_datagen/docker_run.sh

README.md

@@ -57,10 +57,11 @@ To configure the amount of memory available, set the `HADOOP_CLIENT_OPTS` variab
 
 In order to run the container, a `params.ini` file is required. For reference, please see the `params*.ini` files in the repository. The file will be mounted in the container by the `--mount type=bind,source="$(pwd)/params.ini,target="/opt/ldbc_snb_datagen/params.ini"` option. If required, the source path can be set to a different path.
 
-The container outputs its results in the `/opt/ldbc_snb_datagen/out/` directory which contains two sub-directories, `social_network/` and `subsitution_parameters`. In order to save the results of the generation, a directory must be mounted in the container from the host. The driver requires the results be in the datagen repository directory.
+The container outputs its results in the `/opt/ldbc_snb_datagen/out/` directory which contains two sub-directories, `social_network/` and `subsitution_parameters`. In order to save the results of the generation, a directory must be mounted in the container from the host. The driver requires the results be in the datagen repository directory. To generate the data, run the following command which includes changing the owner (`chown`) of the Docker-mounted volumes:
 
 ```bash
-docker run --rm --mount type=bind,source="$(pwd)/",target="/opt/ldbc_snb_datagen/out" --mount type=bind,source="$(pwd)/params.ini",target="/opt/ldbc_snb_datagen/params.ini" ldbc/datagen
+docker run --rm --mount type=bind,source="$(pwd)/",target="/opt/ldbc_snb_datagen/out" --mount type=bind,source="$(pwd)/params.ini",target="/opt/ldbc_snb_datagen/params.ini" ldbc/datagen && \
+  sudo chown -R $USER:$USER social_network/ substitution_parameters/
 ```
 
 If you need to raise the memory limit, use the `-e HADOOP_CLIENT_OPTS="-Xmx..."` parameter to override the default value (`-Xmx8G`).