remove dataset from Dockerfile.ci (#2088)

* remove dataset from Dockerfile.ci

* fix ci

* add set -e

* debug ci

Author: QiJune

.gitignore

@@ -39,6 +39,7 @@ share/python-wheels/
 MANIFEST
 *.ckpt
 elasticdl/go/bin/elasticdl_ps
+data/
 
 # PyInstaller
 #  Usually these files are written by a python script from a template

.travis.yml

@@ -79,6 +79,13 @@ jobs:
         - docker build --target ci -t elasticdl:ci
           -f elasticdl/docker/Dockerfile.ci
           --build-arg BASE_IMAGE="$BASE_IMAGE" .
+        - mkdir -p data && docker run --rm -it --net=host
+          -v $HOME/.cache:/root/.cache
+          -v $HOME/.keras/datasets:/root/.keras/datasets
+          -v $HOME/.kube:/root/.kube
+          -v $HOME/.minikube:/home/$USER/.minikube
+          -v $PWD:/work -w /work
+          elasticdl:dev bash -c "scripts/travis/gen_dataset.sh data"
         - |
           JOB_TYPES=(
               odps
@@ -87,7 +94,7 @@ jobs:
               predict
           )
           for JOB_TYPE in "${JOB_TYPES[@]}"; do
-              bash scripts/travis/run_job.sh $JOB_TYPE
+              bash scripts/travis/run_job.sh $JOB_TYPE $PWD/data
           done
 
 # Only build on this list of branches

elasticdl/docker/Dockerfile.ci

@@ -7,27 +7,6 @@ WORKDIR /
 COPY elasticdl/docker/bashrc /etc/bash.bashrc
 RUN chmod a+rx /etc/bash.bashrc
 
-# Generate data
-# This assumes that the data generation package is independent with the
-# rest part of ElasticDL.  The generated data will be in /data.
-RUN pip install pyrecordio>=0.0.6 && pip install sklearn && pip install pandas
-COPY elasticdl/python/data/recordio_gen/image_label.py /var/image_label.py
-RUN python /var/image_label.py --dataset mnist --fraction 0.15 \
-        --records_per_shard 4096 /data
-
-# Copy frappe dataset
-COPY elasticdl/python/data/recordio_gen/frappe_recordio_gen.py \
-     /var/frappe_recordio_gen.py
-RUN python /var/frappe_recordio_gen.py --data /root/.keras/datasets \
-    --output_dir /data/frappe \
-    --fraction 0.05
-# Copy heart dataset
-COPY elasticdl/python/data/recordio_gen/heart_recordio_gen.py \
-     /var/heart_recordio_gen.py
-RUN python /var/heart_recordio_gen.py --data_dir /root/.keras/datasets \
-    --output_dir /data/heart
-
-
 # Copy model zoo
 COPY model_zoo /model_zoo
 RUN python -m pip install --quiet -r /model_zoo/requirements.txt \
@@ -55,5 +34,5 @@ RUN python -m pip install --quiet /elasticdl-develop-py3-none-any.whl \
 ENV PATH /usr/local/lib/python3.6/dist-packages/elasticdl/go/bin:$PATH
 
 # Generate mnist checkpoint for evaluation and prediction
-COPY scripts/gen_mnist_checkpoint.py /gen_mnist_checkpoint.py
+COPY scripts/travis/gen_mnist_checkpoint.py /gen_mnist_checkpoint.py
 RUN python /gen_mnist_checkpoint.py --checkpoint_dir=/model_zoo/test_data/mnist_ckpt

scripts/client_test.sh

@@ -17,10 +17,10 @@
 JOB_TYPE=$1
 PS_NUM=$2
 WORKER_NUM=$3
+DATA_PATH=$4
 
 MNIST_CKPT_DIR=/model_zoo/test_data/mnist_ckpt/
 
-
 if [[ "$JOB_TYPE" == "train" ]]; then
     elasticdl train \
       --image_name=elasticdl:ci \
@@ -47,8 +47,8 @@ if [[ "$JOB_TYPE" == "train" ]]; then
       --job_name=test-train \
       --log_level=INFO \
       --image_pull_policy=Never \
-      --output=/saved_model/model_output \
-      --volume="host_path=${PWD},mount_path=/saved_model"
+      --output=/data/saved_model/model_output \
+      --volume="host_path=${DATA_PATH},mount_path=/data"
 elif [[ "$JOB_TYPE" == "evaluate" ]]; then
     elasticdl evaluate \
       --image_name=elasticdl:ci \
@@ -71,7 +71,8 @@ elif [[ "$JOB_TYPE" == "evaluate" ]]; then
       --tensorboard_log_dir=/tmp/tensorboard-log \
       --job_name=test-evaluate \
       --log_level=INFO \
-      --image_pull_policy=Never
+      --image_pull_policy=Never \
+      --volume="host_path=${DATA_PATH},mount_path=/data"
 elif [[ "$JOB_TYPE" == "predict" ]]; then
     elasticdl predict \
       --image_name=elasticdl:ci \
@@ -91,7 +92,8 @@ elif [[ "$JOB_TYPE" == "predict" ]]; then
       --num_ps_pods="$PS_NUM" \
       --job_name=test-predict \
       --log_level=INFO \
-      --image_pull_policy=Never
+      --image_pull_policy=Never \
+      --volume="host_path=${DATA_PATH},mount_path=/data"
 elif [[ "$JOB_TYPE" == "odps" ]]; then
     elasticdl train \
       --image_name=elasticdl:ci \

scripts/travis/gen_dataset.sh

@@ -0,0 +1,27 @@
+#!/usr/bin/env bash
+# Copyright 2020 The ElasticDL Authors. All rights reserved.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Generate mnist dataset
+DATA_PATH=$1
+
+python elasticdl/python/data/recordio_gen/image_label.py --dataset mnist --fraction 0.15 \
+        --records_per_shard 4096 "$DATA_PATH"
+
+# Generate frappe dataset
+python elasticdl/python/data/recordio_gen/frappe_recordio_gen.py --data /root/.keras/datasets \
+    --output_dir "$DATA_PATH"/frappe --fraction 0.05
+
+# Generate heart dataset
+python elasticdl/python/data/recordio_gen/heart_recordio_gen.py --data_dir /root/.keras/datasets \
+    --output_dir "$DATA_PATH"/heart

scripts/gen_mnist_checkpoint.py renamed to scripts/travis/gen_mnist_checkpoint.py

@@ -12,7 +12,10 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+set -e
+
 JOB_TYPE=$1
+DATA_PATH=$2
 MAXCOMPUTE_TABLE="odps_integration_build_$TRAVIS_BUILD_NUMBER_$(date +%s)"
 
 if [[ "$JOB_TYPE" == "odps" ]] && \
@@ -38,7 +41,7 @@ else
         -v /home/"$USER"/.minikube/:/home/"$USER"/.minikube/ \
         -v "$PWD":/work \
         -w /work elasticdl:ci \
-        bash -c "scripts/client_test.sh $JOB_TYPE $PS_NUM $WORKER_NUM"
+        bash -c "scripts/client_test.sh $JOB_TYPE $PS_NUM $WORKER_NUM $DATA_PATH"
     python3 scripts/validate_job_status.py "$JOB_TYPE" "$PS_NUM" "$WORKER_NUM"
     if [[ "$JOB_TYPE" == "odps" ]]; then
         bash scripts/travis/cleanup_odps_table.sh