Add a script build.sh to build three pip packages (elasticdl/elasticdl_preprocessing/elasticdl_client) (#2093)

* Modify the tile of building models with structured data using ElasticDL

* Build wheel files using docker

* Format shell scripts

* Format shell command

* Move go unittests into unit_test.sh

* Fix permission to execute shell

* Fix by comments

* Fix shell

* fix shell syntax

* fix shell syntax

* Fix the shell file name

* Fix go unitttest

* Split build_and_test to 2 scripts

* Split build_and_test to 2 scripts

* Set -e for unit_test.sh

* Fix by comments

Author: workingloong

.travis.yml

@@ -56,7 +56,8 @@ jobs:
           -v $HOME/.kube:/root/.kube
           -v $HOME/.minikube:/home/$USER/.minikube
           -v $PWD:/work -w /work
-          elasticdl:dev bash -c "K8S_TESTS=True scripts/build_and_test.sh"
+          elasticdl:dev
+          bash -c "K8S_TESTS=True scripts/build_and_test.sh"
         # Report code coverage to https://codecov.io
         - bash <(curl -s https://codecov.io/bash)
         # Run unit tests related to ODPS (skipped for pull requests from forks)

scripts/build.sh

@@ -0,0 +1,51 @@
+#!/bin/bash
+# Copyright 2020 The ElasticDL Authors. All rights reserved.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -e
+
+# Build elasticdl.org/elasticdl Go package
+GO_PROXY_CN=https://goproxy.cn
+GO_PROXY_ALIYUN=https://developer.aliyun.com/mirror/goproxy
+go env -w GOPROXY="$GO_PROXY_CN","$GO_PROXY_ALIYUN",direct
+
+# NOTE: The following commands requires that the current working directory is
+# the source tree root.
+make -f elasticdl/Makefile
+
+# TODO(QiJune): How about we put the go.mod file in elasticdl/elasticdl/go/ in
+# the Git repo, so we don't need to create it at building time?
+(
+    cp -r elasticdl/go/ /tmp/elasticdl
+    cd "$GOPATH"/pkg/mod/github.com/tensorflow
+    go mod init github.com/tensorflow
+    cd /tmp/elasticdl
+    go mod init elasticdl.org/elasticdl
+    go mod edit -replace \
+        github.com/tensorflow="${GOPATH}"/pkg/mod/github.com/tensorflow
+    go get -u -t k8s.io/[email protected]
+    go mod tidy
+    GOBIN=/tmp go install ./...
+)
+
+# Create elasticdl_preprocessing package
+rm -rf ./build/lib
+python setup_preprocessing.py --quiet bdist_wheel --dist-dir ./build
+# Create elasticdl_client package
+rm -rf ./build/lib
+python setup_client.py --quiet bdist_wheel --dist-dir ./build
+# Create elasticdl package
+mkdir -p ./elasticdl/go/bin
+cp /tmp/elasticdl_ps ./elasticdl/go/bin/
+rm -rf ./build/lib
+python setup.py --quiet bdist_wheel --dist-dir ./build

scripts/build_and_test.sh

@@ -12,45 +12,17 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-set -e
 
-# Build elasticdl.org/elasticdl Go package
-GO_PROXY_CN=https://goproxy.cn
-GO_PROXY_ALIYUN=https://developer.aliyun.com/mirror/goproxy
-go env -w GOPROXY="$GO_PROXY_CN","$GO_PROXY_ALIYUN",direct
+set -e
 
-# NOTE: The following commands requires that the current working directory is
-# the source tree root.
-make -f elasticdl/Makefile
+sh scripts/build.sh
 
-# TODO(QiJune): How about we put the go.mod file in elasticdl/elasticdl/go/ in
-# the Git repo, so we don't need to create it at building time?
 (
-    cp -r elasticdl/go/ /tmp/elasticdl
-    cd "$GOPATH"/pkg/mod/github.com/tensorflow
-    go mod init github.com/tensorflow
     cd /tmp/elasticdl
-    go mod init elasticdl.org/elasticdl
-    go mod edit -replace github.com/tensorflow="${GOPATH}"/pkg/mod/github.com/tensorflow
-    go get -u -t k8s.io/[email protected]
-    go mod tidy
-    GOBIN=/tmp go install ./...
     go test -v -cover ./...
 )
 
-# Run Python unittests
-pytest elasticdl/python/tests elasticdl_preprocessing/tests --cov=elasticdl/python --cov-report=xml
+pytest elasticdl/python/tests elasticdl_preprocessing/tests \
+    --cov=elasticdl/python --cov-report=xml
 mkdir -p ./build
 mv coverage.xml ./build
-
-# Create elasticdl_preprocessing package
-rm -rf ./build/lib
-python setup_preprocessing.py --quiet bdist_wheel --dist-dir ./build
-# Create elasticdl_client package
-rm -rf ./build/lib
-python setup_client.py --quiet bdist_wheel --dist-dir ./build
-# Create elasticdl package
-mkdir -p ./elasticdl/go/bin
-cp /tmp/elasticdl_ps ./elasticdl/go/bin/
-rm -rf ./build/lib
-python setup.py --quiet bdist_wheel --dist-dir ./build

scripts/docker_build_wheel.sh

@@ -0,0 +1,21 @@
+#!/usr/bin/env bash
+# Copyright 2020 The ElasticDL Authors. All rights reserved.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+export BASE_IMAGE=tensorflow/tensorflow:2.1.0-py3
+
+docker build --target dev -t elasticdl:dev -f elasticdl/docker/Dockerfile \
+    --build-arg BASE_IMAGE="$BASE_IMAGE" .
+
+docker run --rm -it --net=host -v "$PWD":/work -w /work elasticdl:dev \
+    bash -c "scripts/build.sh"