ElasticDL Client: Implement "zoo init | build | push" command. (#2082)

* Initial version for the parsers.

* Add unit test for argument parser.

* Add Api module for the command call of ElasticDL client.

* Initial implementation for zoo init and zoo build.

* BASE_IMAGE is not replaced by the variable.

* Update comments

* Add cluster_spec in Dockerfile.

* Add docker_base_url in command line arguments.

* Add args module to build the command line arguments.

* Add more description about the command arguments.

* Add more argument description.

* Do some re-format

* Add the process to copy cluster spec file to the current directory.

* Remove placeholder test cases.

Author: brightcoder01

.isort.cfg

@@ -1,5 +1,5 @@
 [settings]
 multi_line_output=3
 line_length=79
-known_third_party = PIL,deepctr,docker,google,grpc,kubernetes,numpy,odps,pandas,recordio,requests,setuptools,sklearn,tensorflow,yaml
+known_third_party = PIL,deepctr,docker,google,grpc,jinja2,kubernetes,numpy,odps,pandas,recordio,requests,setuptools,sklearn,tensorflow,yaml
 include_trailing_comma=True

elasticdl_client/api.py

@@ -0,0 +1,116 @@
+# Copyright 2020 The ElasticDL Authors. All rights reserved.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import shutil
+
+import docker
+from jinja2 import Template
+
+
+def init_zoo(args):
+    print("Create the Dockerfile for the model zoo.")
+
+    # Copy cluster spec file to the current directory if specified
+    cluster_spec_path = args.cluster_spec
+    cluster_spec_name = None
+    if cluster_spec_path:
+        if not os.path.exists(cluster_spec_path):
+            raise RuntimeError(
+                "The cluster spec {} doesn't exist".format(cluster_spec_path)
+            )
+        shutil.copy2(cluster_spec_path, os.getcwd())
+        cluster_spec_name = os.path.basename(cluster_spec_path)
+
+    # Create the docker file
+    # Build the content from the template and arguments
+    tmpl_str = """\
+FROM {{ BASE_IMAGE }} as base
+
+RUN pip install elasticdl_preprocessing
+RUN pip install elasticdl
+
+COPY . /model_zoo
+{% if EXTRA_PYPI_INDEX %}
+RUN pip install -r /model_zoo/requirements.txt\
+  --extra-index-url={{ EXTRA_PYPI_INDEX }}\
+{% else %}\
+RUN pip install -r /model_zoo/requirements.txt\
+{% endif %}
+
+{% if CLUSTER_SPEC_NAME  %}\
+COPY ./{{ CLUSTER_SPEC_NAME }} /cluster_spec/{{ CLUSTER_SPEC_NAME }}\
+{% endif %}
+"""
+    template = Template(tmpl_str)
+    docker_file_content = template.render(
+        BASE_IMAGE=args.base_image,
+        EXTRA_PYPI_INDEX=args.extra_pypi_index,
+        CLUSTER_SPEC_NAME=cluster_spec_name,
+    )
+
+    with open("./Dockerfile", mode="w+") as f:
+        f.write(docker_file_content)
+
+
+def build_zoo(args):
+    print("Build the image for the model zoo.")
+    # Call docker api to build the image
+    # Validate the image name schema
+    client = _get_docker_client(
+        docker_base_url=args.docker_base_url,
+        docker_tlscert=args.docker_tlscert,
+        docker_tlskey=args.docker_tlskey,
+    )
+    for line in client.build(
+        dockerfile="./Dockerfile",
+        path=args.path,
+        rm=True,
+        tag=args.image,
+        decode=True,
+    ):
+        _print_docker_progress(line)
+
+
+def push_zoo(args):
+    print("Push the image for the model zoo.")
+    # Call docker api to push the image to remote registry
+    client = _get_docker_client(
+        docker_base_url=args.docker_base_url,
+        docker_tlscert=args.docker_tlscert,
+        docker_tlskey=args.docker_tlskey,
+    )
+
+    for line in client.push(args.image, stream=True, decode=True):
+        _print_docker_progress(line)
+
+
+def _get_docker_client(docker_base_url, docker_tlscert, docker_tlskey):
+    if docker_tlscert and docker_tlskey:
+        tls_config = docker.tls.TLSConfig(
+            client_cert=(docker_tlscert, docker_tlskey)
+        )
+        return docker.APIClient(base_url=docker_base_url, tls=tls_config)
+    else:
+        return docker.APIClient(base_url=docker_base_url)
+
+
+def _print_docker_progress(line):
+    error = line.get("error", None)
+    if error:
+        raise RuntimeError("Docker image build: " + error)
+    stream = line.get("stream", None)
+    if stream:
+        print(stream, end="")
+    else:
+        print(line)

elasticdl_client/common/__init__.py

@@ -0,0 +1,12 @@
+# Copyright 2020 The ElasticDL Authors. All rights reserved.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

elasticdl_client/common/args.py

@@ -0,0 +1,78 @@
+# Copyright 2020 The ElasticDL Authors. All rights reserved.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+def add_zoo_init_arguments(parser):
+    parser.add_argument(
+        "--base_image",
+        type=str,
+        default="python:latest",
+        help="Base Docker image.",
+    )
+    parser.add_argument(
+        "--extra_pypi_index",
+        type=str,
+        help="The extra URLs of Python package repository indexes",
+        required=False,
+    )
+    parser.add_argument(
+        "--cluster_spec",
+        type=str,
+        help="The file that contains user-defined cluster specification,"
+        "the file path can be accessed by ElasticDL client.",
+        default="",
+    )
+
+
+def add_zoo_build_arguments(parser):
+    parser.add_argument(
+        "path", type=str, help="The path where the build context locate."
+    )
+    parser.add_argument(
+        "--image",
+        type=str,
+        required=True,
+        help="The name of the docker image we are building for"
+        "this model zoo.",
+    )
+    add_docker_arguments(parser)
+
+
+def add_zoo_push_arguments(parser):
+    parser.add_argument(
+        "image",
+        type=str,
+        help="The name of the docker image for this model zoo.",
+    )
+    add_docker_arguments(parser)
+
+
+def add_docker_arguments(parser):
+    parser.add_argument(
+        "--docker_base_url",
+        type=str,
+        help="URL to the Docker server",
+        default="unix://var/run/docker.sock",
+    )
+    parser.add_argument(
+        "--docker_tlscert",
+        type=str,
+        help="Path to Docker client cert",
+        default="",
+    )
+    parser.add_argument(
+        "--docker_tlskey",
+        type=str,
+        help="Path to Docker client key",
+        default="",
+    )

elasticdl_client/main.py

@@ -11,9 +11,44 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import argparse
+
+from elasticdl_client.api import build_zoo, init_zoo, push_zoo
+from elasticdl_client.common import args
+
+
+def build_argument_parser():
+    parser = argparse.ArgumentParser()
+    subparsers = parser.add_subparsers()
+    subparsers.required = True
+
+    # Initialize the parser for the `elasticdl zoo` commands
+    zoo_parser = subparsers.add_parser("zoo")
+    zoo_subparsers = zoo_parser.add_subparsers()
+    zoo_subparsers.required = True
+
+    # elasticdl zoo init
+    zoo_init_parser = zoo_subparsers.add_parser("init")
+    zoo_init_parser.set_defaults(func=init_zoo)
+    args.add_zoo_init_arguments(zoo_init_parser)
+
+    # elasticdl zoo build
+    zoo_build_parser = zoo_subparsers.add_parser("build")
+    zoo_build_parser.set_defaults(func=build_zoo)
+    args.add_zoo_build_arguments(zoo_build_parser)
+
+    # elasticdl zoo push
+    zoo_push_parser = zoo_subparsers.add_parser("push")
+    zoo_push_parser.set_defaults(func=push_zoo)
+    args.add_zoo_push_arguments(zoo_push_parser)
+
+    return parser
+
 
 def main():
-    pass
+    parser = build_argument_parser()
+    args, _ = parser.parse_known_args()
+    args.func(args)
 
 
 if __name__ == "__main__":

elasticdl_client/requirements.txt

@@ -1,2 +1,3 @@
 kubernetes==10.1.0
-docker
+docker==4.2.1
+Jinja2==2.11.2

elasticdl_client/tests/arg_parser_test.py

@@ -0,0 +1,71 @@
+# Copyright 2020 The ElasticDL Authors. All rights reserved.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import unittest
+
+from elasticdl_client.main import build_argument_parser
+
+
+class ArgParserTest(unittest.TestCase):
+    def setUp(self):
+        self._parser = build_argument_parser()
+
+    def test_parse_zoo_init(self):
+        args = ["zoo", "init"]
+        args = self._parser.parse_args(args)
+        self.assertEqual(args.base_image, "python:latest")
+        args.func(args)
+
+        args = ["zoo", "init", "--base_image=elasticdl:base"]
+        args = self._parser.parse_args(args)
+        self.assertEqual(args.base_image, "elasticdl:base")
+
+        with self.assertRaises(SystemExit):
+            args = ["zoo", "init", "--mock_param=mock_value"]
+            args = self._parser.parse_args(args)
+
+    def test_parse_zoo_build(self):
+        args = [
+            "zoo",
+            "build",
+            "--image=a_docker_registry/bright/elasticdl-wnd:1.0",
+            ".",
+        ]
+        args = self._parser.parse_args(args)
+        self.assertEqual(
+            args.image, "a_docker_registry/bright/elasticdl-wnd:1.0"
+        )
+        self.assertEqual(args.path, ".")
+
+        with self.assertRaises(SystemExit):
+            args = ["zoo", "build", "."]
+            args = self._parser.parse_args(args)
+
+        with self.assertRaises(SystemExit):
+            args = [
+                "zoo",
+                "build",
+                "--image=a_docker_registry/bright/elasticdl-wnd:1.0",
+            ]
+            args = self._parser.parse_args(args)
+
+    def test_parse_zoo_push(self):
+        args = ["zoo", "push", "a_docker_registry/bright/elasticdl-wnd:1.0"]
+        args = self._parser.parse_args(args)
+        self.assertEqual(
+            args.image, "a_docker_registry/bright/elasticdl-wnd:1.0"
+        )
+
+        with self.assertRaises(SystemExit):
+            args = ["zoo", "push"]
+            args = self._parser.parse_args(args)

setup_client.py

@@ -26,4 +26,11 @@
     python_requires=">=3.5",
     packages=find_packages(include=["elasticdl_client*"]),
     package_data={"": ["requirements.txt"]},
+    # TODO(brightcoder01): Use `elasticdl_client` as the binary name
+    # temporarily. After we move all the functinality from main package
+    # to elasticdl_client package, we will rename the binary name to
+    # `elasticdl` and then remove the entry_points in main package.
+    entry_points={
+        "console_scripts": ["elasticdl_client=elasticdl_client.main:main"]
+    },
 )