Release 0.20.0 (#76)

* Pass extra docker argument (#75)

* Pass extra docker argument

* Update data_pipelines_cli/cli_commands/compile.py

Co-authored-by: Andrzej Swatowski <33041023+swtwsk@users.noreply.github.com>

* Pass extra docker argument

* Pass extra docker argument

* Pass extra docker argument

Co-authored-by: p-pekala <piotr.pekala@getindata.com>
Co-authored-by: Andrzej Swatowski <33041023+swtwsk@users.noreply.github.com>

* FIX #76  - Bump version and CHANGELOG for release 0.20.0

Co-authored-by: Piotr Pękala <piotr_pekala@wp.pl>
Co-authored-by: p-pekala <piotr.pekala@getindata.com>
Co-authored-by: Andrzej Swatowski <33041023+swtwsk@users.noreply.github.com>
Co-authored-by: github-actions <github-actions@github.com>

Author: 4 people

CHANGELOG.md

@@ -2,6 +2,10 @@
 
 ## [Unreleased]
 
+## [0.20.0] - 2022-05-04
+
+-   `--docker-args` has been added to `dp compile`
+
 ## [0.19.0] - 2022-04-25
 
 ### Added
@@ -197,7 +201,9 @@
 -   Draft of `dp init`, `dp create`, `dp template new`, `dp template list` and `dp dbt`
 -   Draft of `dp compile` and `dp deploy`
 
-[Unreleased]: https://github.com/getindata/data-pipelines-cli/compare/0.19.0...HEAD
+[Unreleased]: https://github.com/getindata/data-pipelines-cli/compare/0.20.0...HEAD
+
+[0.20.0]: https://github.com/getindata/data-pipelines-cli/compare/0.19.0...0.20.0
 
 [0.19.0]: https://github.com/getindata/data-pipelines-cli/compare/0.18.0...0.19.0
 

data_pipelines_cli/__init__.py

@@ -5,4 +5,4 @@
 pipelines.
 """
 
-version = "0.19.0"
+version = "0.20.0"

data_pipelines_cli/cli_commands/compile.py

@@ -1,6 +1,7 @@
+import json
 import pathlib
 import shutil
-from typing import Optional
+from typing import Dict, Optional
 
 import click
 import yaml
@@ -37,7 +38,9 @@ def _docker_build(docker_args: DockerArgs) -> None:
     docker_client = docker.from_env()
     docker_tag = docker_args.docker_build_tag()
     try:
-        _, logs_generator = docker_client.images.build(path=".", tag=docker_tag)
+        _, logs_generator = docker_client.images.build(
+            path=".", tag=docker_tag, buildargs=docker_args.build_args
+        )
         DockerResponseReader(logs_generator).click_echo_ok_responses()
     except docker.errors.BuildError as err:
         build_log = "\n".join([str(log) for log in err.build_log])
@@ -86,7 +89,12 @@ def _replace_datahub_with_jinja_vars(env: str) -> None:
         yaml.dump(updated_config, datahub_config_file)
 
 
-def compile_project(env: str, docker_tag: Optional[str] = None, docker_build: bool = False) -> None:
+def compile_project(
+    env: str,
+    docker_tag: Optional[str] = None,
+    docker_build: bool = False,
+    docker_build_args: Optional[Dict[str, str]] = None,
+) -> None:
     """
     Create local working directories and build artifacts.
 
@@ -101,7 +109,7 @@ def compile_project(env: str, docker_tag: Optional[str] = None, docker_build: bo
     copy_dag_dir_to_build_dir()
     copy_config_dir_to_build_dir()
 
-    docker_args = DockerArgs(env, docker_tag)
+    docker_args = DockerArgs(env, docker_tag, docker_build_args or {})
     replace_image_settings(docker_args)
     _replace_datahub_with_jinja_vars(env)
 
@@ -133,5 +141,10 @@ def compile_project(env: str, docker_tag: Optional[str] = None, docker_build: bo
 @click.option(
     "--docker-tag", type=str, required=False, help="Image tag of a Docker image to create"
 )
-def compile_project_command(env: str, docker_build: bool, docker_tag: Optional[str]) -> None:
-    compile_project(env, docker_tag, docker_build)
+@click.option(
+    "--docker-args", type=str, required=False, help="Args required to build project in json format"
+)
+def compile_project_command(
+    env: str, docker_build: bool, docker_tag: Optional[str], docker_args: Optional[str]
+) -> None:
+    compile_project(env, docker_tag, docker_build, json.loads(docker_args or "{}"))

data_pipelines_cli/cli_commands/deploy.py

@@ -44,7 +44,7 @@ def __init__(
         provider_kwargs_dict: Optional[Dict[str, Any]],
         datahub_ingest: bool,
     ) -> None:
-        self.docker_args = DockerArgs(env, None) if docker_push else None
+        self.docker_args = DockerArgs(env, None, {}) if docker_push else None
         self.datahub_ingest = datahub_ingest
         self.provider_kwargs_dict = provider_kwargs_dict or {}
         self.env = env

data_pipelines_cli/data_structures.py

@@ -66,10 +66,12 @@ class DockerArgs:
     """URI of the Docker images repository"""
     image_tag: str
     """An image tag"""
+    build_args: Dict[str, str]
 
-    def __init__(self, env: str, image_tag: Optional[str]) -> None:
+    def __init__(self, env: str, image_tag: Optional[str], build_args: Dict[str, str]) -> None:
         self.repository = self._get_docker_repository_uri_from_k8s_config(env)
         self.image_tag = self._get_image_tag_from_k8s_config(env, image_tag)
+        self.build_args = build_args
 
     def docker_build_tag(self) -> str:
         """

setup.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 0.19.0
+current_version = 0.20.0
 
 [bumpversion:file:setup.py]
 

setup.py

@@ -61,7 +61,7 @@
 
 setup(
     name="data_pipelines_cli",
-    version="0.19.0",
+    version="0.20.0",
     description="CLI for data platform",
     long_description=README,
     long_description_content_type="text/markdown",

tests/test_data_structures.py

@@ -68,7 +68,7 @@ def test_build_tag(self, mock_git_revision_hash):
         mock_git_revision_hash.return_value = commit_sha
 
         with patch("data_pipelines_cli.cli_constants.BUILD_DIR", self.build_temp_dir):
-            docker_args = DockerArgs("base", None)
+            docker_args = DockerArgs("base", None, {})
 
         self.assertEqual(f"{repository}:{commit_sha}", docker_args.docker_build_tag())
         self.assertEqual(repository, docker_args.repository)
@@ -79,7 +79,7 @@ def test_given_tag(self):
         image_tag = "my_awesome_tag_eee440b_latest"
 
         with patch("data_pipelines_cli.cli_constants.BUILD_DIR", self.build_temp_dir):
-            docker_args = DockerArgs("base", image_tag)
+            docker_args = DockerArgs("base", image_tag, {})
 
         self.assertEqual(f"{repository}:{image_tag}", docker_args.docker_build_tag())
         self.assertEqual(repository, docker_args.repository)
@@ -90,7 +90,7 @@ def test_set_tag(self):
         image_tag = "some_test_tag_a1s2d3f"
 
         with patch("data_pipelines_cli.cli_constants.BUILD_DIR", self.build_temp_dir):
-            docker_args = DockerArgs("image_tag", None)
+            docker_args = DockerArgs("image_tag", None, {})
 
         self.assertEqual(f"{repository}:{image_tag}", docker_args.docker_build_tag())
         self.assertEqual(repository, docker_args.repository)
@@ -103,12 +103,12 @@ def test_no_repository(self, mock_git_revision_hash):
         mock_git_revision_hash.return_value = commit_sha
 
         with self.assertRaises(DataPipelinesError):
-            _ = DockerArgs("base", None)
+            _ = DockerArgs("base", None, {})
 
     @patch("data_pipelines_cli.data_structures.git_revision_hash")
     def test_no_git_hash(self, mock_git_revision_hash):
         mock_git_revision_hash.return_value = None
 
         with patch("data_pipelines_cli.cli_constants.BUILD_DIR", self.build_temp_dir):
             with self.assertRaises(DataPipelinesError):
-                _ = DockerArgs("base", None)
+                _ = DockerArgs("base", None, {})