Merge pull request #49 from getindata/release-0.11.0

Release 0.11.0

Author: swtwsk

.pre-commit-config.yaml

@@ -1,7 +1,7 @@
 repos:
 
 - repo: https://github.com/pre-commit/mirrors-isort
-  rev: v5.9.3
+  rev: v5.10.1
   hooks:
     - id: isort
 
@@ -33,7 +33,7 @@ repos:
       ]
 
 - repo: https://github.com/pre-commit/mirrors-mypy
-  rev: v0.930
+  rev: v0.931
   hooks:
     - id: mypy
       additional_dependencies:

CHANGELOG.md

@@ -2,6 +2,19 @@
 
 ## [Unreleased]
 
+## [0.11.0] - 2022-01-18
+
+### Added
+
+-   `dp update` command
+-   `dp publish` command for creation of dbt package out of the project.
+
+### Changed
+
+-   Docker response in `deploy` and `compile` gets printed as processed strings instead of plain dictionaries.
+-   `dp compile` parses content of `datahub.yml` and replaces Jinja variables in the form of `var` or `env_var`.
+-   `dags_path` is read from an enved `airflow.yml` file.
+
 ## [0.10.0] - 2022-01-12
 
 ### Changed
@@ -131,7 +144,9 @@
 -   Draft of `dp init`, `dp create`, `dp template new`, `dp template list` and `dp dbt`
 -   Draft of `dp compile` and `dp deploy`
 
-[Unreleased]: https://github.com/getindata/data-pipelines-cli/compare/0.10.0...HEAD
+[Unreleased]: https://github.com/getindata/data-pipelines-cli/compare/0.11.0...HEAD
+
+[0.11.0]: https://github.com/getindata/data-pipelines-cli/compare/0.10.0...0.11.0
 
 [0.10.0]: https://github.com/getindata/data-pipelines-cli/compare/0.9.0...0.10.0
 

README.md

@@ -47,6 +47,11 @@ directly to a template repository. If `<LINK_TO_TEMPLATE_REPOSITORY>` proves to
 
 `dp template-list` lists all added templates.
 
+### Project update
+
+To update your pipeline project use `dp update <PIPELINE_PROJECT-PATH>`. It will sync your existing project with updated
+template version selected by `--vcs-ref` option (default `HEAD`).
+
 ### Project deployment
 
 `dp deploy` will sync with your bucket provider. The provider will be chosen automatically based on the remote URL.

data_pipelines_cli/__init__.py

@@ -1 +1,8 @@
-version = "0.10.0"
+"""
+data-pipelines-cli (dp) is a CLI tool designed for data platform.
+
+dp helps data analysts to create, maintain and make full use of their data
+pipelines.
+"""
+
+version = "0.11.0"

data_pipelines_cli/cli.py

@@ -8,9 +8,11 @@
 from .cli_commands.deploy import deploy_command
 from .cli_commands.init import init_command
 from .cli_commands.prepare_env import prepare_env_command
+from .cli_commands.publish import publish_command
 from .cli_commands.run import run_command
 from .cli_commands.template import list_templates_command
 from .cli_commands.test import test_command
+from .cli_commands.update import update_command
 from .cli_utils import echo_error
 from .errors import DataPipelinesError
 
@@ -35,6 +37,8 @@ def cli() -> None:
 _cli.add_command(deploy_command)
 _cli.add_command(init_command)
 _cli.add_command(prepare_env_command)
+_cli.add_command(publish_command)
 _cli.add_command(run_command)
 _cli.add_command(list_templates_command)
 _cli.add_command(test_command)
+_cli.add_command(update_command)

data_pipelines_cli/cli_commands/clean.py

@@ -19,7 +19,7 @@ def _remove_build_dir() -> None:
 
 
 def clean() -> None:
-    """Deletes local working directories"""
+    """Delete local working directories."""
     _dbt_clean()
     _remove_build_dir()
 

data_pipelines_cli/cli_commands/compile.py

@@ -2,25 +2,25 @@
 import shutil
 
 import click
+import yaml
 
 from ..cli_constants import BUILD_DIR, IMAGE_TAG_TO_REPLACE
-from ..cli_utils import echo_info
+from ..cli_utils import echo_error, echo_info, echo_warning
 from ..config_generation import (
     copy_config_dir_to_build_dir,
     copy_dag_dir_to_build_dir,
     generate_profiles_yml,
 )
 from ..data_structures import DockerArgs
-from ..dbt_utils import run_dbt_command
-from ..errors import DockerNotInstalledError
+from ..dbt_utils import read_dbt_vars_from_configs, run_dbt_command
+from ..docker_response_reader import DockerResponseReader
+from ..errors import (
+    DataPipelinesError,
+    DockerErrorResponseError,
+    DockerNotInstalledError,
+)
 from ..io_utils import replace
-
-
-def _replace_image_tag(k8s_config: pathlib.Path, docker_args: DockerArgs) -> None:
-    echo_info(
-        f"Replacing {IMAGE_TAG_TO_REPLACE} with commit SHA = {docker_args.commit_sha}"
-    )
-    replace(k8s_config, IMAGE_TAG_TO_REPLACE, docker_args.commit_sha)
+from ..jinja import replace_vars_with_values
 
 
 def _docker_build(docker_args: DockerArgs) -> None:
@@ -38,14 +38,15 @@ def _docker_build(docker_args: DockerArgs) -> None:
     docker_client = docker.from_env()
     docker_tag = docker_args.docker_build_tag()
     _, logs_generator = docker_client.images.build(path=".", tag=docker_tag)
-    click.echo(
-        "".join(
-            map(
-                lambda log: log["stream"],
-                filter(lambda log: "stream" in log, logs_generator),
-            )
+    try:
+        DockerResponseReader(logs_generator).click_echo_ok_responses()
+    except DockerErrorResponseError as err:
+        echo_error(err.message)
+        raise DataPipelinesError(
+            "Error raised when pushing Docker image. Ensure that "
+            "Docker image you try to push exists. Maybe try running "
+            "'dp compile' first?"
         )
-    )
 
 
 def _dbt_compile(env: str) -> None:
@@ -66,16 +67,40 @@ def _copy_dbt_manifest() -> None:
 
 
 def _replace_k8s_settings(docker_args: DockerArgs) -> None:
-    k8s_config: pathlib.Path = BUILD_DIR.joinpath("dag", "config", "base", "k8s.yml")
-    _replace_image_tag(k8s_config, docker_args)
+    k8s_config = BUILD_DIR.joinpath("dag", "config", "base", "k8s.yml")
+    echo_info(
+        f"Replacing {IMAGE_TAG_TO_REPLACE} with commit SHA = {docker_args.commit_sha}"
+    )
+    replace(k8s_config, IMAGE_TAG_TO_REPLACE, docker_args.commit_sha)
+
+
+def _replace_datahub_with_jinja_vars(env: str) -> None:
+    datahub_config_path: pathlib.Path = BUILD_DIR.joinpath(
+        "dag", "config", "base", "datahub.yml"
+    )
+
+    if not datahub_config_path.exists():
+        echo_warning(
+            f"File config/base/datahub.yml does not exist in {BUILD_DIR}. "
+            "Content will not be replaced."
+        )
+        return
+
+    echo_info(f"Replacing Jinja variables in {datahub_config_path}.")
+    with open(datahub_config_path, "r") as datahub_config_file:
+        updated_config = replace_vars_with_values(
+            yaml.safe_load(datahub_config_file), read_dbt_vars_from_configs(env)
+        )
+    with open(datahub_config_path, "w") as datahub_config_file:
+        yaml.dump(updated_config, datahub_config_file)
 
 
 def compile_project(
     env: str,
     docker_build: bool = False,
 ) -> None:
     """
-    Create local working directories and build artifacts
+    Create local working directories and build artifacts.
 
     :param env: Name of the environment
     :type env: str
@@ -88,6 +113,7 @@ def compile_project(
 
     docker_args = DockerArgs(env)
     _replace_k8s_settings(docker_args)
+    _replace_datahub_with_jinja_vars(env)
 
     _dbt_compile(env)
     _copy_dbt_manifest()

data_pipelines_cli/cli_commands/create.py

@@ -43,7 +43,7 @@ def _get_template_path(
 
 def create(project_path: str, template_path: Optional[str]) -> None:
     """
-    Create a new project using a template
+    Create a new project using a template.
 
     :param project_path: Path to a directory to create
     :type project_path: str

data_pipelines_cli/cli_commands/deploy.py

@@ -6,20 +6,22 @@
 import yaml
 
 from ..cli_constants import BUILD_DIR
-from ..cli_utils import echo_info, subprocess_run
+from ..cli_utils import echo_error, echo_info, subprocess_run
 from ..config_generation import read_dictionary_from_config_directory
 from ..data_structures import DockerArgs
+from ..docker_response_reader import DockerResponseReader
 from ..errors import (
     AirflowDagsPathKeyError,
     DataPipelinesError,
     DependencyNotInstalledError,
+    DockerErrorResponseError,
     DockerNotInstalledError,
 )
 from ..filesystem_utils import LocalRemoteSync
 
 
 class DeployCommand:
-    """A class used to push and deploy the project to the remote machine"""
+    """A class used to push and deploy the project to the remote machine."""
 
     docker_args: Optional[DockerArgs]
     """Arguments required by the Docker to make a push to the repository.
@@ -45,17 +47,16 @@ def __init__(
         self.provider_kwargs_dict = provider_kwargs_dict or {}
 
         try:
-            self.blob_address_path = (
-                dags_path
-                or read_dictionary_from_config_directory(
-                    BUILD_DIR.joinpath("dag"), "base", "airflow.yml"
-                )["dags_path"]
-            )
+            self.blob_address_path = dags_path or read_dictionary_from_config_directory(
+                BUILD_DIR.joinpath("dag"),
+                env,
+                "airflow.yml",
+            )["dags_path"]
         except KeyError as key_error:
             raise AirflowDagsPathKeyError from key_error
 
     def deploy(self) -> None:
-        """Push and deploy the project to the remote machine
+        """Push and deploy the project to the remote machine.
 
         :raises DependencyNotInstalledError: DataHub or Docker not installed
         :raises DataPipelinesError: Error while pushing Docker image
@@ -81,20 +82,23 @@ def _docker_push(self) -> None:
         echo_info("Pushing Docker image")
         docker_client = docker.from_env()
         docker_args = cast(DockerArgs, self.docker_args)
-        for line in docker_client.images.push(
-            repository=docker_args.repository,
-            tag=docker_args.commit_sha,
-            stream=True,
-            decode=True,
-        ):
-            click.echo(line)
-
-            if "error" in line:
-                raise DataPipelinesError(
-                    "Error raised when pushing Docker image. Ensure that "
-                    "Docker image you try to push exists. Maybe try running "
-                    "'dp compile' first?"
+
+        try:
+            DockerResponseReader(
+                docker_client.images.push(
+                    repository=docker_args.repository,
+                    tag=docker_args.commit_sha,
+                    stream=True,
+                    decode=True,
                 )
+            ).click_echo_ok_responses()
+        except DockerErrorResponseError as err:
+            echo_error(err.message)
+            raise DataPipelinesError(
+                "Error raised when pushing Docker image. Ensure that "
+                "Docker image you try to push exists. Maybe try running "
+                "'dp compile' first?"
+            )
 
     @staticmethod
     def _datahub_ingest() -> None:
@@ -125,7 +129,9 @@ def _sync_bucket(self) -> None:
     name="deploy",
     help="Push and deploy the project to the remote machine",
 )
-@click.option("--env", default="base", type=str, help="Name of the environment")
+@click.option(
+    "--env", default="base", show_default=True, type=str, help="Name of the environment"
+)
 @click.option("--dags-path", required=False, help="Remote storage URI")
 @click.option(
     "--blob-args",

data_pipelines_cli/cli_commands/init.py

@@ -25,7 +25,7 @@ def _download_global_config(config_path: str) -> DataPipelinesConfig:
 
 def init(config_path: Optional[str]) -> None:
     """
-    Configure the tool for the first time
+    Configure the tool for the first time.
 
     :param config_path: URI of the repository with a template of the config file
     :type config_path: Optional[str]