Release 0.14.0 (#55)

* feat: execution env config

Author: github-actions[bot]

CHANGELOG.md

@@ -2,6 +2,8 @@
 
 ## [Unreleased]
 
+## [0.14.0] - 2022-02-02
+
 ## [0.13.0] - 2022-02-01
 
 ## [0.12.0] - 2022-01-31
@@ -150,7 +152,9 @@
 -   Draft of `dp init`, `dp create`, `dp template new`, `dp template list` and `dp dbt`
 -   Draft of `dp compile` and `dp deploy`
 
-[Unreleased]: https://github.com/getindata/data-pipelines-cli/compare/0.13.0...HEAD
+[Unreleased]: https://github.com/getindata/data-pipelines-cli/compare/0.14.0...HEAD
+
+[0.14.0]: https://github.com/getindata/data-pipelines-cli/compare/0.13.0...0.14.0
 
 [0.13.0]: https://github.com/getindata/data-pipelines-cli/compare/0.12.0...0.13.0
 

data_pipelines_cli/__init__.py

@@ -5,4 +5,4 @@
 pipelines.
 """
 
-version = "0.13.0"
+version = "0.14.0"

data_pipelines_cli/cli_commands/compile.py

@@ -66,18 +66,14 @@ def _copy_dbt_manifest() -> None:
     )
 
 
-def _replace_k8s_settings(docker_args: DockerArgs) -> None:
-    k8s_config = BUILD_DIR.joinpath("dag", "config", "base", "k8s.yml")
-    echo_info(
-        f"Replacing {IMAGE_TAG_TO_REPLACE} with commit SHA = {docker_args.commit_sha}"
-    )
+def replace_image_settings(docker_args: DockerArgs) -> None:
+    k8s_config = BUILD_DIR.joinpath("dag", "config", "base", "execution_env.yml")
+    echo_info(f"Replacing {IMAGE_TAG_TO_REPLACE} with commit SHA = {docker_args.commit_sha}")
     replace(k8s_config, IMAGE_TAG_TO_REPLACE, docker_args.commit_sha)
 
 
 def _replace_datahub_with_jinja_vars(env: str) -> None:
-    datahub_config_path: pathlib.Path = BUILD_DIR.joinpath(
-        "dag", "config", "base", "datahub.yml"
-    )
+    datahub_config_path: pathlib.Path = BUILD_DIR.joinpath("dag", "config", "base", "datahub.yml")
 
     if not datahub_config_path.exists():
         echo_warning(
@@ -112,7 +108,7 @@ def compile_project(
     copy_config_dir_to_build_dir()
 
     docker_args = DockerArgs(env)
-    _replace_k8s_settings(docker_args)
+    replace_image_settings(docker_args)
     _replace_datahub_with_jinja_vars(env)
 
     _dbt_compile(env)

data_pipelines_cli/cli_commands/create.py

@@ -65,7 +65,5 @@ def create(project_path: str, template_path: Optional[str]) -> None:
 @click.argument("template-path", nargs=-1)
 def create_command(project_path: str, template_path: Sequence[str]) -> None:
     if template_path and len(template_path) > 1:
-        echo_warning(
-            "dp create expects at most two arguments -- project-path and template-path"
-        )
+        echo_warning("dp create expects at most two arguments -- project-path and template-path")
     create(project_path, template_path[0] if template_path else None)

data_pipelines_cli/cli_commands/deploy.py

@@ -129,9 +129,7 @@ def _sync_bucket(self) -> None:
     name="deploy",
     help="Push and deploy the project to the remote machine",
 )
-@click.option(
-    "--env", default="base", show_default=True, type=str, help="Name of the environment"
-)
+@click.option("--env", default="base", show_default=True, type=str, help="Name of the environment")
 @click.option("--dags-path", required=False, help="Remote storage URI")
 @click.option(
     "--blob-args",

data_pipelines_cli/cli_commands/publish.py

@@ -34,9 +34,7 @@ def _get_database_and_schema_name(manifest: Manifest) -> Tuple[str, str]:
         raise DataPipelinesError("There is no model in 'manifest.json' file.")
 
 
-def _parse_columns_dict_into_table_list(
-    columns: Dict[str, ColumnInfo]
-) -> List[DbtTableColumn]:
+def _parse_columns_dict_into_table_list(columns: Dict[str, ColumnInfo]) -> List[DbtTableColumn]:
     return [
         DbtTableColumn(
             name=column.name,
@@ -64,16 +62,12 @@ def _parse_models_schema(manifest: Manifest) -> List[DbtModel]:
 
 
 def _get_dag_id() -> str:
-    with open(
-        BUILD_DIR.joinpath("dag", "config", "base", "airflow.yml"), "r"
-    ) as airflow_yml:
+    with open(BUILD_DIR.joinpath("dag", "config", "base", "airflow.yml"), "r") as airflow_yml:
         return yaml.safe_load(airflow_yml)["dag"]["dag_id"]
 
 
 def _create_source(project_name: str) -> DbtSource:
-    with open(
-        pathlib.Path.cwd().joinpath("target", "manifest.json"), "r"
-    ) as manifest_json:
+    with open(pathlib.Path.cwd().joinpath("target", "manifest.json"), "r") as manifest_json:
         manifest_dict = json.load(manifest_json)
         manifest = Manifest.from_dict(manifest_dict)
 
@@ -130,9 +124,7 @@ def _clean_repo(packages_repo: pathlib.Path) -> None:
         shutil.rmtree(packages_repo)
 
 
-def _copy_publication_to_repo(
-    package_dest: pathlib.Path, package_path: pathlib.Path
-) -> None:
+def _copy_publication_to_repo(package_dest: pathlib.Path, package_path: pathlib.Path) -> None:
     if package_dest.exists():
         echo_info(f"Removing {package_dest}")
         shutil.rmtree(package_dest)
@@ -145,14 +137,10 @@ def _configure_git_env(repo: Repo, config: Dict[str, Any]) -> None:
     repo.config_writer().set_value("user", "email", config["email"]).release()
 
 
-def _commit_and_push_changes(
-    repo: Repo, project_name: str, project_version: str
-) -> None:
+def _commit_and_push_changes(repo: Repo, project_name: str, project_version: str) -> None:
     echo_info("Publishing")
     repo.git.add(all=True)
-    repo.index.commit(
-        f"Publication from project {project_name}, version: {project_version}"
-    )
+    repo.index.commit(f"Publication from project {project_name}, version: {project_version}")
     origin = repo.remote(name="origin")
     origin.push()
 

data_pipelines_cli/cli_commands/template.py

@@ -13,8 +13,6 @@ def list_templates() -> None:
         click.echo(yaml.dump(tc))
 
 
-@click.command(
-    name="template-list", help="Print a list of all templates saved in the config file"
-)
+@click.command(name="template-list", help="Print a list of all templates saved in the config file")
 def list_templates_command() -> None:
     list_templates()

data_pipelines_cli/data_structures.py

@@ -91,14 +91,14 @@ def _get_docker_repository_uri_from_k8s_config(env: str) -> str:
             read_dictionary_from_config_directory,
         )
 
-        k8s_config = read_dictionary_from_config_directory(
-            BUILD_DIR.joinpath("dag"), env, "k8s.yml"
+        execution_env_config = read_dictionary_from_config_directory(
+            BUILD_DIR.joinpath("dag"), env, "execution_env.yml"
         )
         try:
-            return k8s_config["image"]["repository"]
+            return execution_env_config["image"]["repository"]
         except KeyError as key_error:
             raise DataPipelinesError(
-                f"Could not find 'repository' variable in build/config/{env}/k8s.yml."
+                f"Could not find 'repository' variable in build/config/{env}/execution_env.yml."
             ) from key_error
 
 

data_pipelines_cli/dbt_utils.py

@@ -40,9 +40,7 @@ def _dump_dbt_vars_from_configs_to_string(env: str) -> str:
     return yaml.dump(dbt_vars, default_flow_style=True, width=sys.maxsize)
 
 
-def run_dbt_command(
-    command: Tuple[str, ...], env: str, profiles_path: pathlib.Path
-) -> None:
+def run_dbt_command(command: Tuple[str, ...], env: str, profiles_path: pathlib.Path) -> None:
     """
     Run dbt subprocess in a context of specified *env*.
 

data_pipelines_cli/docker_response_reader.py

@@ -83,9 +83,7 @@ def click_echo_ok_responses(self) -> None:
             click.echo(response.msg)
 
     @staticmethod
-    def _prepare_status(
-        log: Dict[str, Union[str, Dict[str, str]]]
-    ) -> DockerReadResponse:
+    def _prepare_status(log: Dict[str, Union[str, Dict[str, str]]]) -> DockerReadResponse:
         status_message = cast(str, log["status"])
         progress_detail = cast(str, log.get("progressDetail", ""))
         status_id = cast(str, log.get("id", ""))
@@ -98,9 +96,7 @@ def _prepare_status(
         return DockerReadResponse(message, False)
 
     @staticmethod
-    def _prepare_stream(
-        log: Dict[str, Union[str, Dict[str, str]]]
-    ) -> List[DockerReadResponse]:
+    def _prepare_stream(log: Dict[str, Union[str, Dict[str, str]]]) -> List[DockerReadResponse]:
         stream = cast(str, log["stream"])
         return list(
             map(
@@ -110,9 +106,7 @@ def _prepare_stream(
         )
 
     @staticmethod
-    def _prepare_aux(
-        log: Dict[str, Union[str, Dict[str, str]]]
-    ) -> List[DockerReadResponse]:
+    def _prepare_aux(log: Dict[str, Union[str, Dict[str, str]]]) -> List[DockerReadResponse]:
         aux = cast(Dict[str, str], log["aux"])
         to_return = []
         if "Digest" in aux:
@@ -122,21 +116,15 @@ def _prepare_aux(
         return to_return
 
     @staticmethod
-    def _prepare_error_detail(
-        log: Dict[str, Union[str, Dict[str, str]]]
-    ) -> DockerReadResponse:
+    def _prepare_error_detail(log: Dict[str, Union[str, Dict[str, str]]]) -> DockerReadResponse:
         error_detail = cast(Dict[str, str], log["errorDetail"])
         error_message = error_detail.get("message", "")
         error_code = error_detail.get("code", None)
         return DockerReadResponse(
-            "ERROR: "
-            + error_message
-            + (f"\nError code: {error_code}" if error_code else ""),
+            "ERROR: " + error_message + (f"\nError code: {error_code}" if error_code else ""),
             True,
         )
 
     @staticmethod
-    def _prepare_error(
-        log: Dict[str, Union[str, Dict[str, str]]]
-    ) -> DockerReadResponse:
+    def _prepare_error(log: Dict[str, Union[str, Dict[str, str]]]) -> DockerReadResponse:
         return DockerReadResponse("ERROR: " + cast(str, log["error"]), True)