Merge pull request #58 from getindata/release-0.15.0

Release 0.15.0

Author: swtwsk

CHANGELOG.md

@@ -2,6 +2,10 @@
 
 ## [Unreleased]
 
+## [0.15.0] - 2022-02-11
+
+-   Migration to dbt 1.0.1
+
 ## [0.14.0] - 2022-02-02
 
 ## [0.13.0] - 2022-02-01
@@ -152,7 +156,9 @@
 -   Draft of `dp init`, `dp create`, `dp template new`, `dp template list` and `dp dbt`
 -   Draft of `dp compile` and `dp deploy`
 
-[Unreleased]: https://github.com/getindata/data-pipelines-cli/compare/0.14.0...HEAD
+[Unreleased]: https://github.com/getindata/data-pipelines-cli/compare/0.15.0...HEAD
+
+[0.15.0]: https://github.com/getindata/data-pipelines-cli/compare/0.14.0...0.15.0
 
 [0.14.0]: https://github.com/getindata/data-pipelines-cli/compare/0.13.0...0.14.0
 

data_pipelines_cli/__init__.py

@@ -5,4 +5,4 @@
 pipelines.
 """
 
-version = "0.14.0"
+version = "0.15.0"

data_pipelines_cli/cli_commands/compile.py

@@ -5,7 +5,7 @@
 import yaml
 
 from ..cli_constants import BUILD_DIR, IMAGE_TAG_TO_REPLACE
-from ..cli_utils import echo_error, echo_info, echo_warning
+from ..cli_utils import echo_info, echo_warning
 from ..config_generation import (
     copy_config_dir_to_build_dir,
     copy_dag_dir_to_build_dir,
@@ -14,11 +14,7 @@
 from ..data_structures import DockerArgs
 from ..dbt_utils import read_dbt_vars_from_configs, run_dbt_command
 from ..docker_response_reader import DockerResponseReader
-from ..errors import (
-    DataPipelinesError,
-    DockerErrorResponseError,
-    DockerNotInstalledError,
-)
+from ..errors import DockerErrorResponseError, DockerNotInstalledError
 from ..io_utils import replace
 from ..jinja import replace_vars_with_values
 
@@ -31,22 +27,19 @@ def _docker_build(docker_args: DockerArgs) -> None:
     """
     try:
         import docker
+        import docker.errors
     except ModuleNotFoundError:
         raise DockerNotInstalledError()
 
     echo_info("Building Docker image")
     docker_client = docker.from_env()
     docker_tag = docker_args.docker_build_tag()
-    _, logs_generator = docker_client.images.build(path=".", tag=docker_tag)
     try:
+        _, logs_generator = docker_client.images.build(path=".", tag=docker_tag)
         DockerResponseReader(logs_generator).click_echo_ok_responses()
-    except DockerErrorResponseError as err:
-        echo_error(err.message)
-        raise DataPipelinesError(
-            "Error raised when pushing Docker image. Ensure that "
-            "Docker image you try to push exists. Maybe try running "
-            "'dp compile' first?"
-        )
+    except docker.errors.BuildError as err:
+        build_log = "\n".join([str(log) for log in err.build_log])
+        raise DockerErrorResponseError(f"{err.msg}\n{build_log}")
 
 
 def _dbt_compile(env: str) -> None:

data_pipelines_cli/cli_utils.py

@@ -25,6 +25,17 @@ def echo_error(text: str, **kwargs: Any) -> None:
     click.secho(text, file=sys.stderr, fg="red", **kwargs)
 
 
+def echo_suberror(text: str, **kwargs: Any) -> None:
+    """
+    Print a suberror message to stderr using click-specific print function.
+
+    :param text: Message to print
+    :type text: str
+    :param kwargs:
+    """
+    click.secho(text, file=sys.stderr, fg="bright_red", **kwargs)
+
+
 def echo_warning(text: str, **kwargs: Any) -> None:
     """
     Print a warning message to stderr using click-specific print function.

docs/conf.py

@@ -9,6 +9,8 @@
 import os
 import sys
 
+sys.path.insert(0, os.path.abspath("."))
+sys.path.insert(0, os.path.abspath("../"))
 sys.path.insert(0, os.path.abspath("../data_pipelines_cli"))
 
 
@@ -27,7 +29,7 @@
 extensions = [
     "sphinx.ext.autodoc",
     "sphinx.ext.viewcode",
-    "sphinx_click.ext",
+    "sphinx_click",
     "myst_parser",
 ]
 

docs/requirements.txt

@@ -1,4 +1,5 @@
 sphinx-rtd-theme==1.0.0
-sphinx-click>=3.0,<3.1
+sphinx-click>=3.1,<3.2
 myst-parser>=0.16, <0.17
 docutils>=0.17,<0.18
+GitPython==3.1.26

setup.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 0.14.0
+current_version = 0.15.0
 
 [bumpversion:file:setup.py]
 

setup.py

@@ -6,14 +6,15 @@
     README = f.read()
 
 INSTALL_REQUIREMENTS = [
-    "click<8.0",
+    "click>=8.0.3,<9.0",
     "questionary==1.10.0",
     "pyyaml>=5.1, <6.0",
     "types-PyYAML>=6.0",
     "copier==5.1.0",
-    "dbt>=0.21, <0.22",
+    "dbt-core==1.0.1",
     "Jinja2>=2.11,<2.12",
     "fsspec",
+    "packaging>=20.4,<21.0",
 ]
 
 EXTRA_FILESYSTEMS_REQUIRE = {
@@ -44,15 +45,16 @@
     "docs": [
         "sphinx==4.3.1",
         "sphinx-rtd-theme==1.0.0",
-        "sphinx-click>=3.0,<3.1",
+        "sphinx-click>=3.1,<3.2",
         "myst-parser>=0.16, <0.17",
+        "GitPython==3.1.26",
     ],
     **EXTRA_FILESYSTEMS_REQUIRE,
 }
 
 setup(
     name="data_pipelines_cli",
-    version="0.14.0",
+    version="0.15.0",
     description="CLI for data platform",
     long_description=README,
     long_description_content_type="text/markdown",

tests/cli_commands/test_compile.py

@@ -130,7 +130,7 @@ def _mock_docker(**kwargs):
 
         runner = CliRunner()
         with patch.dict(
-            "sys.modules", docker=docker_mock
+            "sys.modules", **{"docker": docker_mock, "docker.errors": MagicMock()}
         ), tempfile.TemporaryDirectory() as tmp_dir, patch(
             "data_pipelines_cli.cli_commands.compile.BUILD_DIR", pathlib.Path(tmp_dir)
         ), patch(
@@ -146,6 +146,40 @@ def _mock_docker(**kwargs):
             self.assertEqual(0, result.exit_code, msg=result.exception)
             self.assertEqual("my_docker_repository_uri:aaa9876aaa", docker_tag)
 
+    @patch("pathlib.Path.cwd", lambda: goldens_dir_path)
+    @patch("data_pipelines_cli.data_structures.git_revision_hash")
+    def test_docker_throw_build_error(self, mock_git_revision_hash):
+        commit_sha = "aaa9876aaa"
+        mock_git_revision_hash.return_value = commit_sha
+
+        class MockException(Exception):
+            msg = "some error message"
+            build_log = ["more", "errors"]
+
+        docker_errors_mock = MagicMock(BuildError=MockException)
+        docker_images_mock = MagicMock(build=MagicMock(side_effect=MockException))
+        docker_client_mock = MagicMock(images=docker_images_mock)
+        docker_mock = MagicMock(from_env=lambda: docker_client_mock, errors=docker_errors_mock)
+
+        with patch.dict(
+            "sys.modules", **{"docker": docker_mock, "docker.errors": docker_errors_mock}
+        ), tempfile.TemporaryDirectory() as tmp_dir, patch(
+            "data_pipelines_cli.cli_commands.compile.BUILD_DIR", pathlib.Path(tmp_dir)
+        ), patch(
+            "data_pipelines_cli.cli_constants.BUILD_DIR", pathlib.Path(tmp_dir)
+        ), patch(
+            "data_pipelines_cli.config_generation.BUILD_DIR", pathlib.Path(tmp_dir)
+        ), patch(
+            "data_pipelines_cli.dbt_utils.BUILD_DIR", pathlib.Path(tmp_dir)
+        ), patch(
+            "data_pipelines_cli.dbt_utils.subprocess_run", self._mock_run
+        ):
+            with self.assertRaises(DataPipelinesError):
+                try:
+                    compile_project("base", True)
+                except MockException:
+                    self.fail()
+
     @patch("pathlib.Path.cwd", lambda: goldens_dir_path)
     @patch("data_pipelines_cli.data_structures.git_revision_hash")
     def test_docker_throw_on_error(self, mock_git_revision_hash):
@@ -159,15 +193,23 @@ def _mock_docker(**_kwargs):
                 '"error":"An image cannot be built."}',
             ]
 
+        class MockException(Exception):
+            pass
+
+        docker_errors_mock = MagicMock()
+        docker_errors_mock.BuildError = MockException
+
         docker_images_mock = MagicMock()
         docker_images_mock.configure_mock(**{"build": _mock_docker})
         docker_client_mock = MagicMock()
         docker_client_mock.configure_mock(**{"images": docker_images_mock})
         docker_mock = MagicMock()
-        docker_mock.configure_mock(**{"from_env": lambda: docker_client_mock})
+        docker_mock.configure_mock(
+            **{"from_env": lambda: docker_client_mock, "errors": docker_errors_mock}
+        )
 
         with patch.dict(
-            "sys.modules", docker=docker_mock
+            "sys.modules", **{"docker": docker_mock, "docker.errors": docker_errors_mock}
         ), tempfile.TemporaryDirectory() as tmp_dir, patch(
             "data_pipelines_cli.cli_commands.compile.BUILD_DIR", pathlib.Path(tmp_dir)
         ), patch(