Jw/snow 2045471 add flag for selecting profile directory (#2210)

feat: [SNOW-2045471] add --profile-dir flag and more strict validations for profiles.yml

Author: sfc-gh-jwilkowski

src/snowflake/cli/__about__.py

@@ -16,7 +16,7 @@
 
 from enum import Enum, unique
 
-VERSION = "3.7.0.dev+dbt0"
+VERSION = "3.8.0.dev+dbt0"
 
 
 @unique

src/snowflake/cli/_plugins/dbt/commands.py

@@ -75,6 +75,11 @@ def deploy_dbt(
         show_default=False,
         default=None,
     ),
+    profiles_dir: Optional[str] = typer.Option(
+        help="Path to directory containing profiles.yml. Defaults to directory provided in --source or current working directory",
+        show_default=False,
+        default=None,
+    ),
     force: Optional[bool] = typer.Option(
         False,
         help="Overwrites conflicting files in the project, if any.",
@@ -84,14 +89,13 @@ def deploy_dbt(
     """
     Copy dbt files and create or update dbt on Snowflake project.
     """
-    if source is None:
-        path = SecurePath.cwd()
-    else:
-        path = SecurePath(source)
+    project_path = SecurePath(source) if source is not None else SecurePath.cwd()
+    profiles_dir_path = SecurePath(profiles_dir) if profiles_dir else project_path
     return QueryResult(
         DBTManager().deploy(
-            path.resolve(),
             name,
+            project_path.resolve(),
+            profiles_dir_path.resolve(),
             force=force,
         )
     )

src/snowflake/cli/_plugins/dbt/manager.py

@@ -14,6 +14,8 @@
 
 from __future__ import annotations
 
+from collections import defaultdict
+
 import yaml
 from snowflake.cli._plugins.object.manager import ObjectManager
 from snowflake.cli._plugins.stage.manager import StageManager
@@ -39,8 +41,9 @@ def exists(name: FQN) -> bool:
 
     def deploy(
         self,
-        path: SecurePath,
         name: FQN,
+        path: SecurePath,
+        profiles_path: SecurePath,
         force: bool,
     ) -> SnowflakeCursor:
         dbt_project_path = path / "dbt_project.yml"
@@ -56,16 +59,7 @@ def deploy(
             except KeyError:
                 raise CliError("`profile` is not defined in dbt_project.yml")
 
-        dbt_profiles_path = path / "profiles.yml"
-        if not dbt_profiles_path.exists():
-            raise CliError(
-                f"profiles.yml does not exist in directory {path.path.absolute()}."
-            )
-
-        with dbt_profiles_path.open(read_file_limit_mb=DEFAULT_SIZE_LIMIT_MB) as fd:
-            profiles = yaml.safe_load(fd)
-            if profile not in profiles:
-                raise CliError(f"profile {profile} is not defined in profiles.yml")
+        self._validate_profiles(profiles_path, profile)
 
         if self.exists(name=name) and force is not True:
             raise CliError(
@@ -79,15 +73,76 @@ def deploy(
             stage_manager.create(stage_fqn, temporary=True)
 
         with cli_console.phase("Copying project files to stage"):
-            results = list(stage_manager.put_recursive(path.path, stage_name))
-            cli_console.step(f"Copied {len(results)} files")
+            result_count = len(list(stage_manager.put_recursive(path.path, stage_name)))
+            if profiles_path != path:
+                stage_manager.put(
+                    str((profiles_path.path / "profiles.yml").absolute()), stage_name
+                )
+                result_count += 1
+            cli_console.step(f"Copied {result_count} files")
 
         with cli_console.phase("Creating DBT project"):
             query = f"""{'CREATE OR REPLACE' if force is True else 'CREATE'} DBT PROJECT {name}
 FROM {stage_name}"""
 
             return self.execute_query(query)
 
+    @staticmethod
+    def _validate_profiles(profiles_path: SecurePath, target_profile: str) -> None:
+        """
+        Validates that:
+         * profiles.yml exists
+         * contain profile specified in dbt_project.yml
+         * no other profiles are defined there
+         * does not contain any confidential data like passwords
+        """
+        profiles_file = profiles_path / "profiles.yml"
+        if not profiles_file.exists():
+            raise CliError(
+                f"profiles.yml does not exist in directory {profiles_path.path.absolute()}."
+            )
+        with profiles_file.open(read_file_limit_mb=DEFAULT_SIZE_LIMIT_MB) as fd:
+            profiles = yaml.safe_load(fd)
+
+        if target_profile not in profiles:
+            raise CliError(f"profile {target_profile} is not defined in profiles.yml")
+
+        errors = defaultdict(list)
+        if len(profiles.keys()) > 1:
+            for profile_name in profiles.keys():
+                if profile_name.lower() != target_profile.lower():
+                    errors[profile_name].append("Remove unnecessary profiles")
+
+        supported_keys = {
+            "database",
+            "account",
+            "type",
+            "user",
+            "role",
+            "warehouse",
+            "schema",
+        }
+        for target_name, target in profiles[target_profile]["outputs"].items():
+            if missing_keys := supported_keys - set(target.keys()):
+                errors[target_profile].append(
+                    f"Missing required fields: {', '.join(sorted(missing_keys))} in target {target_name}"
+                )
+            if unsupported_keys := set(target.keys()) - supported_keys:
+                errors[target_profile].append(
+                    f"Unsupported fields found: {', '.join(sorted(unsupported_keys))} in target {target_name}"
+                )
+            if "type" in target and target["type"].lower() != "snowflake":
+                errors[target_profile].append(
+                    f"Value for type field is invalid. Should be set to `snowflake` in target {target_name}"
+                )
+
+        if errors:
+            message = "Found following errors in profiles.yml. Please fix them before proceeding:"
+            for target, issues in errors.items():
+                message += f"\n{target}"
+                message += "\n * " + "\n * ".join(issues)
+            raise CliError(message)
+
     def execute(
         self, dbt_command: str, name: str, run_async: bool, *dbt_cli_args
     ) -> SnowflakeCursor:

src/snowflake/cli/_plugins/snowpark/snowpark_entity.py

@@ -231,7 +231,7 @@ def _process_requirements(  # TODO: maybe leave all the logic with requirements
                     )
 
                 zip_dir(
-                    source=tmp_dir,
+                    source=tmp_dir.path,
                     dest_zip=bundle_dir / archive_name,
                 )
 

src/snowflake/cli/api/secure_path.py

@@ -46,6 +46,11 @@ def __repr__(self):
     def __truediv__(self, key):
         return SecurePath(self._path / key)
 
+    def __eq__(self, other):
+        if isinstance(other, Path):
+            return self.path == other
+        return self.path == other.path
+
     @property
     def path(self) -> Path:
         """

tests/__snapshots__/test_help_messages.ambr

@@ -3978,13 +3978,17 @@
   |                      [required]                                              |
   +------------------------------------------------------------------------------+
   +- Options --------------------------------------------------------------------+
-  | --source                    TEXT  Path to directory containing dbt files to  |
-  |                                   deploy. Defaults to current working        |
-  |                                   directory.                                 |
-  | --force       --no-force          Overwrites conflicting files in the        |
-  |                                   project, if any.                           |
-  |                                   [default: no-force]                        |
-  | --help    -h                      Show this message and exit.                |
+  | --source                          TEXT  Path to directory containing dbt     |
+  |                                         files to deploy. Defaults to current |
+  |                                         working directory.                   |
+  | --profiles-dir                    TEXT  Path to directory containing         |
+  |                                         profiles.yml. Defaults to directory  |
+  |                                         provided in --source or current      |
+  |                                         working directory                    |
+  | --force             --no-force          Overwrites conflicting files in the  |
+  |                                         project, if any.                     |
+  |                                         [default: no-force]                  |
+  | --help          -h                      Show this message and exit.          |
   +------------------------------------------------------------------------------+
   +- Connection configuration ---------------------------------------------------+
   | --connection,--environment     -c      TEXT     Name of the connection, as   |
@@ -4064,6 +4068,8 @@
   |                                              to console.                     |
   | --enhanced-exit-codes                        Differentiate exit error codes  |
   |                                              based on failure type.          |
+  |                                              [env var:                       |
+  |                                              SNOWFLAKE_ENHANCED_EXIT_CODES]  |
   +------------------------------------------------------------------------------+
 
 
@@ -4164,6 +4170,8 @@
   |                                              to console.                     |
   | --enhanced-exit-codes                        Differentiate exit error codes  |
   |                                              based on failure type.          |
+  |                                              [env var:                       |
+  |                                              SNOWFLAKE_ENHANCED_EXIT_CODES]  |
   +------------------------------------------------------------------------------+
   +- Commands -------------------------------------------------------------------+
   | build      Execute build command on Snowflake.                               |
@@ -4276,6 +4284,8 @@
   |                                              to console.                     |
   | --enhanced-exit-codes                        Differentiate exit error codes  |
   |                                              based on failure type.          |
+  |                                              [env var:                       |
+  |                                              SNOWFLAKE_ENHANCED_EXIT_CODES]  |
   +------------------------------------------------------------------------------+
   +- Commands -------------------------------------------------------------------+
   | build      Execute build command on Snowflake.                               |
@@ -4388,6 +4398,8 @@
   |                                              to console.                     |
   | --enhanced-exit-codes                        Differentiate exit error codes  |
   |                                              based on failure type.          |
+  |                                              [env var:                       |
+  |                                              SNOWFLAKE_ENHANCED_EXIT_CODES]  |
   +------------------------------------------------------------------------------+
   +- Commands -------------------------------------------------------------------+
   | build      Execute build command on Snowflake.                               |
@@ -4500,6 +4512,8 @@
   |                                              to console.                     |
   | --enhanced-exit-codes                        Differentiate exit error codes  |
   |                                              based on failure type.          |
+  |                                              [env var:                       |
+  |                                              SNOWFLAKE_ENHANCED_EXIT_CODES]  |
   +------------------------------------------------------------------------------+
   +- Commands -------------------------------------------------------------------+
   | build      Execute build command on Snowflake.                               |
@@ -4612,6 +4626,8 @@
   |                                              to console.                     |
   | --enhanced-exit-codes                        Differentiate exit error codes  |
   |                                              based on failure type.          |
+  |                                              [env var:                       |
+  |                                              SNOWFLAKE_ENHANCED_EXIT_CODES]  |
   +------------------------------------------------------------------------------+
   +- Commands -------------------------------------------------------------------+
   | build      Execute build command on Snowflake.                               |
@@ -4724,6 +4740,8 @@
   |                                              to console.                     |
   | --enhanced-exit-codes                        Differentiate exit error codes  |
   |                                              based on failure type.          |
+  |                                              [env var:                       |
+  |                                              SNOWFLAKE_ENHANCED_EXIT_CODES]  |
   +------------------------------------------------------------------------------+
   +- Commands -------------------------------------------------------------------+
   | build      Execute build command on Snowflake.                               |
@@ -4836,6 +4854,8 @@
   |                                              to console.                     |
   | --enhanced-exit-codes                        Differentiate exit error codes  |
   |                                              based on failure type.          |
+  |                                              [env var:                       |
+  |                                              SNOWFLAKE_ENHANCED_EXIT_CODES]  |
   +------------------------------------------------------------------------------+
   +- Commands -------------------------------------------------------------------+
   | build      Execute build command on Snowflake.                               |
@@ -4948,6 +4968,8 @@
   |                                              to console.                     |
   | --enhanced-exit-codes                        Differentiate exit error codes  |
   |                                              based on failure type.          |
+  |                                              [env var:                       |
+  |                                              SNOWFLAKE_ENHANCED_EXIT_CODES]  |
   +------------------------------------------------------------------------------+
   +- Commands -------------------------------------------------------------------+
   | build      Execute build command on Snowflake.                               |
@@ -5060,6 +5082,8 @@
   |                                              to console.                     |
   | --enhanced-exit-codes                        Differentiate exit error codes  |
   |                                              based on failure type.          |
+  |                                              [env var:                       |
+  |                                              SNOWFLAKE_ENHANCED_EXIT_CODES]  |
   +------------------------------------------------------------------------------+
   +- Commands -------------------------------------------------------------------+
   | build      Execute build command on Snowflake.                               |
@@ -5172,6 +5196,8 @@
   |                                              to console.                     |
   | --enhanced-exit-codes                        Differentiate exit error codes  |
   |                                              based on failure type.          |
+  |                                              [env var:                       |
+  |                                              SNOWFLAKE_ENHANCED_EXIT_CODES]  |
   +------------------------------------------------------------------------------+
   +- Commands -------------------------------------------------------------------+
   | build      Execute build command on Snowflake.                               |
@@ -5284,6 +5310,8 @@
   |                                              to console.                     |
   | --enhanced-exit-codes                        Differentiate exit error codes  |
   |                                              based on failure type.          |
+  |                                              [env var:                       |
+  |                                              SNOWFLAKE_ENHANCED_EXIT_CODES]  |
   +------------------------------------------------------------------------------+
   +- Commands -------------------------------------------------------------------+
   | build      Execute build command on Snowflake.                               |
@@ -5399,6 +5427,8 @@
   |                                              to console.                     |
   | --enhanced-exit-codes                        Differentiate exit error codes  |
   |                                              based on failure type.          |
+  |                                              [env var:                       |
+  |                                              SNOWFLAKE_ENHANCED_EXIT_CODES]  |
   +------------------------------------------------------------------------------+
 
 
@@ -15948,6 +15978,8 @@
   |                                              to console.                     |
   | --enhanced-exit-codes                        Differentiate exit error codes  |
   |                                              based on failure type.          |
+  |                                              [env var:                       |
+  |                                              SNOWFLAKE_ENHANCED_EXIT_CODES]  |
   +------------------------------------------------------------------------------+
   +- Commands -------------------------------------------------------------------+
   | build      Execute build command on Snowflake.                               |

tests/dbt/test_dbt_commands.py

@@ -14,6 +14,7 @@
 
 from __future__ import annotations
 
+from pathlib import Path
 from unittest import mock
 
 import pytest
@@ -65,7 +66,15 @@ def dbt_project_path(self, tmp_path_factory):
                 {
                     "dev": {
                         "outputs": {
-                            "local": {"account": "test_account", "database": "testdb"}
+                            "local": {
+                                "account": "test_account",
+                                "database": "testdb",
+                                "role": "test_role",
+                                "schema": "test_schema",
+                                "type": "snowflake",
+                                "user": "test_user",
+                                "warehouse": "test_warehouse",
+                            }
                         }
                     }
                 },
@@ -148,6 +157,40 @@ def test_force_flag_uses_create_or_replace(
             "CREATE OR REPLACE DBT PROJECT"
         )
 
+    @mock.patch("snowflake.cli._plugins.dbt.manager.StageManager.put_recursive")
+    @mock.patch("snowflake.cli._plugins.dbt.manager.StageManager.put")
+    @mock.patch("snowflake.cli._plugins.dbt.manager.StageManager.create")
+    def test_dbt_deploy_with_custom_profiles_dir(
+        self,
+        _mock_create,
+        mock_put,
+        _mock_put_recursive,
+        mock_connect,
+        runner,
+        dbt_project_path,
+        mock_exists,
+    ):
+        new_profiles_directory = Path(dbt_project_path) / "dbt_profiles"
+        new_profiles_directory.mkdir(parents=True, exist_ok=True)
+        profiles_file = dbt_project_path / "profiles.yml"
+        profiles_file.rename(new_profiles_directory / "profiles.yml")
+
+        result = runner.invoke(
+            [
+                "dbt",
+                "deploy",
+                "TEST_PIPELINE",
+                f"--source={dbt_project_path}",
+                f"--profiles-dir={new_profiles_directory}",
+            ]
+        )
+
+        assert result.exit_code == 0, result.output
+        mock_put.assert_called_once_with(
+            str(new_profiles_directory / "profiles.yml"),
+            "@MockDatabase.MockSchema.dbt_TEST_PIPELINE_stage",
+        )
+
     def test_raises_when_dbt_project_yml_is_not_available(
         self, dbt_project_path, mock_connect, runner
     ):