Skip to content

Commit ef916e6

Browse files
sfc-gh-jwilkowskisfc-gh-jwilkowski
committed
Jw/snow 2045471 add flag for selecting profile directory (#2210)
feat: [SNOW-2045471] add --profile-dir flag and more strict validations for profiles.yml
1 parent af36b8a commit ef916e6

File tree

10 files changed

+341
-34
lines changed

10 files changed

+341
-34
lines changed

‎src/snowflake/cli/__about__.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,7 @@
1616

1717
from enum import Enum, unique
1818

19-
VERSION = "3.7.0.dev+dbt0"
19+
VERSION = "3.8.0.dev+dbt0"
2020

2121

2222
@unique

‎src/snowflake/cli/_plugins/dbt/commands.py‎

Lines changed: 9 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -75,6 +75,11 @@ def deploy_dbt(
7575
show_default=False,
7676
default=None,
7777
),
78+
profiles_dir: Optional[str] = typer.Option(
79+
help="Path to directory containing profiles.yml. Defaults to directory provided in --source or current working directory",
80+
show_default=False,
81+
default=None,
82+
),
7883
force: Optional[bool] = typer.Option(
7984
False,
8085
help="Overwrites conflicting files in the project, if any.",
@@ -84,14 +89,13 @@ def deploy_dbt(
8489
"""
8590
Copy dbt files and create or update dbt on Snowflake project.
8691
"""
87-
if source is None:
88-
path = SecurePath.cwd()
89-
else:
90-
path = SecurePath(source)
92+
project_path = SecurePath(source) if source is not None else SecurePath.cwd()
93+
profiles_dir_path = SecurePath(profiles_dir) if profiles_dir else project_path
9194
return QueryResult(
9295
DBTManager().deploy(
93-
path.resolve(),
9496
name,
97+
project_path.resolve(),
98+
profiles_dir_path.resolve(),
9599
force=force,
96100
)
97101
)

‎src/snowflake/cli/_plugins/dbt/manager.py‎

Lines changed: 68 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,8 @@
1414

1515
from __future__ import annotations
1616

17+
from collections import defaultdict
18+
1719
import yaml
1820
from snowflake.cli._plugins.object.manager import ObjectManager
1921
from snowflake.cli._plugins.stage.manager import StageManager
@@ -39,8 +41,9 @@ def exists(name: FQN) -> bool:
3941

4042
def deploy(
4143
self,
42-
path: SecurePath,
4344
name: FQN,
45+
path: SecurePath,
46+
profiles_path: SecurePath,
4447
force: bool,
4548
) -> SnowflakeCursor:
4649
dbt_project_path = path / "dbt_project.yml"
@@ -56,16 +59,7 @@ def deploy(
5659
except KeyError:
5760
raise CliError("`profile` is not defined in dbt_project.yml")
5861

59-
dbt_profiles_path = path / "profiles.yml"
60-
if not dbt_profiles_path.exists():
61-
raise CliError(
62-
f"profiles.yml does not exist in directory {path.path.absolute()}."
63-
)
64-
65-
with dbt_profiles_path.open(read_file_limit_mb=DEFAULT_SIZE_LIMIT_MB) as fd:
66-
profiles = yaml.safe_load(fd)
67-
if profile not in profiles:
68-
raise CliError(f"profile {profile} is not defined in profiles.yml")
62+
self._validate_profiles(profiles_path, profile)
6963

7064
if self.exists(name=name) and force is not True:
7165
raise CliError(
@@ -79,15 +73,76 @@ def deploy(
7973
stage_manager.create(stage_fqn, temporary=True)
8074

8175
with cli_console.phase("Copying project files to stage"):
82-
results = list(stage_manager.put_recursive(path.path, stage_name))
83-
cli_console.step(f"Copied {len(results)} files")
76+
result_count = len(list(stage_manager.put_recursive(path.path, stage_name)))
77+
if profiles_path != path:
78+
stage_manager.put(
79+
str((profiles_path.path / "profiles.yml").absolute()), stage_name
80+
)
81+
result_count += 1
82+
cli_console.step(f"Copied {result_count} files")
8483

8584
with cli_console.phase("Creating DBT project"):
8685
query = f"""{'CREATE OR REPLACE' if force is True else 'CREATE'} DBT PROJECT {name}
8786
FROM {stage_name}"""
8887

8988
return self.execute_query(query)
9089

90+
@staticmethod
91+
def _validate_profiles(profiles_path: SecurePath, target_profile: str) -> None:
92+
"""
93+
Validates that:
94+
* profiles.yml exists
95+
* contain profile specified in dbt_project.yml
96+
* no other profiles are defined there
97+
* does not contain any confidential data like passwords
98+
"""
99+
profiles_file = profiles_path / "profiles.yml"
100+
if not profiles_file.exists():
101+
raise CliError(
102+
f"profiles.yml does not exist in directory {profiles_path.path.absolute()}."
103+
)
104+
with profiles_file.open(read_file_limit_mb=DEFAULT_SIZE_LIMIT_MB) as fd:
105+
profiles = yaml.safe_load(fd)
106+
107+
if target_profile not in profiles:
108+
raise CliError(f"profile {target_profile} is not defined in profiles.yml")
109+
110+
errors = defaultdict(list)
111+
if len(profiles.keys()) > 1:
112+
for profile_name in profiles.keys():
113+
if profile_name.lower() != target_profile.lower():
114+
errors[profile_name].append("Remove unnecessary profiles")
115+
116+
supported_keys = {
117+
"database",
118+
"account",
119+
"type",
120+
"user",
121+
"role",
122+
"warehouse",
123+
"schema",
124+
}
125+
for target_name, target in profiles[target_profile]["outputs"].items():
126+
if missing_keys := supported_keys - set(target.keys()):
127+
errors[target_profile].append(
128+
f"Missing required fields: {', '.join(sorted(missing_keys))} in target {target_name}"
129+
)
130+
if unsupported_keys := set(target.keys()) - supported_keys:
131+
errors[target_profile].append(
132+
f"Unsupported fields found: {', '.join(sorted(unsupported_keys))} in target {target_name}"
133+
)
134+
if "type" in target and target["type"].lower() != "snowflake":
135+
errors[target_profile].append(
136+
f"Value for type field is invalid. Should be set to `snowflake` in target {target_name}"
137+
)
138+
139+
if errors:
140+
message = "Found following errors in profiles.yml. Please fix them before proceeding:"
141+
for target, issues in errors.items():
142+
message += f"\n{target}"
143+
message += "\n * " + "\n * ".join(issues)
144+
raise CliError(message)
145+
91146
def execute(
92147
self, dbt_command: str, name: str, run_async: bool, *dbt_cli_args
93148
) -> SnowflakeCursor:

‎src/snowflake/cli/_plugins/snowpark/snowpark_entity.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -231,7 +231,7 @@ def _process_requirements( # TODO: maybe leave all the logic with requirements
231231
)
232232

233233
zip_dir(
234-
source=tmp_dir,
234+
source=tmp_dir.path,
235235
dest_zip=bundle_dir / archive_name,
236236
)
237237

‎src/snowflake/cli/api/secure_path.py‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -46,6 +46,11 @@ def __repr__(self):
4646
def __truediv__(self, key):
4747
return SecurePath(self._path / key)
4848

49+
def __eq__(self, other):
50+
if isinstance(other, Path):
51+
return self.path == other
52+
return self.path == other.path
53+
4954
@property
5055
def path(self) -> Path:
5156
"""

‎tests/__snapshots__/test_help_messages.ambr‎

Lines changed: 39 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -3978,13 +3978,17 @@
39783978
| [required] |
39793979
+------------------------------------------------------------------------------+
39803980
+- Options --------------------------------------------------------------------+
3981-
| --source TEXT Path to directory containing dbt files to |
3982-
| deploy. Defaults to current working |
3983-
| directory. |
3984-
| --force --no-force Overwrites conflicting files in the |
3985-
| project, if any. |
3986-
| [default: no-force] |
3987-
| --help -h Show this message and exit. |
3981+
| --source TEXT Path to directory containing dbt |
3982+
| files to deploy. Defaults to current |
3983+
| working directory. |
3984+
| --profiles-dir TEXT Path to directory containing |
3985+
| profiles.yml. Defaults to directory |
3986+
| provided in --source or current |
3987+
| working directory |
3988+
| --force --no-force Overwrites conflicting files in the |
3989+
| project, if any. |
3990+
| [default: no-force] |
3991+
| --help -h Show this message and exit. |
39883992
+------------------------------------------------------------------------------+
39893993
+- Connection configuration ---------------------------------------------------+
39903994
| --connection,--environment -c TEXT Name of the connection, as |
@@ -4064,6 +4068,8 @@
40644068
| to console. |
40654069
| --enhanced-exit-codes Differentiate exit error codes |
40664070
| based on failure type. |
4071+
| [env var: |
4072+
| SNOWFLAKE_ENHANCED_EXIT_CODES] |
40674073
+------------------------------------------------------------------------------+
40684074

40694075

@@ -4164,6 +4170,8 @@
41644170
| to console. |
41654171
| --enhanced-exit-codes Differentiate exit error codes |
41664172
| based on failure type. |
4173+
| [env var: |
4174+
| SNOWFLAKE_ENHANCED_EXIT_CODES] |
41674175
+------------------------------------------------------------------------------+
41684176
+- Commands -------------------------------------------------------------------+
41694177
| build Execute build command on Snowflake. |
@@ -4276,6 +4284,8 @@
42764284
| to console. |
42774285
| --enhanced-exit-codes Differentiate exit error codes |
42784286
| based on failure type. |
4287+
| [env var: |
4288+
| SNOWFLAKE_ENHANCED_EXIT_CODES] |
42794289
+------------------------------------------------------------------------------+
42804290
+- Commands -------------------------------------------------------------------+
42814291
| build Execute build command on Snowflake. |
@@ -4388,6 +4398,8 @@
43884398
| to console. |
43894399
| --enhanced-exit-codes Differentiate exit error codes |
43904400
| based on failure type. |
4401+
| [env var: |
4402+
| SNOWFLAKE_ENHANCED_EXIT_CODES] |
43914403
+------------------------------------------------------------------------------+
43924404
+- Commands -------------------------------------------------------------------+
43934405
| build Execute build command on Snowflake. |
@@ -4500,6 +4512,8 @@
45004512
| to console. |
45014513
| --enhanced-exit-codes Differentiate exit error codes |
45024514
| based on failure type. |
4515+
| [env var: |
4516+
| SNOWFLAKE_ENHANCED_EXIT_CODES] |
45034517
+------------------------------------------------------------------------------+
45044518
+- Commands -------------------------------------------------------------------+
45054519
| build Execute build command on Snowflake. |
@@ -4612,6 +4626,8 @@
46124626
| to console. |
46134627
| --enhanced-exit-codes Differentiate exit error codes |
46144628
| based on failure type. |
4629+
| [env var: |
4630+
| SNOWFLAKE_ENHANCED_EXIT_CODES] |
46154631
+------------------------------------------------------------------------------+
46164632
+- Commands -------------------------------------------------------------------+
46174633
| build Execute build command on Snowflake. |
@@ -4724,6 +4740,8 @@
47244740
| to console. |
47254741
| --enhanced-exit-codes Differentiate exit error codes |
47264742
| based on failure type. |
4743+
| [env var: |
4744+
| SNOWFLAKE_ENHANCED_EXIT_CODES] |
47274745
+------------------------------------------------------------------------------+
47284746
+- Commands -------------------------------------------------------------------+
47294747
| build Execute build command on Snowflake. |
@@ -4836,6 +4854,8 @@
48364854
| to console. |
48374855
| --enhanced-exit-codes Differentiate exit error codes |
48384856
| based on failure type. |
4857+
| [env var: |
4858+
| SNOWFLAKE_ENHANCED_EXIT_CODES] |
48394859
+------------------------------------------------------------------------------+
48404860
+- Commands -------------------------------------------------------------------+
48414861
| build Execute build command on Snowflake. |
@@ -4948,6 +4968,8 @@
49484968
| to console. |
49494969
| --enhanced-exit-codes Differentiate exit error codes |
49504970
| based on failure type. |
4971+
| [env var: |
4972+
| SNOWFLAKE_ENHANCED_EXIT_CODES] |
49514973
+------------------------------------------------------------------------------+
49524974
+- Commands -------------------------------------------------------------------+
49534975
| build Execute build command on Snowflake. |
@@ -5060,6 +5082,8 @@
50605082
| to console. |
50615083
| --enhanced-exit-codes Differentiate exit error codes |
50625084
| based on failure type. |
5085+
| [env var: |
5086+
| SNOWFLAKE_ENHANCED_EXIT_CODES] |
50635087
+------------------------------------------------------------------------------+
50645088
+- Commands -------------------------------------------------------------------+
50655089
| build Execute build command on Snowflake. |
@@ -5172,6 +5196,8 @@
51725196
| to console. |
51735197
| --enhanced-exit-codes Differentiate exit error codes |
51745198
| based on failure type. |
5199+
| [env var: |
5200+
| SNOWFLAKE_ENHANCED_EXIT_CODES] |
51755201
+------------------------------------------------------------------------------+
51765202
+- Commands -------------------------------------------------------------------+
51775203
| build Execute build command on Snowflake. |
@@ -5284,6 +5310,8 @@
52845310
| to console. |
52855311
| --enhanced-exit-codes Differentiate exit error codes |
52865312
| based on failure type. |
5313+
| [env var: |
5314+
| SNOWFLAKE_ENHANCED_EXIT_CODES] |
52875315
+------------------------------------------------------------------------------+
52885316
+- Commands -------------------------------------------------------------------+
52895317
| build Execute build command on Snowflake. |
@@ -5399,6 +5427,8 @@
53995427
| to console. |
54005428
| --enhanced-exit-codes Differentiate exit error codes |
54015429
| based on failure type. |
5430+
| [env var: |
5431+
| SNOWFLAKE_ENHANCED_EXIT_CODES] |
54025432
+------------------------------------------------------------------------------+
54035433

54045434

@@ -15948,6 +15978,8 @@
1594815978
| to console. |
1594915979
| --enhanced-exit-codes Differentiate exit error codes |
1595015980
| based on failure type. |
15981+
| [env var: |
15982+
| SNOWFLAKE_ENHANCED_EXIT_CODES] |
1595115983
+------------------------------------------------------------------------------+
1595215984
+- Commands -------------------------------------------------------------------+
1595315985
| build Execute build command on Snowflake. |

‎tests/dbt/test_dbt_commands.py‎

Lines changed: 44 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@
1414

1515
from __future__ import annotations
1616

17+
from pathlib import Path
1718
from unittest import mock
1819

1920
import pytest
@@ -65,7 +66,15 @@ def dbt_project_path(self, tmp_path_factory):
6566
{
6667
"dev": {
6768
"outputs": {
68-
"local": {"account": "test_account", "database": "testdb"}
69+
"local": {
70+
"account": "test_account",
71+
"database": "testdb",
72+
"role": "test_role",
73+
"schema": "test_schema",
74+
"type": "snowflake",
75+
"user": "test_user",
76+
"warehouse": "test_warehouse",
77+
}
6978
}
7079
}
7180
},
@@ -148,6 +157,40 @@ def test_force_flag_uses_create_or_replace(
148157
"CREATE OR REPLACE DBT PROJECT"
149158
)
150159

160+
@mock.patch("snowflake.cli._plugins.dbt.manager.StageManager.put_recursive")
161+
@mock.patch("snowflake.cli._plugins.dbt.manager.StageManager.put")
162+
@mock.patch("snowflake.cli._plugins.dbt.manager.StageManager.create")
163+
def test_dbt_deploy_with_custom_profiles_dir(
164+
self,
165+
_mock_create,
166+
mock_put,
167+
_mock_put_recursive,
168+
mock_connect,
169+
runner,
170+
dbt_project_path,
171+
mock_exists,
172+
):
173+
new_profiles_directory = Path(dbt_project_path) / "dbt_profiles"
174+
new_profiles_directory.mkdir(parents=True, exist_ok=True)
175+
profiles_file = dbt_project_path / "profiles.yml"
176+
profiles_file.rename(new_profiles_directory / "profiles.yml")
177+
178+
result = runner.invoke(
179+
[
180+
"dbt",
181+
"deploy",
182+
"TEST_PIPELINE",
183+
f"--source={dbt_project_path}",
184+
f"--profiles-dir={new_profiles_directory}",
185+
]
186+
)
187+
188+
assert result.exit_code == 0, result.output
189+
mock_put.assert_called_once_with(
190+
str(new_profiles_directory / "profiles.yml"),
191+
"@MockDatabase.MockSchema.dbt_TEST_PIPELINE_stage",
192+
)
193+
151194
def test_raises_when_dbt_project_yml_is_not_available(
152195
self, dbt_project_path, mock_connect, runner
153196
):

0 commit comments

Comments
 (0)