From 15d2e76949101b14d86902387630678f099820e4 Mon Sep 17 00:00:00 2001 From: Patryk Czajka Date: Thu, 10 Apr 2025 15:12:48 +0200 Subject: [PATCH 01/54] Update release notes for 3.7.0 --- RELEASE-NOTES.md | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/RELEASE-NOTES.md b/RELEASE-NOTES.md index 0641ea823a..386ac718a7 100644 --- a/RELEASE-NOTES.md +++ b/RELEASE-NOTES.md @@ -18,6 +18,15 @@ ## Deprecations +## New additions + +## Fixes and improvements + + +# v3.7.0 + +## Deprecations + ## New additions * Added `--prune` flag to `deploy` commands, which removes files that exist in the stage, but not in the local filesystem. From 9692a04e88c10d0df15657fff3305335cf963fbb Mon Sep 17 00:00:00 2001 From: Patryk Czajka Date: Thu, 10 Apr 2025 15:52:55 +0200 Subject: [PATCH 02/54] Bump version to v3.7.0-rc0 (#2193) --- src/snowflake/cli/__about__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/snowflake/cli/__about__.py b/src/snowflake/cli/__about__.py index 19520dac3c..28f4a9a39d 100644 --- a/src/snowflake/cli/__about__.py +++ b/src/snowflake/cli/__about__.py @@ -16,7 +16,7 @@ from enum import Enum, unique -VERSION = "3.7.0.dev0" +VERSION = "3.7.0rc0" @unique From e3ab63fd844e772e83ae90effdc181096c5bd07e Mon Sep 17 00:00:00 2001 From: Patryk Czajka Date: Tue, 15 Apr 2025 17:28:53 +0200 Subject: [PATCH 03/54] Cherrypicks v3.7.0 rc1 (#2205) * Bump version to v3.7.0-rc1 * `Snow logs` fixes part 2: logs strike back (#2203) * Added log levels * test fix * test fix * Snow 2042008 fix mac binary builds (#2204) * Fix binary building scripts for MacOS * Remove leftovers * run SPCS check on release PRs (#2206) * SNOW-2040358 update all scripts if snow is not in the path (#2202) * SNOW-2040358 update all scripts if snow is not in the path * SNOW-2040358 update release notes --------- Co-authored-by: Jan Sikorski <132985823+sfc-gh-jsikorski@users.noreply.github.com> Co-authored-by: Marcin Raba --- .github/workflows/test_integration_spcs.yaml | 3 + RELEASE-NOTES.md | 1 + pyproject.toml | 1 - scripts/packaging/build_darwin_package.sh | 69 +++++++++++-------- scripts/packaging/macos/postinstall | 31 +++++---- scripts/packaging/setup_darwin.sh | 39 ----------- src/snowflake/cli/__about__.py | 2 +- src/snowflake/cli/_plugins/logs/commands.py | 17 ++++- src/snowflake/cli/_plugins/logs/manager.py | 61 ++++------------ src/snowflake/cli/_plugins/logs/utils.py | 60 ++++++++++++++++ tests/__snapshots__/test_help_messages.ambr | 29 ++++---- tests/logs/__snapshots__/test_logs.ambr | 14 +++- tests/logs/test_logs.py | 14 ++++ ...est_logs_manager.py => test_logs_utils.py} | 43 +++++++++++- .../test_data/projects/snowpark_v2/c.py | 7 ++ 15 files changed, 242 insertions(+), 149 deletions(-) delete mode 100644 scripts/packaging/setup_darwin.sh create mode 100644 src/snowflake/cli/_plugins/logs/utils.py rename tests/logs/{test_logs_manager.py => test_logs_utils.py} (68%) diff --git a/.github/workflows/test_integration_spcs.yaml b/.github/workflows/test_integration_spcs.yaml index 00973e2821..1f60f5b6ac 100644 --- a/.github/workflows/test_integration_spcs.yaml +++ b/.github/workflows/test_integration_spcs.yaml @@ -1,6 +1,9 @@ name: SPCS Integration testing on: + pull_request: + branches: + - release* push: tags: - "v*" # Push events to matching v*, i.e. v1.0, v20.15.10 diff --git a/RELEASE-NOTES.md b/RELEASE-NOTES.md index 386ac718a7..0e78d67655 100644 --- a/RELEASE-NOTES.md +++ b/RELEASE-NOTES.md @@ -34,6 +34,7 @@ * Added `snow helper check-snowsql-env-vars` which reports environment variables from SnowSQL with replacements in CLI. ## Fixes and improvements +* Updated MacOS postinstall script to update PATH if snow not exist. # v3.6.0 diff --git a/pyproject.toml b/pyproject.toml index ad9868837d..5dc387f5e3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -108,7 +108,6 @@ features = ["development", "packaging"] build-isolated-binary = ["python scripts/packaging/build_isolated_binary_with_hatch.py"] build-binaries = ["./scripts/packaging/build_binaries.sh"] build-packages = ["./scripts/packaging/build_packages.sh"] -package-darwin-binaries = ["./scripts/packaging/build_darwin_package.sh"] build-all = [ "./scripts/packaging/build_binaries.sh", "./scripts/packaging/build_packages.sh", diff --git a/scripts/packaging/build_darwin_package.sh b/scripts/packaging/build_darwin_package.sh index b4cf4c9c1e..cbcbe3a81a 100755 --- a/scripts/packaging/build_darwin_package.sh +++ b/scripts/packaging/build_darwin_package.sh @@ -1,19 +1,44 @@ #!/usr/bin/env bash set -xeuo pipefail -git config --global --add safe.directory /snowflake-cli -brew install -q tree - -ROOT_DIR=$(git rev-parse --show-toplevel) -PACKAGING_DIR=$ROOT_DIR/scripts/packaging - SYSTEM=$(uname -s | tr '[:upper:]' '[:lower:]') MACHINE=$(uname -m | tr '[:upper:]' '[:lower:]') PLATFORM="${SYSTEM}-${MACHINE}" +echo "--- creating virtualenv ---" +python3.11 -m venv venv +. venv/bin/activate +python --version + +echo "--- installing dependencies ---" +pip install hatch + +# install cargo +if [[ ${MACHINE} == "arm64" ]]; then + echo "installing cargo on arm64" + curl https://sh.rustup.rs -sSf | bash -s -- -y +elif [[ ${MACHINE} == "x86_64" ]]; then + echo "installing cargo on x86_64" + curl https://sh.rustup.rs -sSf | bash -s -- -y --no-modify-path + source $HOME/.cargo/env +else + echo "Unsupported machine: ${MACHINE}" + exit 1 +fi +rustup default stable + + +echo "--- setup variables ---" +BRANCH=${branch} +REVISION=$(git rev-parse ${svnRevision}) CLI_VERSION=$(hatch version) +STAGE_URL="s3://sfc-eng-jenkins/repository/snowflake-cli/${releaseType}/${SYSTEM}_${MACHINE}/${REVISION}/" + +ROOT_DIR=$(git rev-parse --show-toplevel) +PACKAGING_DIR=$ROOT_DIR/scripts/packaging DIST_DIR=$ROOT_DIR/dist + BINARY_NAME="snow-${CLI_VERSION}" APP_NAME="SnowflakeCLI.app" APP_DIR=$DIST_DIR/app @@ -21,6 +46,7 @@ APP_SCRIPTS=$APP_DIR/scripts CODESIGN_IDENTITY="Developer ID Application: Snowflake Computing INC. (W4NT6CRQ7U)" PRODUCTSIGN_IDENTITY="Developer ID Installer: Snowflake Computing INC. (W4NT6CRQ7U)" + loginfo() { logger -s -p INFO -- $1 } @@ -29,26 +55,6 @@ clean_build_workspace() { rm -rf $DIST_DIR || true } -install_cargo() { - curl https://sh.rustup.rs -sSf > rustup-init.sh - - if [[ ${MACHINE} == "arm64" ]]; then - sudo bash rustup-init.sh -y - . $HOME/.cargo/env - elif [[ ${MACHINE} == "x86_64" ]]; then - export CARGO_HOME="$HOME/.cargo" - export RUSTUP_HOME="$HOME/.rustup" - bash -s rustup-init.sh -y - . $HOME/.cargo/env - rustup default stable - else - echo "Unsupported machine: ${MACHINE}" - exit 1 - fi - - rm rustup-init.sh -} - create_app_template() { rm -r ${APP_DIR}/${APP_NAME} || true mkdir -p ${APP_DIR}/${APP_NAME}/Contents/MacOS @@ -61,9 +67,9 @@ loginfo "---------------------------------" security find-identity -v -p codesigning loginfo "---------------------------------" -clean_build_workspace -install_cargo +echo "--- build binary ---" +clean_build_workspace hatch -e packaging run build-isolated-binary create_app_template mv $DIST_DIR/binary/${BINARY_NAME} ${APP_DIR}/${APP_NAME}/Contents/MacOS/snow @@ -118,7 +124,6 @@ prepare_postinstall_script() { prepare_postinstall_script ls -l $DIST_DIR -tree -d $DIST_DIR chmod +x $APP_SCRIPTS/postinstall @@ -209,3 +214,9 @@ validate_installation() { } validate_installation $DIST_DIR/snowflake-cli-${CLI_VERSION}-${SYSTEM}-${MACHINE}.pkg + +echo "--- Upload artifacts to AWS ---" +ls -la ./dist/ +echo "${STAGE_URL}" +command -v aws +aws s3 cp ./dist/ ${STAGE_URL} --recursive --exclude "*" --include="snowflake-cli-${CLI_VERSION}*.pkg" diff --git a/scripts/packaging/macos/postinstall b/scripts/packaging/macos/postinstall index 2f5b70b198..b42938bda4 100755 --- a/scripts/packaging/macos/postinstall +++ b/scripts/packaging/macos/postinstall @@ -3,6 +3,7 @@ # $2 is the install location # SNOWFLAKE_CLI_COMMENT="# added by Snowflake SnowflakeCLI installer v1.0" +RC_FILES=(~/.zprofile ~/.zshrc ~/.profile ~/.bash_profile ~/.bashrc) function add_dest_path_to_profile() { local dest=$1 @@ -18,19 +19,21 @@ export PATH=$dest:\$PATH" >>$profile echo "[DEBUG] Parameters: $1 $2" SNOWFLAKE_CLI_DEST=$2/SnowflakeCLI.app/Contents/MacOS/ -SNOWFLAKE_CLI_LOGIN_SHELL=~/.profile -if [[ -e ~/.zprofile ]]; then - SNOWFLAKE_CLI_LOGIN_SHELL=~/.zprofile -elif [[ -e ~/.zshrc ]]; then - SNOWFLAKE_CLI_LOGIN_SHELL=~/.zshrc -elif [[ -e ~/.profile ]]; then - SNOWFLAKE_CLI_LOGIN_SHELL=~/.profile -elif [[ -e ~/.bash_profile ]]; then - SNOWFLAKE_CLI_LOGIN_SHELL=~/.bash_profile -elif [[ -e ~/.bashrc ]]; then - SNOWFLAKE_CLI_LOGIN_SHELL=~/.bashrc -fi +# List of potential login shell RC files + +# Check if the path is already in the PATH variable +if [[ ":$PATH:" == *":$SNOWFLAKE_CLI_DEST:"* ]]; then + echo "[INFO] Path $SNOWFLAKE_CLI_DEST is already in PATH. No changes needed." +else + for rc_file in "${RC_FILES[@]}"; do + # Expand tilde (~) to the user's home directory + rc_file_expanded=$(eval echo "$rc_file") -if ! grep -q -E "^$SNOWFLAKE_CLI_COMMENT" $SNOWFLAKE_CLI_LOGIN_SHELL; then - add_dest_path_to_profile $SNOWFLAKE_CLI_DEST $SNOWFLAKE_CLI_LOGIN_SHELL + if [[ -e "$rc_file_expanded" ]]; then + # Add the PATH update to the file + add_dest_path_to_profile "$SNOWFLAKE_CLI_DEST" "$rc_file_expanded" + else + echo "[INFO] $rc_file_expanded does not exist, skipping..." + fi + done fi diff --git a/scripts/packaging/setup_darwin.sh b/scripts/packaging/setup_darwin.sh deleted file mode 100644 index 68b66f875d..0000000000 --- a/scripts/packaging/setup_darwin.sh +++ /dev/null @@ -1,39 +0,0 @@ -#!/usr/bin/env bash -set -o pipefail - -echo "Setting up the Snowflake CLI build environment" -MACHINE=$(uname -m) - -ensure_pyenv_installation() { - if ! command -v pyenv &>/dev/null; then - echo "pyenv not found, installing..." - arch -${MACHINE} brew install pyenv - else - echo "pyenv already installed" - fi -} - -activate_pyenv() { - export PYENV_ROOT="$HOME/.pyenv" - [[ -d $PYENV_ROOT/bin ]] && export PATH="$PYENV_ROOT/bin:$PATH" - eval "$(pyenv init - bash)" -} - -ensure_hatch_installation() { - if ! command -v hatch &>/dev/null; then - echo "hatch not found, installing..." - arch -${MACHINE} brew install hatch - else - echo "hatch already installed" - arch -${MACHINE} brew upgrade hatch - fi -} - -ensure_python_installation() { - pyenv versions - pyenv install -s 3.10 - pyenv install -s 3.11 - pyenv global 3.11 - python --version - pip install -U pip uv hatch awscli -} diff --git a/src/snowflake/cli/__about__.py b/src/snowflake/cli/__about__.py index 28f4a9a39d..ef59c59b44 100644 --- a/src/snowflake/cli/__about__.py +++ b/src/snowflake/cli/__about__.py @@ -16,7 +16,7 @@ from enum import Enum, unique -VERSION = "3.7.0rc0" +VERSION = "3.7.0rc1" @unique diff --git a/src/snowflake/cli/_plugins/logs/commands.py b/src/snowflake/cli/_plugins/logs/commands.py index 67c7a5d820..eccfafefb3 100644 --- a/src/snowflake/cli/_plugins/logs/commands.py +++ b/src/snowflake/cli/_plugins/logs/commands.py @@ -4,9 +4,11 @@ import typer from click import ClickException -from snowflake.cli._plugins.logs.manager import LogsManager, LogsQueryRow +from snowflake.cli._plugins.logs.manager import LogsManager +from snowflake.cli._plugins.logs.utils import LOG_LEVELS, LogsQueryRow from snowflake.cli._plugins.object.commands import NameArgument, ObjectArgument from snowflake.cli.api.commands.snow_typer import SnowTyperFactory +from snowflake.cli.api.exceptions import CliArgumentError from snowflake.cli.api.identifiers import FQN from snowflake.cli.api.output.types import ( CommandResult, @@ -41,11 +43,22 @@ def get_logs( "--table", help="The table to query for logs. If not provided, the default table will be used", ), + log_level: Optional[str] = typer.Option( + "INFO", + "--log-level", + help="The log level to filter by. If not provided, INFO will be used", + ), **options, ): """ Retrieves logs for a given object. """ + + if log_level and not log_level.upper() in LOG_LEVELS: + raise CliArgumentError( + f"Invalid log level. Please choose from {', '.join(LOG_LEVELS)}" + ) + if refresh_time and to: raise ClickException( "You cannot set both --refresh and --to parameters. Please check the values" @@ -61,6 +74,7 @@ def get_logs( from_time=from_time, refresh_time=refresh_time, event_table=event_table, + log_level=log_level, ) logs = itertools.chain( (MessageResult(log.log_message) for logs in logs_stream for log in logs) @@ -72,6 +86,7 @@ def get_logs( from_time=from_time, to_time=to_time, event_table=event_table, + log_level=log_level, ) logs = (MessageResult(log.log_message) for log in logs_iterable) # type: ignore diff --git a/src/snowflake/cli/_plugins/logs/manager.py b/src/snowflake/cli/_plugins/logs/manager.py index 5a2db4822d..d999dfb3c2 100644 --- a/src/snowflake/cli/_plugins/logs/manager.py +++ b/src/snowflake/cli/_plugins/logs/manager.py @@ -1,26 +1,19 @@ import time from datetime import datetime from textwrap import dedent -from typing import Iterable, List, NamedTuple, Optional, Tuple +from typing import Iterable, List, Optional -from click import ClickException +from snowflake.cli._plugins.logs.utils import ( + LogsQueryRow, + get_timestamp_query, + parse_log_levels_for_query, + sanitize_logs, +) from snowflake.cli._plugins.object.commands import NameArgument, ObjectArgument from snowflake.cli.api.identifiers import FQN from snowflake.cli.api.sql_execution import SqlExecutionMixin from snowflake.connector.cursor import SnowflakeCursor -LogsQueryRow = NamedTuple( - "LogsQueryRow", - [ - ("timestamp", datetime), - ("database_name", str), - ("schema_name", str), - ("object_name", str), - ("log_level", str), - ("log_message", str), - ], -) - class LogsManager(SqlExecutionMixin): def stream_logs( @@ -30,6 +23,7 @@ def stream_logs( object_name: FQN = NameArgument, from_time: Optional[datetime] = None, event_table: Optional[str] = None, + log_level: Optional[str] = "INFO", ) -> Iterable[List[LogsQueryRow]]: try: previous_end = from_time @@ -41,6 +35,7 @@ def stream_logs( from_time=previous_end, to_time=None, event_table=event_table, + log_level=log_level, ).fetchall() if raw_logs: @@ -60,6 +55,7 @@ def get_logs( from_time: Optional[datetime] = None, to_time: Optional[datetime] = None, event_table: Optional[str] = None, + log_level: Optional[str] = "INFO", ) -> Iterable[LogsQueryRow]: """ Basic function to get a single batch of logs from the server @@ -71,9 +67,10 @@ def get_logs( from_time=from_time, to_time=to_time, event_table=event_table, + log_level=log_level, ) - return self.sanitize_logs(logs) + return sanitize_logs(logs) def get_raw_logs( self, @@ -82,6 +79,7 @@ def get_raw_logs( from_time: Optional[datetime] = None, to_time: Optional[datetime] = None, event_table: Optional[str] = None, + log_level: Optional[str] = "INFO", ) -> SnowflakeCursor: table = event_table if event_table else "SNOWFLAKE.TELEMETRY.EVENTS" @@ -97,9 +95,9 @@ def get_raw_logs( value::string as log_message FROM {table} WHERE record_type = 'LOG' - AND (record:severity_text = 'INFO' or record:severity_text is NULL ) + AND (record:severity_text IN ({parse_log_levels_for_query((log_level))}) or record:severity_text is NULL ) AND object_name = '{object_name}' - {self._get_timestamp_query(from_time, to_time)} + {get_timestamp_query(from_time, to_time)} ORDER BY timestamp; """ ).strip() @@ -107,32 +105,3 @@ def get_raw_logs( result = self.execute_query(query) return result - - def _get_timestamp_query( - self, from_time: Optional[datetime], to_time: Optional[datetime] - ): - if from_time and to_time and from_time > to_time: - raise ClickException( - "From_time cannot be later than to_time. Please check the values" - ) - query = [] - - if from_time is not None: - query.append( - f"AND timestamp >= TO_TIMESTAMP_LTZ('{from_time.isoformat()}')\n" - ) - - if to_time is not None: - query.append( - f"AND timestamp <= TO_TIMESTAMP_LTZ('{to_time.isoformat()}')\n" - ) - - return "".join(query) - - def sanitize_logs(self, logs: SnowflakeCursor | List[Tuple]) -> List[LogsQueryRow]: - try: - return [LogsQueryRow(*log) for log in logs] - except TypeError: - raise ClickException( - "Logs table has incorrect format. Please check the logs_table in your database" - ) diff --git a/src/snowflake/cli/_plugins/logs/utils.py b/src/snowflake/cli/_plugins/logs/utils.py new file mode 100644 index 0000000000..dbd45da433 --- /dev/null +++ b/src/snowflake/cli/_plugins/logs/utils.py @@ -0,0 +1,60 @@ +from datetime import datetime +from typing import List, NamedTuple, Optional, Tuple + +from snowflake.cli.api.exceptions import CliArgumentError, CliSqlError +from snowflake.connector.cursor import SnowflakeCursor + +LOG_LEVELS = ["TRACE", "DEBUG", "INFO", "WARN", "ERROR", "FATAL"] + +LogsQueryRow = NamedTuple( + "LogsQueryRow", + [ + ("timestamp", datetime), + ("database_name", str), + ("schema_name", str), + ("object_name", str), + ("log_level", str), + ("log_message", str), + ], +) + + +def sanitize_logs(logs: SnowflakeCursor | List[Tuple]) -> List[LogsQueryRow]: + try: + return [LogsQueryRow(*log) for log in logs] + except TypeError: + raise CliSqlError( + "Logs table has incorrect format. Please check the logs_table in your database" + ) + + +def get_timestamp_query(from_time: Optional[datetime], to_time: Optional[datetime]): + if from_time and to_time and from_time > to_time: + raise CliArgumentError( + "From_time cannot be later than to_time. Please check the values" + ) + query = [] + + if from_time is not None: + query.append(f"AND timestamp >= TO_TIMESTAMP_LTZ('{from_time.isoformat()}')\n") + + if to_time is not None: + query.append(f"AND timestamp <= TO_TIMESTAMP_LTZ('{to_time.isoformat()}')\n") + + return "".join(query) + + +def get_log_levels(log_level: str): + if log_level.upper() not in LOG_LEVELS and log_level != "": + raise CliArgumentError( + f"Invalid log level. Please choose from {', '.join(LOG_LEVELS)}" + ) + + if log_level == "": + log_level = "INFO" + + return LOG_LEVELS[LOG_LEVELS.index(log_level.upper()) :] + + +def parse_log_levels_for_query(log_level: str): + return ", ".join(f"'{level}'" for level in get_log_levels(log_level)) diff --git a/tests/__snapshots__/test_help_messages.ambr b/tests/__snapshots__/test_help_messages.ambr index 57f0e6eff5..ebf0de778c 100644 --- a/tests/__snapshots__/test_help_messages.ambr +++ b/tests/__snapshots__/test_help_messages.ambr @@ -5118,19 +5118,22 @@ | [required] | +------------------------------------------------------------------------------+ +- Options --------------------------------------------------------------------+ - | --from TEXT The start time of the logs to retrieve. Accepts | - | all ISO8061 formats | - | [default: None] | - | --to TEXT The end time of the logs to retrieve. Accepts | - | all ISO8061 formats | - | [default: None] | - | --refresh INTEGER If set, the logs will be streamed with the given | - | refresh time in seconds | - | [default: None] | - | --table TEXT The table to query for logs. If not provided, | - | the default table will be used | - | [default: None] | - | --help -h Show this message and exit. | + | --from TEXT The start time of the logs to retrieve. | + | Accepts all ISO8061 formats | + | [default: None] | + | --to TEXT The end time of the logs to retrieve. Accepts | + | all ISO8061 formats | + | [default: None] | + | --refresh INTEGER If set, the logs will be streamed with the | + | given refresh time in seconds | + | [default: None] | + | --table TEXT The table to query for logs. If not provided, | + | the default table will be used | + | [default: None] | + | --log-level TEXT The log level to filter by. If not provided, | + | INFO will be used | + | [default: INFO] | + | --help -h Show this message and exit. | +------------------------------------------------------------------------------+ +- Connection configuration ---------------------------------------------------+ | --connection,--environment -c TEXT Name of the connection, as | diff --git a/tests/logs/__snapshots__/test_logs.ambr b/tests/logs/__snapshots__/test_logs.ambr index c01ff40ebd..89517c6405 100644 --- a/tests/logs/__snapshots__/test_logs.ambr +++ b/tests/logs/__snapshots__/test_logs.ambr @@ -10,7 +10,7 @@ value::string as log_message FROM SNOWFLAKE.TELEMETRY.EVENTS WHERE record_type = 'LOG' - AND (record:severity_text = 'INFO' or record:severity_text is NULL ) + AND (record:severity_text IN ('INFO', 'WARN', 'ERROR', 'FATAL') or record:severity_text is NULL ) AND object_name = 'bar' AND timestamp >= TO_TIMESTAMP_LTZ('2022-02-02T02:02:02') AND timestamp <= TO_TIMESTAMP_LTZ('2022-02-03T02:02:02') @@ -29,7 +29,7 @@ value::string as log_message FROM bar WHERE record_type = 'LOG' - AND (record:severity_text = 'INFO' or record:severity_text is NULL ) + AND (record:severity_text IN ('INFO', 'WARN', 'ERROR', 'FATAL') or record:severity_text is NULL ) AND object_name = 'bar' AND timestamp >= TO_TIMESTAMP_LTZ('2022-02-02T02:02:02') AND timestamp <= TO_TIMESTAMP_LTZ('2022-02-03T02:02:02') @@ -48,7 +48,7 @@ value::string as log_message FROM foo WHERE record_type = 'LOG' - AND (record:severity_text = 'INFO' or record:severity_text is NULL ) + AND (record:severity_text IN ('INFO', 'WARN', 'ERROR', 'FATAL') or record:severity_text is NULL ) AND object_name = 'bar' AND timestamp >= TO_TIMESTAMP_LTZ('2022-02-02T02:02:02') AND timestamp <= TO_TIMESTAMP_LTZ('2022-02-03T02:02:02') @@ -56,6 +56,14 @@ ORDER BY timestamp; ''' # --- +# name: test_if_incorrect_log_level_causes_error + ''' + +- Error ----------------------------------------------------------------------+ + | Invalid log level. Please choose from TRACE, DEBUG, INFO, WARN, ERROR, FATAL | + +------------------------------------------------------------------------------+ + + ''' +# --- # name: test_providing_time_in_incorrect_format_causes_error[2024-11-03 12:00:00 UTC---from] ''' +- Error ----------------------------------------------------------------------+ diff --git a/tests/logs/test_logs.py b/tests/logs/test_logs.py index f8da712c96..718c1a0c98 100644 --- a/tests/logs/test_logs.py +++ b/tests/logs/test_logs.py @@ -77,3 +77,17 @@ def test_correct_query_is_constructed(mock_connect, mock_ctx, runner, snapshot, queries = ctx.get_queries() assert len(queries) == 1 assert queries[0] == snapshot + + +def test_if_incorrect_log_level_causes_error(runner, snapshot): + result = runner.invoke( + [ + "logs", + "table", + "test_table", + "--log-level", + "NOTALEVEL", + ] + ) + assert result.exit_code == 1 + assert result.output == snapshot diff --git a/tests/logs/test_logs_manager.py b/tests/logs/test_logs_utils.py similarity index 68% rename from tests/logs/test_logs_manager.py rename to tests/logs/test_logs_utils.py index bf98494d54..d6df2eec77 100644 --- a/tests/logs/test_logs_manager.py +++ b/tests/logs/test_logs_utils.py @@ -5,7 +5,11 @@ from snowflake.cli._plugins.logs.commands import ( get_datetime_from_string, ) -from snowflake.cli._plugins.logs.manager import LogsManager +from snowflake.cli._plugins.logs.utils import ( + get_log_levels, + get_timestamp_query, + parse_log_levels_for_query, +) DATETIME_FORMAT = "%Y-%m-%d %H:%M:%S" @@ -74,10 +78,45 @@ def test_if_passing_to_time_earlier_than_from_time_raiser_error(): to_time = from_time - timedelta(hours=1) with pytest.raises(ClickException) as e: - LogsManager()._get_timestamp_query(from_time=from_time, to_time=to_time) # noqa + get_timestamp_query(from_time=from_time, to_time=to_time) # noqa assert ( str(e.value) == "From_time cannot be later than to_time. Please check the values" ) assert e.value.exit_code == 1 + + +@pytest.mark.parametrize( + "log_level,expected", + [ + ("", ["INFO", "WARN", "ERROR", "FATAL"]), + ("TRACE", ["TRACE", "DEBUG", "INFO", "WARN", "ERROR", "FATAL"]), + ("DEBUG", ["DEBUG", "INFO", "WARN", "ERROR", "FATAL"]), + ("INFO", ["INFO", "WARN", "ERROR", "FATAL"]), + ("WARN", ["WARN", "ERROR", "FATAL"]), + ("ERROR", ["ERROR", "FATAL"]), + ("FATAL", ["FATAL"]), + ("fatal", ["FATAL"]), + ("eRrOr", ["ERROR", "FATAL"]), + ], +) +def test_if_log_levels_list_is_correctly_filtered(log_level, expected): + result = get_log_levels(log_level) + + assert result == expected + + +@pytest.mark.parametrize( + "level,expected", + [ + ("", "'INFO', 'WARN', 'ERROR', 'FATAL'"), + ("INFO", "'INFO', 'WARN', 'ERROR', 'FATAL'"), + ("DEBUG", "'DEBUG', 'INFO', 'WARN', 'ERROR', 'FATAL'"), + ("wArN", "'WARN', 'ERROR', 'FATAL'"), + ], +) +def test_if_log_level_gives_correct_query(level, expected): + result = parse_log_levels_for_query(level) + + assert result == expected diff --git a/tests_integration/test_data/projects/snowpark_v2/c.py b/tests_integration/test_data/projects/snowpark_v2/c.py index 3ab4a6d6cc..14aed7edba 100644 --- a/tests_integration/test_data/projects/snowpark_v2/c.py +++ b/tests_integration/test_data/projects/snowpark_v2/c.py @@ -3,7 +3,14 @@ # test import import syrupy +import logging + +log = logging.getLogger("SnowCLI_Logs_Test") def hello_function(name: str) -> str: + log.debug("This is a debug message") + log.info("This is an info message") + log.warning("This is a warning message") + log.error("This is an error message") return f"Hello {name}!" From e4bc8de83f5c8cf599c659c2e410ed67697cdfe0 Mon Sep 17 00:00:00 2001 From: Patryk Czajka Date: Wed, 16 Apr 2025 12:34:18 +0200 Subject: [PATCH 04/54] Cherrypicks v3.7.0 (#2208) Bump version to v3.7.0 --- src/snowflake/cli/__about__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/snowflake/cli/__about__.py b/src/snowflake/cli/__about__.py index ef59c59b44..de42820ac0 100644 --- a/src/snowflake/cli/__about__.py +++ b/src/snowflake/cli/__about__.py @@ -16,7 +16,7 @@ from enum import Enum, unique -VERSION = "3.7.0rc1" +VERSION = "3.7.0" @unique From c39ed5e57719e3f2ba4a32b823c885beb900835b Mon Sep 17 00:00:00 2001 From: Patryk Czajka Date: Mon, 28 Apr 2025 14:54:08 +0200 Subject: [PATCH 05/54] Cherry picks to 3.7.1 (#2221) * Bump version to v3.7.1 * Fix connection certificate issues (#2220) * commit confitming certifi problematic version * Temporarly pin certifi to 2025.1.31 * update release notes * adding faster query to `snow spcs image-registry login` (#2222) * Patch * Post- review fix * Update RELEASE-NOTES.md --------- Co-authored-by: Patryk Czajka * update release notes --------- Co-authored-by: Jan Sikorski <132985823+sfc-gh-jsikorski@users.noreply.github.com> --- RELEASE-NOTES.md | 10 +++++ pyproject.toml | 1 + snyk/requirements.txt | 1 + src/snowflake/cli/__about__.py | 2 +- .../_plugins/spcs/image_registry/manager.py | 21 +++++++--- tests/spcs/test_registry.py | 40 +++++++++++++++++-- 6 files changed, 64 insertions(+), 11 deletions(-) diff --git a/RELEASE-NOTES.md b/RELEASE-NOTES.md index 0e78d67655..6bca31632b 100644 --- a/RELEASE-NOTES.md +++ b/RELEASE-NOTES.md @@ -23,6 +23,16 @@ ## Fixes and improvements +# v3.7.1 + +## Deprecations + +## New additions + +## Fixes and improvements +* Fix certificate connection issues. +* Fix `snow spcs image-registry login` slow query problem. + # v3.7.0 ## Deprecations diff --git a/pyproject.toml b/pyproject.toml index 5dc387f5e3..1d8a1bb17c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -25,6 +25,7 @@ requires-python = ">=3.10" description = "Snowflake CLI" readme = "README.md" dependencies = [ + "certifi==2025.1.31", "GitPython==3.1.44", "jinja2==3.1.6", "packaging", diff --git a/snyk/requirements.txt b/snyk/requirements.txt index 7cd1828e5f..9bd44418e4 100644 --- a/snyk/requirements.txt +++ b/snyk/requirements.txt @@ -1,3 +1,4 @@ +certifi==2025.1.31 GitPython==3.1.44 jinja2==3.1.6 packaging diff --git a/src/snowflake/cli/__about__.py b/src/snowflake/cli/__about__.py index de42820ac0..31cafe5a12 100644 --- a/src/snowflake/cli/__about__.py +++ b/src/snowflake/cli/__about__.py @@ -16,7 +16,7 @@ from enum import Enum, unique -VERSION = "3.7.0" +VERSION = "3.7.1" @unique diff --git a/src/snowflake/cli/_plugins/spcs/image_registry/manager.py b/src/snowflake/cli/_plugins/spcs/image_registry/manager.py index fe051beec7..4d11287e47 100644 --- a/src/snowflake/cli/_plugins/spcs/image_registry/manager.py +++ b/src/snowflake/cli/_plugins/spcs/image_registry/manager.py @@ -74,12 +74,21 @@ def _has_url_scheme(self, url: str): return re.fullmatch(r"^.*//.+", url) is not None def get_registry_url(self) -> str: - repositories_query = "show image repositories in account" - result_set = self.execute_query(repositories_query, cursor_class=DictCursor) - results = result_set.fetchall() - if len(results) == 0: - raise NoImageRepositoriesFoundError() - sample_repository_url = results[0]["repository_url"] + images_query = "show image repositories in schema snowflake.images;" + images_result = self.execute_query(images_query, cursor_class=DictCursor) + + results = images_result.fetchone() + + if not results: + # fallback to account level query - slower one, so we try to avoid it if possible + repositories_query = "show image repositories in account" + result_set = self.execute_query(repositories_query, cursor_class=DictCursor) + results = result_set.fetchone() + + if not results: + raise NoImageRepositoriesFoundError() + + sample_repository_url = results["repository_url"] if not self._has_url_scheme(sample_repository_url): sample_repository_url = f"//{sample_repository_url}" return urlparse(sample_repository_url).netloc diff --git a/tests/spcs/test_registry.py b/tests/spcs/test_registry.py index 816d1a7532..be8e14d06c 100644 --- a/tests/spcs/test_registry.py +++ b/tests/spcs/test_registry.py @@ -73,11 +73,40 @@ def test_get_registry_url(mock_execute, mock_conn, mock_cursor): ] mock_execute.return_value = mock_cursor( - rows=[{col: row for col, row in zip(MOCK_REPO_COLUMNS, mock_row)}], + rows=[{}, {col: row for col, row in zip(MOCK_REPO_COLUMNS, mock_row)}], columns=MOCK_REPO_COLUMNS, ) result = RegistryManager().get_registry_url() expected_query = "show image repositories in account" + assert mock_execute.call_count == 2 + mock_execute.assert_any_call(expected_query, cursor_class=DictCursor) + assert result == "orgname-alias.registry.snowflakecomputing.com" + + +@mock.patch("snowflake.cli._plugins.spcs.image_registry.manager.RegistryManager._conn") +@mock.patch( + "snowflake.cli._plugins.spcs.image_registry.manager.RegistryManager.execute_query" +) +def test_get_registry_url_with_schema_query(mock_execute, mock_conn, mock_cursor): + mock_row = [ + "2023-01-01 00:00:00", + "IMAGES", + "DB", + "SCHEMA", + "orgname-alias.registry.snowflakecomputing.com/DB/SCHEMA/IMAGES", + "TEST_ROLE", + "ROLE", + "", + ] + + mock_execute.return_value = mock_cursor( + rows=[{col: row for col, row in zip(MOCK_REPO_COLUMNS, mock_row)}], + columns=MOCK_REPO_COLUMNS, + ) + + result = RegistryManager().get_registry_url() + expected_query = "show image repositories in schema snowflake.images;" + mock_execute.assert_called_once_with(expected_query, cursor_class=DictCursor) assert result == "orgname-alias.registry.snowflakecomputing.com" @@ -88,14 +117,17 @@ def test_get_registry_url(mock_execute, mock_conn, mock_cursor): ) def test_get_registry_url_no_repositories(mock_execute, mock_conn, mock_cursor): mock_execute.return_value = mock_cursor( - rows=[], + rows=[{}, {}], columns=MOCK_REPO_COLUMNS, ) with pytest.raises(NoImageRepositoriesFoundError): RegistryManager().get_registry_url() - expected_query = "show image repositories in account" - mock_execute.assert_called_once_with(expected_query, cursor_class=DictCursor) + expected_query1 = "show image repositories in schema snowflake.images;" + expected_query2 = "show image repositories in account" + assert mock_execute.call_count == 2 + mock_execute.assert_any_call(expected_query1, cursor_class=DictCursor) + mock_execute.assert_any_call(expected_query2, cursor_class=DictCursor) @mock.patch( From 6f4fe4ca952089435ac0913e1d49b20a4d687376 Mon Sep 17 00:00:00 2001 From: Patryk Czajka Date: Mon, 12 May 2025 11:48:38 +0200 Subject: [PATCH 06/54] Bump version to v3.7.2 --- src/snowflake/cli/__about__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/snowflake/cli/__about__.py b/src/snowflake/cli/__about__.py index 31cafe5a12..67f19f8b9c 100644 --- a/src/snowflake/cli/__about__.py +++ b/src/snowflake/cli/__about__.py @@ -16,7 +16,7 @@ from enum import Enum, unique -VERSION = "3.7.1" +VERSION = "3.7.2" @unique From f03b9bcc1bc1a05bccc84b90933653b92153cffa Mon Sep 17 00:00:00 2001 From: Patryk Czajka Date: Mon, 12 May 2025 11:53:08 +0200 Subject: [PATCH 07/54] bump snowflake-connector-python to 3.15 --- pyproject.toml | 2 +- snyk/requirements.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 1d8a1bb17c..1fa495b1c9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -37,7 +37,7 @@ dependencies = [ "requirements-parser==0.11.0", "rich==14.0.0", "setuptools==78.1.0", - "snowflake-connector-python[secure-local-storage]==3.14.0", + "snowflake-connector-python[secure-local-storage]==3.15.0", 'snowflake-snowpark-python>=1.15.0,<1.26.0;python_version < "3.12"', 'snowflake.core==1.2.0; python_version < "3.12"', "tomlkit==0.13.2", diff --git a/snyk/requirements.txt b/snyk/requirements.txt index 9bd44418e4..d1e27518e5 100644 --- a/snyk/requirements.txt +++ b/snyk/requirements.txt @@ -10,7 +10,7 @@ requests==2.32.3 requirements-parser==0.11.0 rich==14.0.0 setuptools==78.1.0 -snowflake-connector-python[secure-local-storage]==3.14.0 +snowflake-connector-python[secure-local-storage]==3.15.0 snowflake-snowpark-python>=1.15.0,<1.26.0;python_version < "3.12" snowflake.core==1.2.0; python_version < "3.12" tomlkit==0.13.2 From 07b0553a6c74325a115ba6507218602d17d846a6 Mon Sep 17 00:00:00 2001 From: Patryk Czajka Date: Mon, 5 May 2025 11:02:54 +0200 Subject: [PATCH 08/54] Remove certifi depencency (#2238) --- pyproject.toml | 1 - snyk/requirements.txt | 1 - 2 files changed, 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 1fa495b1c9..fb27a5d88d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -25,7 +25,6 @@ requires-python = ">=3.10" description = "Snowflake CLI" readme = "README.md" dependencies = [ - "certifi==2025.1.31", "GitPython==3.1.44", "jinja2==3.1.6", "packaging", diff --git a/snyk/requirements.txt b/snyk/requirements.txt index d1e27518e5..812a79786e 100644 --- a/snyk/requirements.txt +++ b/snyk/requirements.txt @@ -1,4 +1,3 @@ -certifi==2025.1.31 GitPython==3.1.44 jinja2==3.1.6 packaging From a3f3f671fe26fe63f02bc5d756dedb88353e053c Mon Sep 17 00:00:00 2001 From: Patryk Czajka Date: Mon, 12 May 2025 10:56:03 +0200 Subject: [PATCH 09/54] fix error showing for --help messages (#2272) --- pyproject.toml | 1 + snyk/requirements.txt | 1 + 2 files changed, 2 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index fb27a5d88d..0debbdfd6e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -25,6 +25,7 @@ requires-python = ">=3.10" description = "Snowflake CLI" readme = "README.md" dependencies = [ + "click==8.1.8", "GitPython==3.1.44", "jinja2==3.1.6", "packaging", diff --git a/snyk/requirements.txt b/snyk/requirements.txt index 812a79786e..613d703352 100644 --- a/snyk/requirements.txt +++ b/snyk/requirements.txt @@ -1,3 +1,4 @@ +click==8.1.8 GitPython==3.1.44 jinja2==3.1.6 packaging From c58c33a166381cfd73644553f96c42f096210ff5 Mon Sep 17 00:00:00 2001 From: Patryk Czajka Date: Mon, 12 May 2025 11:59:51 +0200 Subject: [PATCH 10/54] update release notes --- RELEASE-NOTES.md | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/RELEASE-NOTES.md b/RELEASE-NOTES.md index 6bca31632b..3a7e49f969 100644 --- a/RELEASE-NOTES.md +++ b/RELEASE-NOTES.md @@ -23,6 +23,16 @@ ## Fixes and improvements +# v3.7.2 + +## Deprecations + +## New additions + +## Fixes and improvements +* Fix error appearing on help messages after click BCR update. + + # v3.7.1 ## Deprecations From 65a20589671d73a4c525bd07a7903660bbc9c1c2 Mon Sep 17 00:00:00 2001 From: Patryk Czajka Date: Thu, 10 Apr 2025 15:52:25 +0200 Subject: [PATCH 11/54] Bump release notes 3.7.0 (#2192) * Update release notes for 3.7.0 * Bump dev version to 3.8.0 --- src/snowflake/cli/__about__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/snowflake/cli/__about__.py b/src/snowflake/cli/__about__.py index 67f19f8b9c..2ba4191de3 100644 --- a/src/snowflake/cli/__about__.py +++ b/src/snowflake/cli/__about__.py @@ -16,7 +16,7 @@ from enum import Enum, unique -VERSION = "3.7.2" +VERSION = "3.8.0.dev0" @unique From 2a9244f4ce812a399ca9194815e77be0bf7f7a43 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 15 Apr 2025 09:31:46 +0200 Subject: [PATCH 12/54] Bump pydantic from 2.11.2 to 2.11.3 (#2200) Bumps [pydantic](https://github.com/pydantic/pydantic) from 2.11.2 to 2.11.3. - [Release notes](https://github.com/pydantic/pydantic/releases) - [Changelog](https://github.com/pydantic/pydantic/blob/main/HISTORY.md) - [Commits](https://github.com/pydantic/pydantic/compare/v2.11.2...v2.11.3) --- updated-dependencies: - dependency-name: pydantic dependency-version: 2.11.3 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- pyproject.toml | 2 +- snyk/requirements.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 0debbdfd6e..faffd044b4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -31,7 +31,7 @@ dependencies = [ "packaging", "pip", "pluggy==1.5.0", - "pydantic==2.11.2", + "pydantic==2.11.3", "PyYAML==6.0.2", "requests==2.32.3", "requirements-parser==0.11.0", diff --git a/snyk/requirements.txt b/snyk/requirements.txt index 613d703352..9e08998556 100644 --- a/snyk/requirements.txt +++ b/snyk/requirements.txt @@ -4,7 +4,7 @@ jinja2==3.1.6 packaging pip pluggy==1.5.0 -pydantic==2.11.2 +pydantic==2.11.3 PyYAML==6.0.2 requests==2.32.3 requirements-parser==0.11.0 From 8f89372d5126c93ae19ba94677614231d73bc018 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 15 Apr 2025 09:32:16 +0200 Subject: [PATCH 13/54] Bump pytest-httpserver from 1.1.2 to 1.1.3 (#2199) Bumps [pytest-httpserver](https://github.com/csernazs/pytest-httpserver) from 1.1.2 to 1.1.3. - [Release notes](https://github.com/csernazs/pytest-httpserver/releases) - [Changelog](https://github.com/csernazs/pytest-httpserver/blob/master/CHANGES.rst) - [Commits](https://github.com/csernazs/pytest-httpserver/compare/1.1.2...1.1.3) --- updated-dependencies: - dependency-name: pytest-httpserver dependency-version: 1.1.3 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- pyproject.toml | 2 +- snyk/requirements.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index faffd044b4..6857da5769 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -62,7 +62,7 @@ development = [ "pre-commit>=3.5.0", "pytest==8.3.5", "pytest-randomly==3.16.0", - "pytest-httpserver==1.1.2", + "pytest-httpserver==1.1.3", "syrupy==4.9.1", "factory-boy==3.3.3", "Faker==37.1.0", diff --git a/snyk/requirements.txt b/snyk/requirements.txt index 9e08998556..30023b7e72 100644 --- a/snyk/requirements.txt +++ b/snyk/requirements.txt @@ -20,7 +20,7 @@ coverage==7.8.0 pre-commit>=3.5.0 pytest==8.3.5 pytest-randomly==3.16.0 -pytest-httpserver==1.1.2 +pytest-httpserver==1.1.3 syrupy==4.9.1 factory-boy==3.3.3 Faker==37.1.0 From c65bdbd11cf3fec2f9446060efd4b7628b52436a Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 15 Apr 2025 09:37:23 +0200 Subject: [PATCH 14/54] Update urllib3 requirement from <2.4,>=1.24.3 to >=1.24.3,<2.5 (#2198) Updates the requirements on [urllib3](https://github.com/urllib3/urllib3) to permit the latest version. - [Release notes](https://github.com/urllib3/urllib3/releases) - [Changelog](https://github.com/urllib3/urllib3/blob/main/CHANGES.rst) - [Commits](https://github.com/urllib3/urllib3/compare/1.24.3...2.4.0) --- updated-dependencies: - dependency-name: urllib3 dependency-version: 2.4.0 dependency-type: direct:production ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- pyproject.toml | 2 +- snyk/requirements.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 6857da5769..28604cebf6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -42,7 +42,7 @@ dependencies = [ 'snowflake.core==1.2.0; python_version < "3.12"', "tomlkit==0.13.2", "typer==0.15.2", - "urllib3>=1.24.3,<2.4", + "urllib3>=1.24.3,<2.5", ] classifiers = [ "Development Status :: 5 - Production/Stable", diff --git a/snyk/requirements.txt b/snyk/requirements.txt index 30023b7e72..c103a7b6e6 100644 --- a/snyk/requirements.txt +++ b/snyk/requirements.txt @@ -15,7 +15,7 @@ snowflake-snowpark-python>=1.15.0,<1.26.0;python_version < "3.12" snowflake.core==1.2.0; python_version < "3.12" tomlkit==0.13.2 typer==0.15.2 -urllib3>=1.24.3,<2.4 +urllib3>=1.24.3,<2.5 coverage==7.8.0 pre-commit>=3.5.0 pytest==8.3.5 From 5798b1bbd777c1fa8141225faea4ec294c646e02 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 15 Apr 2025 08:03:06 +0000 Subject: [PATCH 15/54] Bump snowflake-core from 1.2.0 to 1.3.0 (#2201) * Bump snowflake-core from 1.2.0 to 1.3.0 Bumps snowflake-core from 1.2.0 to 1.3.0. --- updated-dependencies: - dependency-name: snowflake-core dependency-version: 1.3.0 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] * bump: update snowflake.core to version 1.3.0 --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Adam Stus --- pyproject.toml | 2 +- snyk/requirements.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 28604cebf6..b2ea75954b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -39,7 +39,7 @@ dependencies = [ "setuptools==78.1.0", "snowflake-connector-python[secure-local-storage]==3.15.0", 'snowflake-snowpark-python>=1.15.0,<1.26.0;python_version < "3.12"', - 'snowflake.core==1.2.0; python_version < "3.12"', + 'snowflake.core==1.3.0; python_version < "3.12"', "tomlkit==0.13.2", "typer==0.15.2", "urllib3>=1.24.3,<2.5", diff --git a/snyk/requirements.txt b/snyk/requirements.txt index c103a7b6e6..aaaeb3ffaa 100644 --- a/snyk/requirements.txt +++ b/snyk/requirements.txt @@ -12,7 +12,7 @@ rich==14.0.0 setuptools==78.1.0 snowflake-connector-python[secure-local-storage]==3.15.0 snowflake-snowpark-python>=1.15.0,<1.26.0;python_version < "3.12" -snowflake.core==1.2.0; python_version < "3.12" +snowflake.core==1.3.0; python_version < "3.12" tomlkit==0.13.2 typer==0.15.2 urllib3>=1.24.3,<2.5 From 49d7dc05e0a5f8828de058012a17cb2f9dc0791e Mon Sep 17 00:00:00 2001 From: Jakub Wilkowski Date: Thu, 13 Feb 2025 18:18:20 +0100 Subject: [PATCH 16/54] feat: [SNOW-1890085] add dbt command group; add dbt list command --- .../commands_registration/builtin_plugins.py | 2 + src/snowflake/cli/_plugins/dbt/__init__.py | 13 ++++++ src/snowflake/cli/_plugins/dbt/commands.py | 41 +++++++++++++++++++ src/snowflake/cli/_plugins/dbt/manager.py | 24 +++++++++++ src/snowflake/cli/_plugins/dbt/plugin_spec.py | 30 ++++++++++++++ tests/dbt/__init__.py | 13 ++++++ tests/dbt/test_dbt_commands.py | 34 +++++++++++++++ tests/test_help_messages.py | 2 +- 8 files changed, 158 insertions(+), 1 deletion(-) create mode 100644 src/snowflake/cli/_plugins/dbt/__init__.py create mode 100644 src/snowflake/cli/_plugins/dbt/commands.py create mode 100644 src/snowflake/cli/_plugins/dbt/manager.py create mode 100644 src/snowflake/cli/_plugins/dbt/plugin_spec.py create mode 100644 tests/dbt/__init__.py create mode 100644 tests/dbt/test_dbt_commands.py diff --git a/src/snowflake/cli/_app/commands_registration/builtin_plugins.py b/src/snowflake/cli/_app/commands_registration/builtin_plugins.py index 14012a48c4..f3c8a75e6d 100644 --- a/src/snowflake/cli/_app/commands_registration/builtin_plugins.py +++ b/src/snowflake/cli/_app/commands_registration/builtin_plugins.py @@ -15,6 +15,7 @@ from snowflake.cli._plugins.auth.keypair import plugin_spec as auth_plugin_spec from snowflake.cli._plugins.connection import plugin_spec as connection_plugin_spec from snowflake.cli._plugins.cortex import plugin_spec as cortex_plugin_spec +from snowflake.cli._plugins.dbt import plugin_spec as dbt_plugin_spec from snowflake.cli._plugins.git import plugin_spec as git_plugin_spec from snowflake.cli._plugins.helpers import plugin_spec as migrate_plugin_spec from snowflake.cli._plugins.init import plugin_spec as init_plugin_spec @@ -52,6 +53,7 @@ def get_builtin_plugin_name_to_plugin_spec(): "init": init_plugin_spec, "workspace": workspace_plugin_spec, "plugin": plugin_plugin_spec, + "dbt": dbt_plugin_spec, "logs": logs_plugin_spec, } diff --git a/src/snowflake/cli/_plugins/dbt/__init__.py b/src/snowflake/cli/_plugins/dbt/__init__.py new file mode 100644 index 0000000000..e612998b27 --- /dev/null +++ b/src/snowflake/cli/_plugins/dbt/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2025 Snowflake Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/src/snowflake/cli/_plugins/dbt/commands.py b/src/snowflake/cli/_plugins/dbt/commands.py new file mode 100644 index 0000000000..ad05d2f958 --- /dev/null +++ b/src/snowflake/cli/_plugins/dbt/commands.py @@ -0,0 +1,41 @@ +# Copyright (c) 2025 Snowflake Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import annotations + +import logging + +from snowflake.cli._plugins.dbt.manager import DBTManager +from snowflake.cli.api.commands.snow_typer import SnowTyperFactory +from snowflake.cli.api.output.types import CommandResult, QueryResult + +app = SnowTyperFactory( + name="dbt", + help="Manages dbt on Snowflake projects", + is_hidden=lambda: True, +) +log = logging.getLogger(__name__) + + +@app.command( + "list", + requires_connection=True, +) +def list_dbts( + **options, +) -> CommandResult: + """ + List all dbt projects on Snowflake. + """ + return QueryResult(DBTManager().list()) diff --git a/src/snowflake/cli/_plugins/dbt/manager.py b/src/snowflake/cli/_plugins/dbt/manager.py new file mode 100644 index 0000000000..9a8619b31e --- /dev/null +++ b/src/snowflake/cli/_plugins/dbt/manager.py @@ -0,0 +1,24 @@ +# Copyright (c) 2025 Snowflake Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import annotations + +from snowflake.cli.api.sql_execution import SqlExecutionMixin +from snowflake.connector.cursor import SnowflakeCursor + + +class DBTManager(SqlExecutionMixin): + def list(self) -> SnowflakeCursor: # noqa: A003 + query = f"show dbt" + return self.execute_query(query) diff --git a/src/snowflake/cli/_plugins/dbt/plugin_spec.py b/src/snowflake/cli/_plugins/dbt/plugin_spec.py new file mode 100644 index 0000000000..59c15975df --- /dev/null +++ b/src/snowflake/cli/_plugins/dbt/plugin_spec.py @@ -0,0 +1,30 @@ +# Copyright (c) 2025 Snowflake Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from snowflake.cli._plugins.dbt import commands +from snowflake.cli.api.plugins.command import ( + SNOWCLI_ROOT_COMMAND_PATH, + CommandSpec, + CommandType, + plugin_hook_impl, +) + + +@plugin_hook_impl +def command_spec(): + return CommandSpec( + parent_command_path=SNOWCLI_ROOT_COMMAND_PATH, + command_type=CommandType.COMMAND_GROUP, + typer_instance=commands.app.create_instance(), + ) diff --git a/tests/dbt/__init__.py b/tests/dbt/__init__.py new file mode 100644 index 0000000000..e612998b27 --- /dev/null +++ b/tests/dbt/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2025 Snowflake Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/tests/dbt/test_dbt_commands.py b/tests/dbt/test_dbt_commands.py new file mode 100644 index 0000000000..3fe0ad8b16 --- /dev/null +++ b/tests/dbt/test_dbt_commands.py @@ -0,0 +1,34 @@ +# Copyright (c) 2025 Snowflake Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from unittest import mock + +import pytest + + +@pytest.fixture +def mock_connect(mock_ctx): + with mock.patch("snowflake.connector.connect") as _fixture: + ctx = mock_ctx() + _fixture.return_value = ctx + _fixture.mocked_ctx = _fixture.return_value + yield _fixture + + +def test_list_dbt_objects(mock_connect, runner): + + result = runner.invoke(["dbt", "list"]) + + assert result.exit_code == 0, result.output + assert mock_connect.mocked_ctx.get_query() == "show dbt" diff --git a/tests/test_help_messages.py b/tests/test_help_messages.py index f7f1ed2074..d18d46d184 100644 --- a/tests/test_help_messages.py +++ b/tests/test_help_messages.py @@ -34,7 +34,7 @@ def iter_through_all_commands(command_groups_only: bool = False): Generator iterating through all commands. Paths are yielded as List[str] """ - ignore_plugins = ["helpers", "cortex", "workspace"] + ignore_plugins = ["helpers", "cortex", "workspace", "dbt"] no_command: List[str] = [] yield no_command From 9b07ee186fd35c801b97402c8d3e10d739cdc605 Mon Sep 17 00:00:00 2001 From: Jakub Wilkowski Date: Fri, 14 Feb 2025 13:18:44 +0100 Subject: [PATCH 17/54] feat: [SNOW-1890085] add dbt execute command --- src/snowflake/cli/_plugins/dbt/commands.py | 26 ++++++++++++ src/snowflake/cli/_plugins/dbt/manager.py | 8 +++- tests/dbt/test_dbt_commands.py | 47 +++++++++++++++++++++- 3 files changed, 78 insertions(+), 3 deletions(-) diff --git a/src/snowflake/cli/_plugins/dbt/commands.py b/src/snowflake/cli/_plugins/dbt/commands.py index ad05d2f958..f4d0054ac4 100644 --- a/src/snowflake/cli/_plugins/dbt/commands.py +++ b/src/snowflake/cli/_plugins/dbt/commands.py @@ -16,6 +16,7 @@ import logging +import typer from snowflake.cli._plugins.dbt.manager import DBTManager from snowflake.cli.api.commands.snow_typer import SnowTyperFactory from snowflake.cli.api.output.types import CommandResult, QueryResult @@ -39,3 +40,28 @@ def list_dbts( List all dbt projects on Snowflake. """ return QueryResult(DBTManager().list()) + + +@app.command( + "execute", + requires_connection=True, + context_settings={"allow_extra_args": True, "ignore_unknown_options": True}, +) +def execute( + ctx: typer.Context, + dbt_command: str = typer.Argument( + help="dbt command to execute, i. e. run, compile, seed...", + ), + name: str = typer.Option( + default=..., + help="Name of the dbt object to execute command on.", + ), + **options, +) -> CommandResult: + """ + Execute command on dbt in Snowflake project. + """ + # ctx.args are parameters that were not captured as known cli params (those are in **options). + # as a consequence, we don't support passing params known to snowflake cli further to dbt + dbt_cli_args = ctx.args + return QueryResult(DBTManager().execute(dbt_command, name, *dbt_cli_args)) diff --git a/src/snowflake/cli/_plugins/dbt/manager.py b/src/snowflake/cli/_plugins/dbt/manager.py index 9a8619b31e..5d4a0ef58e 100644 --- a/src/snowflake/cli/_plugins/dbt/manager.py +++ b/src/snowflake/cli/_plugins/dbt/manager.py @@ -20,5 +20,11 @@ class DBTManager(SqlExecutionMixin): def list(self) -> SnowflakeCursor: # noqa: A003 - query = f"show dbt" + query = "SHOW DBT" + return self.execute_query(query) + + def execute(self, dbt_command: str, name: str, *dbt_cli_args): + query = f"EXECUTE DBT {name} {dbt_command}" + if dbt_cli_args: + query += " " + " ".join([arg for arg in dbt_cli_args]) return self.execute_query(query) diff --git a/tests/dbt/test_dbt_commands.py b/tests/dbt/test_dbt_commands.py index 3fe0ad8b16..760072c408 100644 --- a/tests/dbt/test_dbt_commands.py +++ b/tests/dbt/test_dbt_commands.py @@ -26,9 +26,52 @@ def mock_connect(mock_ctx): yield _fixture -def test_list_dbt_objects(mock_connect, runner): +def test_dbt_list(mock_connect, runner): result = runner.invoke(["dbt", "list"]) assert result.exit_code == 0, result.output - assert mock_connect.mocked_ctx.get_query() == "show dbt" + assert mock_connect.mocked_ctx.get_query() == "SHOW DBT" + + +@pytest.mark.parametrize( + "args,expected_query", + [ + ( + ["dbt", "execute", "compile", "--name=pipeline_name"], + "EXECUTE DBT pipeline_name compile", + ), + ( + [ + "dbt", + "execute", + "compile", + "--name=pipeline_name", + "-f", + "--select @source:snowplow,tag:nightly models/export", + ], + "EXECUTE DBT pipeline_name compile -f --select @source:snowplow,tag:nightly models/export", + ), + ( + ["dbt", "execute", "compile", "--name=pipeline_name", "--vars '{foo:bar}'"], + "EXECUTE DBT pipeline_name compile --vars '{foo:bar}'", + ), + ( + [ + "dbt", + "execute", + "compile", + "--name=pipeline_name", + "--format=JSON", + "--debug", + ], + "EXECUTE DBT pipeline_name compile", + ), + ], +) +def test_dbt_execute(mock_connect, runner, args, expected_query): + + result = runner.invoke(args) + + assert result.exit_code == 0, result.output + assert mock_connect.mocked_ctx.get_query() == expected_query From e07938f0f979be169221948fd27928f25462e1a8 Mon Sep 17 00:00:00 2001 From: Jakub Wilkowski Date: Tue, 18 Feb 2025 14:09:03 +0100 Subject: [PATCH 18/54] refactor: [SNOW-1890085] reimplement dbt execute as pass-through command group --- src/snowflake/cli/_plugins/dbt/commands.py | 63 ++++++++++++++------- src/snowflake/cli/_plugins/dbt/constants.py | 20 +++++++ tests/dbt/test_dbt_commands.py | 48 ++++++++++++---- 3 files changed, 97 insertions(+), 34 deletions(-) create mode 100644 src/snowflake/cli/_plugins/dbt/constants.py diff --git a/src/snowflake/cli/_plugins/dbt/commands.py b/src/snowflake/cli/_plugins/dbt/commands.py index f4d0054ac4..a28a070d2a 100644 --- a/src/snowflake/cli/_plugins/dbt/commands.py +++ b/src/snowflake/cli/_plugins/dbt/commands.py @@ -15,9 +15,12 @@ from __future__ import annotations import logging +from typing import Annotated import typer +from snowflake.cli._plugins.dbt.constants import DBT_COMMANDS from snowflake.cli._plugins.dbt.manager import DBTManager +from snowflake.cli.api.commands.decorators import global_options_with_connection from snowflake.cli.api.commands.snow_typer import SnowTyperFactory from snowflake.cli.api.output.types import CommandResult, QueryResult @@ -37,31 +40,47 @@ def list_dbts( **options, ) -> CommandResult: """ - List all dbt projects on Snowflake. + List all dbt on Snowflake projects. """ return QueryResult(DBTManager().list()) -@app.command( - "execute", - requires_connection=True, - context_settings={"allow_extra_args": True, "ignore_unknown_options": True}, +# `execute` is a pass through command group, meaning that all params after command should be passed over as they are, +# suppressing usual CLI behaviour for displaying help or formatting options. +dbt_execute_app = SnowTyperFactory( + name="execute", + help="Execute a dbt command", ) -def execute( - ctx: typer.Context, - dbt_command: str = typer.Argument( - help="dbt command to execute, i. e. run, compile, seed...", - ), - name: str = typer.Option( - default=..., - help="Name of the dbt object to execute command on.", - ), +app.add_typer(dbt_execute_app) + + +@dbt_execute_app.callback() +@global_options_with_connection +def before_callback( + name: Annotated[ + str, typer.Argument(help="Name of the dbt object to execute command on.") + ], **options, -) -> CommandResult: - """ - Execute command on dbt in Snowflake project. - """ - # ctx.args are parameters that were not captured as known cli params (those are in **options). - # as a consequence, we don't support passing params known to snowflake cli further to dbt - dbt_cli_args = ctx.args - return QueryResult(DBTManager().execute(dbt_command, name, *dbt_cli_args)) +): + """Handles global options passed before the command and takes pipeline name to be accessed through child context later""" + pass + + +for cmd in DBT_COMMANDS: + + @dbt_execute_app.command( + name=cmd, + requires_connection=False, + requires_global_options=False, + context_settings={"allow_extra_args": True, "ignore_unknown_options": True}, + help=f"Execute {cmd} command on dbt on Snowflake project.", + add_help_option=False, + ) + def _dbt_execute( + ctx: typer.Context, + ) -> CommandResult: + # TODO: figure out how to present logs to users + dbt_cli_args = ctx.args + dbt_command = ctx.command.name + name = ctx.parent.params["name"] + return QueryResult(DBTManager().execute(dbt_command, name, *dbt_cli_args)) diff --git a/src/snowflake/cli/_plugins/dbt/constants.py b/src/snowflake/cli/_plugins/dbt/constants.py new file mode 100644 index 0000000000..4085d7ad0d --- /dev/null +++ b/src/snowflake/cli/_plugins/dbt/constants.py @@ -0,0 +1,20 @@ +DBT_COMMANDS = [ + "build", + "clean", + "clone", + "compile", + "debug", + "deps", + "docs", + "init", + "list", + "parse", + "retry", + "run", + "run", + "seed", + "show", + "snapshot", + "source", + "test", +] diff --git a/tests/dbt/test_dbt_commands.py b/tests/dbt/test_dbt_commands.py index 760072c408..804ceec931 100644 --- a/tests/dbt/test_dbt_commands.py +++ b/tests/dbt/test_dbt_commands.py @@ -37,35 +37,59 @@ def test_dbt_list(mock_connect, runner): @pytest.mark.parametrize( "args,expected_query", [ - ( - ["dbt", "execute", "compile", "--name=pipeline_name"], - "EXECUTE DBT pipeline_name compile", + pytest.param( + [ + "dbt", + "execute", + "pipeline_name", + "test", + ], + "EXECUTE DBT pipeline_name test", + id="simple-command", ), - ( + pytest.param( [ "dbt", "execute", - "compile", - "--name=pipeline_name", + "pipeline_name", + "run", "-f", "--select @source:snowplow,tag:nightly models/export", ], - "EXECUTE DBT pipeline_name compile -f --select @source:snowplow,tag:nightly models/export", + "EXECUTE DBT pipeline_name run -f --select @source:snowplow,tag:nightly models/export", + id="with-dbt-options", ), - ( - ["dbt", "execute", "compile", "--name=pipeline_name", "--vars '{foo:bar}'"], + pytest.param( + ["dbt", "execute", "pipeline_name", "compile", "--vars '{foo:bar}'"], "EXECUTE DBT pipeline_name compile --vars '{foo:bar}'", + id="with-dbt-vars", ), - ( + pytest.param( [ "dbt", "execute", + "pipeline_name", "compile", - "--name=pipeline_name", - "--format=JSON", + "--format=TXT", # collision with CLI's option; unsupported option + "-v", # collision with CLI's option + "-h", "--debug", + "--info", + "--config-file=/", + ], + "EXECUTE DBT pipeline_name compile --format=TXT -v -h --debug --info --config-file=/", + id="with-dbt-conflicting-options", + ), + pytest.param( + [ + "dbt", + "execute", + "--format=JSON", + "pipeline_name", + "compile", ], "EXECUTE DBT pipeline_name compile", + id="with-cli-flag", ), ], ) From 8d3a27e3cce0de1a2fab0c2b2107638e602a4155 Mon Sep 17 00:00:00 2001 From: Jakub Wilkowski Date: Thu, 20 Feb 2025 13:06:20 +0100 Subject: [PATCH 19/54] feat: [SNOW-1890085] implement dbt deploy command --- src/snowflake/cli/_plugins/dbt/commands.py | 40 ++++- src/snowflake/cli/_plugins/dbt/manager.py | 25 +++ tests/dbt/test_dbt_commands.py | 200 +++++++++++++-------- 3 files changed, 188 insertions(+), 77 deletions(-) diff --git a/src/snowflake/cli/_plugins/dbt/commands.py b/src/snowflake/cli/_plugins/dbt/commands.py index a28a070d2a..072b82bd54 100644 --- a/src/snowflake/cli/_plugins/dbt/commands.py +++ b/src/snowflake/cli/_plugins/dbt/commands.py @@ -15,13 +15,16 @@ from __future__ import annotations import logging -from typing import Annotated +from pathlib import Path +from typing import Optional import typer from snowflake.cli._plugins.dbt.constants import DBT_COMMANDS from snowflake.cli._plugins.dbt.manager import DBTManager from snowflake.cli.api.commands.decorators import global_options_with_connection +from snowflake.cli.api.commands.flags import identifier_argument from snowflake.cli.api.commands.snow_typer import SnowTyperFactory +from snowflake.cli.api.identifiers import FQN from snowflake.cli.api.output.types import CommandResult, QueryResult app = SnowTyperFactory( @@ -32,6 +35,9 @@ log = logging.getLogger(__name__) +DBTNameArgument = identifier_argument(sf_object="DBT Object", example="my_pipeline") + + @app.command( "list", requires_connection=True, @@ -45,6 +51,34 @@ def list_dbts( return QueryResult(DBTManager().list()) +@app.command( + "deploy", + requires_connection=True, +) +def deploy_dbt( + name: FQN = DBTNameArgument, + source: Optional[str] = typer.Option( + help="Path to directory containing dbt files to deploy. Defaults to current working directory.", + show_default=False, + default=None, + ), + force: Optional[bool] = typer.Option( + False, + help="Overwrites conflicting files in the project, if any.", + ), + **options, +) -> CommandResult: + """ + Copy dbt files and create or update dbt on Snowflake project. + """ + # TODO: options for DBT version? + if source is None: + path = Path.cwd() + else: + path = Path(source) + return QueryResult(DBTManager().deploy(path.resolve(), name, force=force)) + + # `execute` is a pass through command group, meaning that all params after command should be passed over as they are, # suppressing usual CLI behaviour for displaying help or formatting options. dbt_execute_app = SnowTyperFactory( @@ -57,9 +91,7 @@ def list_dbts( @dbt_execute_app.callback() @global_options_with_connection def before_callback( - name: Annotated[ - str, typer.Argument(help="Name of the dbt object to execute command on.") - ], + name: FQN = DBTNameArgument, **options, ): """Handles global options passed before the command and takes pipeline name to be accessed through child context later""" diff --git a/src/snowflake/cli/_plugins/dbt/manager.py b/src/snowflake/cli/_plugins/dbt/manager.py index 5d4a0ef58e..b2b73b5306 100644 --- a/src/snowflake/cli/_plugins/dbt/manager.py +++ b/src/snowflake/cli/_plugins/dbt/manager.py @@ -14,6 +14,12 @@ from __future__ import annotations +from pathlib import Path + +from click import ClickException +from snowflake.cli._plugins.stage.manager import StageManager +from snowflake.cli.api.console import cli_console +from snowflake.cli.api.identifiers import FQN from snowflake.cli.api.sql_execution import SqlExecutionMixin from snowflake.connector.cursor import SnowflakeCursor @@ -23,6 +29,25 @@ def list(self) -> SnowflakeCursor: # noqa: A003 query = "SHOW DBT" return self.execute_query(query) + def deploy(self, path: Path, name: FQN, force: bool) -> SnowflakeCursor: + # TODO: what to do with force? + if not path.joinpath("dbt_project.yml").exists(): + raise ClickException(f"dbt_project.yml does not exist in provided path.") + + with cli_console.phase("Creating temporary stage"): + stage_manager = StageManager() + stage_fqn = FQN.from_string(f"dbt_{name}_stage").using_context() + stage_name = stage_manager.get_standard_stage_prefix(stage_fqn) + stage_manager.create(stage_fqn, temporary=True) + + with cli_console.phase("Copying project files to stage"): + results = list(stage_manager.put_recursive(path, stage_name)) + cli_console.step(f"Copied {len(results)} files") + + with cli_console.phase("Creating DBT project"): + query = f"CREATE OR REPLACE DBT {name} FROM {stage_name}" + return self.execute_query(query) + def execute(self, dbt_command: str, name: str, *dbt_cli_args): query = f"EXECUTE DBT {name} {dbt_command}" if dbt_cli_args: diff --git a/tests/dbt/test_dbt_commands.py b/tests/dbt/test_dbt_commands.py index 804ceec931..3609f10ee6 100644 --- a/tests/dbt/test_dbt_commands.py +++ b/tests/dbt/test_dbt_commands.py @@ -12,9 +12,12 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import annotations + from unittest import mock import pytest +from snowflake.cli.api.identifiers import FQN @pytest.fixture @@ -26,76 +29,127 @@ def mock_connect(mock_ctx): yield _fixture -def test_dbt_list(mock_connect, runner): - - result = runner.invoke(["dbt", "list"]) - - assert result.exit_code == 0, result.output - assert mock_connect.mocked_ctx.get_query() == "SHOW DBT" - - -@pytest.mark.parametrize( - "args,expected_query", - [ - pytest.param( - [ - "dbt", - "execute", - "pipeline_name", - "test", - ], - "EXECUTE DBT pipeline_name test", - id="simple-command", - ), - pytest.param( - [ - "dbt", - "execute", - "pipeline_name", - "run", - "-f", - "--select @source:snowplow,tag:nightly models/export", - ], - "EXECUTE DBT pipeline_name run -f --select @source:snowplow,tag:nightly models/export", - id="with-dbt-options", - ), - pytest.param( - ["dbt", "execute", "pipeline_name", "compile", "--vars '{foo:bar}'"], - "EXECUTE DBT pipeline_name compile --vars '{foo:bar}'", - id="with-dbt-vars", - ), - pytest.param( - [ - "dbt", - "execute", - "pipeline_name", - "compile", - "--format=TXT", # collision with CLI's option; unsupported option - "-v", # collision with CLI's option - "-h", - "--debug", - "--info", - "--config-file=/", - ], - "EXECUTE DBT pipeline_name compile --format=TXT -v -h --debug --info --config-file=/", - id="with-dbt-conflicting-options", - ), - pytest.param( - [ - "dbt", - "execute", - "--format=JSON", - "pipeline_name", - "compile", - ], - "EXECUTE DBT pipeline_name compile", - id="with-cli-flag", - ), - ], -) -def test_dbt_execute(mock_connect, runner, args, expected_query): - - result = runner.invoke(args) - - assert result.exit_code == 0, result.output - assert mock_connect.mocked_ctx.get_query() == expected_query +class TestDBTList: + def test_dbt_list(self, mock_connect, runner): + + result = runner.invoke(["dbt", "list"]) + + assert result.exit_code == 0, result.output + assert mock_connect.mocked_ctx.get_query() == "SHOW DBT" + + +class TestDBTDeploy: + @pytest.fixture + def dbt_project_path(self, tmp_path_factory): + source_path = tmp_path_factory.mktemp("dbt_project") + dbt_file = source_path / "dbt_project.yml" + dbt_file.touch() + yield source_path + + @pytest.fixture + def mock_cli_console(self): + with mock.patch("snowflake.cli.api.console") as _fixture: + yield _fixture + + @mock.patch("snowflake.cli._plugins.dbt.manager.StageManager.put_recursive") + @mock.patch("snowflake.cli._plugins.dbt.manager.StageManager.create") + def test_deploys_project_from_source( + self, mock_create, mock_put_recursive, mock_connect, runner, dbt_project_path + ): + + result = runner.invoke( + ["dbt", "deploy", "TEST_PIPELINE", f"--source={dbt_project_path}"] + ) + + assert result.exit_code == 0, result.output + assert ( + mock_connect.mocked_ctx.get_query() + == "CREATE OR REPLACE DBT TEST_PIPELINE FROM @MockDatabase.MockSchema.dbt_TEST_PIPELINE_stage" + ) + stage_fqn = FQN.from_string(f"dbt_TEST_PIPELINE_stage").using_context() + mock_create.assert_called_once_with(stage_fqn, temporary=True) + mock_put_recursive.assert_called_once_with( + dbt_project_path, "@MockDatabase.MockSchema.dbt_TEST_PIPELINE_stage" + ) + + def test_raises_when_dbt_project_is_not_available( + self, dbt_project_path, mock_connect, runner + ): + dbt_file = dbt_project_path / "dbt_project.yml" + dbt_file.unlink() + + result = runner.invoke( + ["dbt", "deploy", "TEST_PIPELINE", f"--source={dbt_project_path}"] + ) + + assert result.exit_code == 1, result.output + assert "dbt_project.yml does not exist in provided path." in result.output + assert mock_connect.mocked_ctx.get_query() == "" + + +class TestDBTExecute: + @pytest.mark.parametrize( + "args,expected_query", + [ + pytest.param( + [ + "dbt", + "execute", + "pipeline_name", + "test", + ], + "EXECUTE DBT pipeline_name test", + id="simple-command", + ), + pytest.param( + [ + "dbt", + "execute", + "pipeline_name", + "run", + "-f", + "--select @source:snowplow,tag:nightly models/export", + ], + "EXECUTE DBT pipeline_name run -f --select @source:snowplow,tag:nightly models/export", + id="with-dbt-options", + ), + pytest.param( + ["dbt", "execute", "pipeline_name", "compile", "--vars '{foo:bar}'"], + "EXECUTE DBT pipeline_name compile --vars '{foo:bar}'", + id="with-dbt-vars", + ), + pytest.param( + [ + "dbt", + "execute", + "pipeline_name", + "compile", + "--format=TXT", # collision with CLI's option; unsupported option + "-v", # collision with CLI's option + "-h", + "--debug", + "--info", + "--config-file=/", + ], + "EXECUTE DBT pipeline_name compile --format=TXT -v -h --debug --info --config-file=/", + id="with-dbt-conflicting-options", + ), + pytest.param( + [ + "dbt", + "execute", + "--format=JSON", + "pipeline_name", + "compile", + ], + "EXECUTE DBT pipeline_name compile", + id="with-cli-flag", + ), + ], + ) + def test_dbt_execute(self, mock_connect, runner, args, expected_query): + + result = runner.invoke(args) + + assert result.exit_code == 0, result.output + assert mock_connect.mocked_ctx.get_query() == expected_query From 6a452748ec2c5fff4e763e3c05e571ffb336fa88 Mon Sep 17 00:00:00 2001 From: Jakub Wilkowski Date: Fri, 21 Feb 2025 12:17:11 +0100 Subject: [PATCH 20/54] feat: [SNOW-1890085] implement StdoutExecutionMixin for demo purposes --- src/snowflake/cli/_plugins/dbt/manager.py | 15 ++++++++++++++- src/snowflake/cli/api/feature_flags.py | 1 + 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/src/snowflake/cli/_plugins/dbt/manager.py b/src/snowflake/cli/_plugins/dbt/manager.py index b2b73b5306..e45e624155 100644 --- a/src/snowflake/cli/_plugins/dbt/manager.py +++ b/src/snowflake/cli/_plugins/dbt/manager.py @@ -19,12 +19,25 @@ from click import ClickException from snowflake.cli._plugins.stage.manager import StageManager from snowflake.cli.api.console import cli_console +from snowflake.cli.api.feature_flags import FeatureFlag from snowflake.cli.api.identifiers import FQN from snowflake.cli.api.sql_execution import SqlExecutionMixin from snowflake.connector.cursor import SnowflakeCursor -class DBTManager(SqlExecutionMixin): +class StdoutExecutionMixin(SqlExecutionMixin): + def execute_query(self, query, **kwargs): + if FeatureFlag.ENABLE_DBT_POC.is_enabled(): + from unittest.mock import MagicMock + + cli_console.message(f"Sending query: {query}") + mock_cursor = MagicMock() + mock_cursor.description = [] + return mock_cursor + return super().execute_query(query, **kwargs) + + +class DBTManager(StdoutExecutionMixin): def list(self) -> SnowflakeCursor: # noqa: A003 query = "SHOW DBT" return self.execute_query(query) diff --git a/src/snowflake/cli/api/feature_flags.py b/src/snowflake/cli/api/feature_flags.py index df63155ceb..adbc3846d3 100644 --- a/src/snowflake/cli/api/feature_flags.py +++ b/src/snowflake/cli/api/feature_flags.py @@ -69,3 +69,4 @@ class FeatureFlag(FeatureFlagMixin): ENABLE_SNOWPARK_GLOB_SUPPORT = BooleanFlag("ENABLE_SNOWPARK_GLOB_SUPPORT", False) ENABLE_SPCS_SERVICE_EVENTS = BooleanFlag("ENABLE_SPCS_SERVICE_EVENTS", False) ENABLE_AUTH_KEYPAIR = BooleanFlag("ENABLE_AUTH_KEYPAIR", False) + ENABLE_DBT_POC = BooleanFlag("ENABLE_DBT_POC", False) From 4f83a281c5ef7172d4ebe4fa55526b7a38907291 Mon Sep 17 00:00:00 2001 From: Jakub Wilkowski Date: Tue, 25 Feb 2025 10:34:18 +0100 Subject: [PATCH 21/54] fix: [SNOW-1890085] rename sql object DBT -> DBT PROJECT --- src/snowflake/cli/_plugins/dbt/manager.py | 6 +++--- tests/dbt/test_dbt_commands.py | 14 +++++++------- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/src/snowflake/cli/_plugins/dbt/manager.py b/src/snowflake/cli/_plugins/dbt/manager.py index e45e624155..74b157db03 100644 --- a/src/snowflake/cli/_plugins/dbt/manager.py +++ b/src/snowflake/cli/_plugins/dbt/manager.py @@ -39,7 +39,7 @@ def execute_query(self, query, **kwargs): class DBTManager(StdoutExecutionMixin): def list(self) -> SnowflakeCursor: # noqa: A003 - query = "SHOW DBT" + query = "SHOW DBT PROJECT" return self.execute_query(query) def deploy(self, path: Path, name: FQN, force: bool) -> SnowflakeCursor: @@ -58,11 +58,11 @@ def deploy(self, path: Path, name: FQN, force: bool) -> SnowflakeCursor: cli_console.step(f"Copied {len(results)} files") with cli_console.phase("Creating DBT project"): - query = f"CREATE OR REPLACE DBT {name} FROM {stage_name}" + query = f"CREATE OR REPLACE DBT PROJECT {name} FROM {stage_name}" return self.execute_query(query) def execute(self, dbt_command: str, name: str, *dbt_cli_args): - query = f"EXECUTE DBT {name} {dbt_command}" + query = f"EXECUTE DBT PROJECT {name} {dbt_command}" if dbt_cli_args: query += " " + " ".join([arg for arg in dbt_cli_args]) return self.execute_query(query) diff --git a/tests/dbt/test_dbt_commands.py b/tests/dbt/test_dbt_commands.py index 3609f10ee6..50a2fced1b 100644 --- a/tests/dbt/test_dbt_commands.py +++ b/tests/dbt/test_dbt_commands.py @@ -35,7 +35,7 @@ def test_dbt_list(self, mock_connect, runner): result = runner.invoke(["dbt", "list"]) assert result.exit_code == 0, result.output - assert mock_connect.mocked_ctx.get_query() == "SHOW DBT" + assert mock_connect.mocked_ctx.get_query() == "SHOW DBT PROJECT" class TestDBTDeploy: @@ -64,7 +64,7 @@ def test_deploys_project_from_source( assert result.exit_code == 0, result.output assert ( mock_connect.mocked_ctx.get_query() - == "CREATE OR REPLACE DBT TEST_PIPELINE FROM @MockDatabase.MockSchema.dbt_TEST_PIPELINE_stage" + == "CREATE OR REPLACE DBT PROJECT TEST_PIPELINE FROM @MockDatabase.MockSchema.dbt_TEST_PIPELINE_stage" ) stage_fqn = FQN.from_string(f"dbt_TEST_PIPELINE_stage").using_context() mock_create.assert_called_once_with(stage_fqn, temporary=True) @@ -98,7 +98,7 @@ class TestDBTExecute: "pipeline_name", "test", ], - "EXECUTE DBT pipeline_name test", + "EXECUTE DBT PROJECT pipeline_name test", id="simple-command", ), pytest.param( @@ -110,12 +110,12 @@ class TestDBTExecute: "-f", "--select @source:snowplow,tag:nightly models/export", ], - "EXECUTE DBT pipeline_name run -f --select @source:snowplow,tag:nightly models/export", + "EXECUTE DBT PROJECT pipeline_name run -f --select @source:snowplow,tag:nightly models/export", id="with-dbt-options", ), pytest.param( ["dbt", "execute", "pipeline_name", "compile", "--vars '{foo:bar}'"], - "EXECUTE DBT pipeline_name compile --vars '{foo:bar}'", + "EXECUTE DBT PROJECT pipeline_name compile --vars '{foo:bar}'", id="with-dbt-vars", ), pytest.param( @@ -131,7 +131,7 @@ class TestDBTExecute: "--info", "--config-file=/", ], - "EXECUTE DBT pipeline_name compile --format=TXT -v -h --debug --info --config-file=/", + "EXECUTE DBT PROJECT pipeline_name compile --format=TXT -v -h --debug --info --config-file=/", id="with-dbt-conflicting-options", ), pytest.param( @@ -142,7 +142,7 @@ class TestDBTExecute: "pipeline_name", "compile", ], - "EXECUTE DBT pipeline_name compile", + "EXECUTE DBT PROJECT pipeline_name compile", id="with-cli-flag", ), ], From 500bac55a47e9ad0e8b18de0eb4c09cb009b9bc6 Mon Sep 17 00:00:00 2001 From: Jakub Wilkowski Date: Tue, 25 Feb 2025 12:00:17 +0100 Subject: [PATCH 22/54] feat: [SNOW-1890085] dbt deploy: add support for dbt-version flag --- src/snowflake/cli/_plugins/dbt/commands.py | 8 +++- src/snowflake/cli/_plugins/dbt/manager.py | 18 ++++++++- tests/dbt/test_dbt_commands.py | 44 +++++++++++++++++++++- 3 files changed, 66 insertions(+), 4 deletions(-) diff --git a/src/snowflake/cli/_plugins/dbt/commands.py b/src/snowflake/cli/_plugins/dbt/commands.py index 072b82bd54..30bd54c169 100644 --- a/src/snowflake/cli/_plugins/dbt/commands.py +++ b/src/snowflake/cli/_plugins/dbt/commands.py @@ -66,6 +66,10 @@ def deploy_dbt( False, help="Overwrites conflicting files in the project, if any.", ), + dbt_version: Optional[str] = typer.Option( + None, + help="Version of dbt tool to be used. Taken from dbt_project.yml if not provided.", + ), **options, ) -> CommandResult: """ @@ -76,7 +80,9 @@ def deploy_dbt( path = Path.cwd() else: path = Path(source) - return QueryResult(DBTManager().deploy(path.resolve(), name, force=force)) + return QueryResult( + DBTManager().deploy(path.resolve(), name, dbt_version, force=force) + ) # `execute` is a pass through command group, meaning that all params after command should be passed over as they are, diff --git a/src/snowflake/cli/_plugins/dbt/manager.py b/src/snowflake/cli/_plugins/dbt/manager.py index 74b157db03..a10e4784b2 100644 --- a/src/snowflake/cli/_plugins/dbt/manager.py +++ b/src/snowflake/cli/_plugins/dbt/manager.py @@ -15,7 +15,9 @@ from __future__ import annotations from pathlib import Path +from typing import Optional +import yaml from click import ClickException from snowflake.cli._plugins.stage.manager import StageManager from snowflake.cli.api.console import cli_console @@ -42,11 +44,23 @@ def list(self) -> SnowflakeCursor: # noqa: A003 query = "SHOW DBT PROJECT" return self.execute_query(query) - def deploy(self, path: Path, name: FQN, force: bool) -> SnowflakeCursor: + def deploy( + self, path: Path, name: FQN, dbt_version: Optional[str], force: bool + ) -> SnowflakeCursor: # TODO: what to do with force? if not path.joinpath("dbt_project.yml").exists(): raise ClickException(f"dbt_project.yml does not exist in provided path.") + if dbt_version is None: + with path.joinpath("dbt_project.yml").open() as fd: + dbt_project_config = yaml.safe_load(fd) + try: + dbt_version = dbt_project_config["version"] + except (KeyError, TypeError): + raise ClickException( + f"dbt-version was not provided and is not available in dbt_project.yml" + ) + with cli_console.phase("Creating temporary stage"): stage_manager = StageManager() stage_fqn = FQN.from_string(f"dbt_{name}_stage").using_context() @@ -58,7 +72,7 @@ def deploy(self, path: Path, name: FQN, force: bool) -> SnowflakeCursor: cli_console.step(f"Copied {len(results)} files") with cli_console.phase("Creating DBT project"): - query = f"CREATE OR REPLACE DBT PROJECT {name} FROM {stage_name}" + query = f"CREATE OR REPLACE DBT PROJECT {name} FROM {stage_name} DBT_VERSION='{dbt_version}'" return self.execute_query(query) def execute(self, dbt_command: str, name: str, *dbt_cli_args): diff --git a/tests/dbt/test_dbt_commands.py b/tests/dbt/test_dbt_commands.py index 50a2fced1b..72a2efebbe 100644 --- a/tests/dbt/test_dbt_commands.py +++ b/tests/dbt/test_dbt_commands.py @@ -17,6 +17,7 @@ from unittest import mock import pytest +import yaml from snowflake.cli.api.identifiers import FQN @@ -44,6 +45,8 @@ def dbt_project_path(self, tmp_path_factory): source_path = tmp_path_factory.mktemp("dbt_project") dbt_file = source_path / "dbt_project.yml" dbt_file.touch() + with dbt_file.open(mode="w") as fd: + yaml.dump({"version": "1.2.3"}, fd) yield source_path @pytest.fixture @@ -64,7 +67,7 @@ def test_deploys_project_from_source( assert result.exit_code == 0, result.output assert ( mock_connect.mocked_ctx.get_query() - == "CREATE OR REPLACE DBT PROJECT TEST_PIPELINE FROM @MockDatabase.MockSchema.dbt_TEST_PIPELINE_stage" + == "CREATE OR REPLACE DBT PROJECT TEST_PIPELINE FROM @MockDatabase.MockSchema.dbt_TEST_PIPELINE_stage DBT_VERSION='1.2.3'" ) stage_fqn = FQN.from_string(f"dbt_TEST_PIPELINE_stage").using_context() mock_create.assert_called_once_with(stage_fqn, temporary=True) @@ -72,6 +75,27 @@ def test_deploys_project_from_source( dbt_project_path, "@MockDatabase.MockSchema.dbt_TEST_PIPELINE_stage" ) + @mock.patch("snowflake.cli._plugins.dbt.manager.StageManager.put_recursive") + @mock.patch("snowflake.cli._plugins.dbt.manager.StageManager.create") + def test_dbt_version_from_option_has_precedence_over_file( + self, _mock_create, _mock_put_recursive, mock_connect, runner, dbt_project_path + ): + result = runner.invoke( + [ + "dbt", + "deploy", + "TEST_PIPELINE", + f"--source={dbt_project_path}", + "--dbt-version=2.3.4", + ] + ) + + assert result.exit_code == 0, result.output + assert ( + mock_connect.mocked_ctx.get_query() + == "CREATE OR REPLACE DBT PROJECT TEST_PIPELINE FROM @MockDatabase.MockSchema.dbt_TEST_PIPELINE_stage DBT_VERSION='2.3.4'" + ) + def test_raises_when_dbt_project_is_not_available( self, dbt_project_path, mock_connect, runner ): @@ -86,6 +110,24 @@ def test_raises_when_dbt_project_is_not_available( assert "dbt_project.yml does not exist in provided path." in result.output assert mock_connect.mocked_ctx.get_query() == "" + def test_raises_when_dbt_project_version_is_not_specified( + self, dbt_project_path, mock_connect, runner + ): + dbt_file = dbt_project_path / "dbt_project.yml" + with dbt_file.open(mode="w") as fd: + yaml.dump({}, fd) + + result = runner.invoke( + ["dbt", "deploy", "TEST_PIPELINE", f"--source={dbt_project_path}"] + ) + + assert result.exit_code == 1, result.output + assert ( + "dbt-version was not provided and is not available in dbt_project.yml" + in result.output + ) + assert mock_connect.mocked_ctx.get_query() == "" + class TestDBTExecute: @pytest.mark.parametrize( From fdd33e8fe71f143b03745ff0ac6bc4cce5f04fc8 Mon Sep 17 00:00:00 2001 From: Jakub Wilkowski Date: Tue, 25 Feb 2025 12:55:02 +0100 Subject: [PATCH 23/54] feat: [SNOW-1890085] dbt deploy: add support for dbt-adapter-version flag --- src/snowflake/cli/_plugins/dbt/commands.py | 7 +++++- src/snowflake/cli/_plugins/dbt/manager.py | 9 +++++-- tests/dbt/test_dbt_commands.py | 29 ++++++++++++++++++---- 3 files changed, 37 insertions(+), 8 deletions(-) diff --git a/src/snowflake/cli/_plugins/dbt/commands.py b/src/snowflake/cli/_plugins/dbt/commands.py index 30bd54c169..13047b3002 100644 --- a/src/snowflake/cli/_plugins/dbt/commands.py +++ b/src/snowflake/cli/_plugins/dbt/commands.py @@ -70,6 +70,9 @@ def deploy_dbt( None, help="Version of dbt tool to be used. Taken from dbt_project.yml if not provided.", ), + dbt_adapter_version: str = typer.Option( + help="dbt-snowflake adapter version to be used", + ), **options, ) -> CommandResult: """ @@ -81,7 +84,9 @@ def deploy_dbt( else: path = Path(source) return QueryResult( - DBTManager().deploy(path.resolve(), name, dbt_version, force=force) + DBTManager().deploy( + path.resolve(), name, dbt_version, dbt_adapter_version, force=force + ) ) diff --git a/src/snowflake/cli/_plugins/dbt/manager.py b/src/snowflake/cli/_plugins/dbt/manager.py index a10e4784b2..f5020abbcd 100644 --- a/src/snowflake/cli/_plugins/dbt/manager.py +++ b/src/snowflake/cli/_plugins/dbt/manager.py @@ -45,7 +45,12 @@ def list(self) -> SnowflakeCursor: # noqa: A003 return self.execute_query(query) def deploy( - self, path: Path, name: FQN, dbt_version: Optional[str], force: bool + self, + path: Path, + name: FQN, + dbt_version: Optional[str], + dbt_adapter_version: str, + force: bool, ) -> SnowflakeCursor: # TODO: what to do with force? if not path.joinpath("dbt_project.yml").exists(): @@ -72,7 +77,7 @@ def deploy( cli_console.step(f"Copied {len(results)} files") with cli_console.phase("Creating DBT project"): - query = f"CREATE OR REPLACE DBT PROJECT {name} FROM {stage_name} DBT_VERSION='{dbt_version}'" + query = f"CREATE OR REPLACE DBT PROJECT {name} FROM {stage_name} DBT_VERSION='{dbt_version}' DBT_ADAPTER_VERSION='{dbt_adapter_version}'" return self.execute_query(query) def execute(self, dbt_command: str, name: str, *dbt_cli_args): diff --git a/tests/dbt/test_dbt_commands.py b/tests/dbt/test_dbt_commands.py index 72a2efebbe..5a23215f8a 100644 --- a/tests/dbt/test_dbt_commands.py +++ b/tests/dbt/test_dbt_commands.py @@ -61,13 +61,19 @@ def test_deploys_project_from_source( ): result = runner.invoke( - ["dbt", "deploy", "TEST_PIPELINE", f"--source={dbt_project_path}"] + [ + "dbt", + "deploy", + "TEST_PIPELINE", + f"--source={dbt_project_path}", + "--dbt-adapter-version=3.4.5", + ] ) assert result.exit_code == 0, result.output assert ( mock_connect.mocked_ctx.get_query() - == "CREATE OR REPLACE DBT PROJECT TEST_PIPELINE FROM @MockDatabase.MockSchema.dbt_TEST_PIPELINE_stage DBT_VERSION='1.2.3'" + == "CREATE OR REPLACE DBT PROJECT TEST_PIPELINE FROM @MockDatabase.MockSchema.dbt_TEST_PIPELINE_stage DBT_VERSION='1.2.3' DBT_ADAPTER_VERSION='3.4.5'" ) stage_fqn = FQN.from_string(f"dbt_TEST_PIPELINE_stage").using_context() mock_create.assert_called_once_with(stage_fqn, temporary=True) @@ -87,13 +93,14 @@ def test_dbt_version_from_option_has_precedence_over_file( "TEST_PIPELINE", f"--source={dbt_project_path}", "--dbt-version=2.3.4", + "--dbt-adapter-version=3.4.5", ] ) assert result.exit_code == 0, result.output assert ( mock_connect.mocked_ctx.get_query() - == "CREATE OR REPLACE DBT PROJECT TEST_PIPELINE FROM @MockDatabase.MockSchema.dbt_TEST_PIPELINE_stage DBT_VERSION='2.3.4'" + == "CREATE OR REPLACE DBT PROJECT TEST_PIPELINE FROM @MockDatabase.MockSchema.dbt_TEST_PIPELINE_stage DBT_VERSION='2.3.4' DBT_ADAPTER_VERSION='3.4.5'" ) def test_raises_when_dbt_project_is_not_available( @@ -103,7 +110,13 @@ def test_raises_when_dbt_project_is_not_available( dbt_file.unlink() result = runner.invoke( - ["dbt", "deploy", "TEST_PIPELINE", f"--source={dbt_project_path}"] + [ + "dbt", + "deploy", + "TEST_PIPELINE", + f"--source={dbt_project_path}", + "--dbt-adapter-version=3.4.5", + ], ) assert result.exit_code == 1, result.output @@ -118,7 +131,13 @@ def test_raises_when_dbt_project_version_is_not_specified( yaml.dump({}, fd) result = runner.invoke( - ["dbt", "deploy", "TEST_PIPELINE", f"--source={dbt_project_path}"] + [ + "dbt", + "deploy", + "TEST_PIPELINE", + f"--source={dbt_project_path}", + "--dbt-adapter-version=3.4.5", + ] ) assert result.exit_code == 1, result.output From 560822bd474acc5e4a3f79ca2a120440a80d3f67 Mon Sep 17 00:00:00 2001 From: Jakub Wilkowski Date: Tue, 25 Feb 2025 14:17:07 +0100 Subject: [PATCH 24/54] feat: [SNOW-1890085] dbt deploy: provide main file param --- src/snowflake/cli/_plugins/dbt/manager.py | 14 +++++++++++--- tests/dbt/test_dbt_commands.py | 8 ++++++-- 2 files changed, 17 insertions(+), 5 deletions(-) diff --git a/src/snowflake/cli/_plugins/dbt/manager.py b/src/snowflake/cli/_plugins/dbt/manager.py index f5020abbcd..fb3e1ad9c7 100644 --- a/src/snowflake/cli/_plugins/dbt/manager.py +++ b/src/snowflake/cli/_plugins/dbt/manager.py @@ -53,11 +53,12 @@ def deploy( force: bool, ) -> SnowflakeCursor: # TODO: what to do with force? - if not path.joinpath("dbt_project.yml").exists(): + dbt_project_path = path.joinpath("dbt_project.yml") + if not dbt_project_path.exists(): raise ClickException(f"dbt_project.yml does not exist in provided path.") if dbt_version is None: - with path.joinpath("dbt_project.yml").open() as fd: + with dbt_project_path.open() as fd: dbt_project_config = yaml.safe_load(fd) try: dbt_version = dbt_project_config["version"] @@ -77,7 +78,10 @@ def deploy( cli_console.step(f"Copied {len(results)} files") with cli_console.phase("Creating DBT project"): - query = f"CREATE OR REPLACE DBT PROJECT {name} FROM {stage_name} DBT_VERSION='{dbt_version}' DBT_ADAPTER_VERSION='{dbt_adapter_version}'" + staged_dbt_project_path = self._get_dbt_project_stage_path(stage_name) + query = f"""CREATE OR REPLACE DBT PROJECT {name} +FROM {stage_name} MAIN_FILE='{staged_dbt_project_path}' +DBT_VERSION='{dbt_version}' DBT_ADAPTER_VERSION='{dbt_adapter_version}'""" return self.execute_query(query) def execute(self, dbt_command: str, name: str, *dbt_cli_args): @@ -85,3 +89,7 @@ def execute(self, dbt_command: str, name: str, *dbt_cli_args): if dbt_cli_args: query += " " + " ".join([arg for arg in dbt_cli_args]) return self.execute_query(query) + + @staticmethod + def _get_dbt_project_stage_path(stage_name): + return "/".join([stage_name, "dbt_project.yml"]) diff --git a/tests/dbt/test_dbt_commands.py b/tests/dbt/test_dbt_commands.py index 5a23215f8a..f8cac2fda9 100644 --- a/tests/dbt/test_dbt_commands.py +++ b/tests/dbt/test_dbt_commands.py @@ -73,7 +73,9 @@ def test_deploys_project_from_source( assert result.exit_code == 0, result.output assert ( mock_connect.mocked_ctx.get_query() - == "CREATE OR REPLACE DBT PROJECT TEST_PIPELINE FROM @MockDatabase.MockSchema.dbt_TEST_PIPELINE_stage DBT_VERSION='1.2.3' DBT_ADAPTER_VERSION='3.4.5'" + == """CREATE OR REPLACE DBT PROJECT TEST_PIPELINE +FROM @MockDatabase.MockSchema.dbt_TEST_PIPELINE_stage MAIN_FILE='@MockDatabase.MockSchema.dbt_TEST_PIPELINE_stage/dbt_project.yml' +DBT_VERSION='1.2.3' DBT_ADAPTER_VERSION='3.4.5'""" ) stage_fqn = FQN.from_string(f"dbt_TEST_PIPELINE_stage").using_context() mock_create.assert_called_once_with(stage_fqn, temporary=True) @@ -100,7 +102,9 @@ def test_dbt_version_from_option_has_precedence_over_file( assert result.exit_code == 0, result.output assert ( mock_connect.mocked_ctx.get_query() - == "CREATE OR REPLACE DBT PROJECT TEST_PIPELINE FROM @MockDatabase.MockSchema.dbt_TEST_PIPELINE_stage DBT_VERSION='2.3.4' DBT_ADAPTER_VERSION='3.4.5'" + == """CREATE OR REPLACE DBT PROJECT TEST_PIPELINE +FROM @MockDatabase.MockSchema.dbt_TEST_PIPELINE_stage MAIN_FILE='@MockDatabase.MockSchema.dbt_TEST_PIPELINE_stage/dbt_project.yml' +DBT_VERSION='2.3.4' DBT_ADAPTER_VERSION='3.4.5'""" ) def test_raises_when_dbt_project_is_not_available( From 202f379a47168dc18a9892727292991fcec5d9eb Mon Sep 17 00:00:00 2001 From: Jakub Wilkowski Date: Wed, 26 Feb 2025 12:01:52 +0100 Subject: [PATCH 25/54] feat: [SNOW-1890085] dbt deploy: use --force to create or replace --- src/snowflake/cli/_plugins/dbt/manager.py | 3 +-- tests/dbt/test_dbt_commands.py | 25 +++++++++++++++++++++-- 2 files changed, 24 insertions(+), 4 deletions(-) diff --git a/src/snowflake/cli/_plugins/dbt/manager.py b/src/snowflake/cli/_plugins/dbt/manager.py index fb3e1ad9c7..e41b8cb052 100644 --- a/src/snowflake/cli/_plugins/dbt/manager.py +++ b/src/snowflake/cli/_plugins/dbt/manager.py @@ -52,7 +52,6 @@ def deploy( dbt_adapter_version: str, force: bool, ) -> SnowflakeCursor: - # TODO: what to do with force? dbt_project_path = path.joinpath("dbt_project.yml") if not dbt_project_path.exists(): raise ClickException(f"dbt_project.yml does not exist in provided path.") @@ -79,7 +78,7 @@ def deploy( with cli_console.phase("Creating DBT project"): staged_dbt_project_path = self._get_dbt_project_stage_path(stage_name) - query = f"""CREATE OR REPLACE DBT PROJECT {name} + query = f"""{'CREATE OR REPLACE' if force is True else 'CREATE'} DBT PROJECT {name} FROM {stage_name} MAIN_FILE='{staged_dbt_project_path}' DBT_VERSION='{dbt_version}' DBT_ADAPTER_VERSION='{dbt_adapter_version}'""" return self.execute_query(query) diff --git a/tests/dbt/test_dbt_commands.py b/tests/dbt/test_dbt_commands.py index f8cac2fda9..2d5b301bec 100644 --- a/tests/dbt/test_dbt_commands.py +++ b/tests/dbt/test_dbt_commands.py @@ -73,7 +73,7 @@ def test_deploys_project_from_source( assert result.exit_code == 0, result.output assert ( mock_connect.mocked_ctx.get_query() - == """CREATE OR REPLACE DBT PROJECT TEST_PIPELINE + == """CREATE DBT PROJECT TEST_PIPELINE FROM @MockDatabase.MockSchema.dbt_TEST_PIPELINE_stage MAIN_FILE='@MockDatabase.MockSchema.dbt_TEST_PIPELINE_stage/dbt_project.yml' DBT_VERSION='1.2.3' DBT_ADAPTER_VERSION='3.4.5'""" ) @@ -102,11 +102,32 @@ def test_dbt_version_from_option_has_precedence_over_file( assert result.exit_code == 0, result.output assert ( mock_connect.mocked_ctx.get_query() - == """CREATE OR REPLACE DBT PROJECT TEST_PIPELINE + == """CREATE DBT PROJECT TEST_PIPELINE FROM @MockDatabase.MockSchema.dbt_TEST_PIPELINE_stage MAIN_FILE='@MockDatabase.MockSchema.dbt_TEST_PIPELINE_stage/dbt_project.yml' DBT_VERSION='2.3.4' DBT_ADAPTER_VERSION='3.4.5'""" ) + @mock.patch("snowflake.cli._plugins.dbt.manager.StageManager.put_recursive") + @mock.patch("snowflake.cli._plugins.dbt.manager.StageManager.create") + def test_force_flag_uses_create_or_replace( + self, _mock_create, _mock_put_recursive, mock_connect, runner, dbt_project_path + ): + result = runner.invoke( + [ + "dbt", + "deploy", + "TEST_PIPELINE", + f"--source={dbt_project_path}", + "--force", + "--dbt-adapter-version=3.4.5", + ] + ) + + assert result.exit_code == 0, result.output + assert mock_connect.mocked_ctx.get_query().startswith( + "CREATE OR REPLACE DBT PROJECT" + ) + def test_raises_when_dbt_project_is_not_available( self, dbt_project_path, mock_connect, runner ): From db1b4d1bf2705e15451d5ddea93fc9a44c5b69fb Mon Sep 17 00:00:00 2001 From: Jakub Wilkowski Date: Wed, 26 Feb 2025 12:22:08 +0100 Subject: [PATCH 26/54] feat: [SNOW-1890085] dbt deploy: add execute_in_warehouse flag --- src/snowflake/cli/_plugins/dbt/commands.py | 10 ++++++++- src/snowflake/cli/_plugins/dbt/manager.py | 3 +++ tests/dbt/test_dbt_commands.py | 25 ++++++++++++++++++++++ 3 files changed, 37 insertions(+), 1 deletion(-) diff --git a/src/snowflake/cli/_plugins/dbt/commands.py b/src/snowflake/cli/_plugins/dbt/commands.py index 13047b3002..ca4ba22029 100644 --- a/src/snowflake/cli/_plugins/dbt/commands.py +++ b/src/snowflake/cli/_plugins/dbt/commands.py @@ -73,6 +73,9 @@ def deploy_dbt( dbt_adapter_version: str = typer.Option( help="dbt-snowflake adapter version to be used", ), + execute_in_warehouse: Optional[str] = typer.Option( + None, help="Warehouse to use when running `dbt execute` commands" + ), **options, ) -> CommandResult: """ @@ -85,7 +88,12 @@ def deploy_dbt( path = Path(source) return QueryResult( DBTManager().deploy( - path.resolve(), name, dbt_version, dbt_adapter_version, force=force + path.resolve(), + name, + dbt_version, + dbt_adapter_version, + execute_in_warehouse, + force=force, ) ) diff --git a/src/snowflake/cli/_plugins/dbt/manager.py b/src/snowflake/cli/_plugins/dbt/manager.py index e41b8cb052..f394c1adfd 100644 --- a/src/snowflake/cli/_plugins/dbt/manager.py +++ b/src/snowflake/cli/_plugins/dbt/manager.py @@ -50,6 +50,7 @@ def deploy( name: FQN, dbt_version: Optional[str], dbt_adapter_version: str, + execute_in_warehouse: Optional[str], force: bool, ) -> SnowflakeCursor: dbt_project_path = path.joinpath("dbt_project.yml") @@ -81,6 +82,8 @@ def deploy( query = f"""{'CREATE OR REPLACE' if force is True else 'CREATE'} DBT PROJECT {name} FROM {stage_name} MAIN_FILE='{staged_dbt_project_path}' DBT_VERSION='{dbt_version}' DBT_ADAPTER_VERSION='{dbt_adapter_version}'""" + if execute_in_warehouse: + query += f" WAREHOUSE='{execute_in_warehouse}'" return self.execute_query(query) def execute(self, dbt_command: str, name: str, *dbt_cli_args): diff --git a/tests/dbt/test_dbt_commands.py b/tests/dbt/test_dbt_commands.py index 2d5b301bec..65e396064a 100644 --- a/tests/dbt/test_dbt_commands.py +++ b/tests/dbt/test_dbt_commands.py @@ -14,6 +14,7 @@ from __future__ import annotations +from textwrap import dedent from unittest import mock import pytest @@ -128,6 +129,30 @@ def test_force_flag_uses_create_or_replace( "CREATE OR REPLACE DBT PROJECT" ) + @mock.patch("snowflake.cli._plugins.dbt.manager.StageManager.put_recursive") + @mock.patch("snowflake.cli._plugins.dbt.manager.StageManager.create") + def test_execute_in_warehouse( + self, _mock_create, _mock_put_recursive, mock_connect, runner, dbt_project_path + ): + + result = runner.invoke( + [ + "dbt", + "deploy", + "TEST_PIPELINE", + f"--source={dbt_project_path}", + "--dbt-adapter-version=3.4.5", + "--execute-in-warehouse=XL", + ] + ) + + assert result.exit_code == 0, result.output + assert mock_connect.mocked_ctx.get_query() == dedent( + """CREATE DBT PROJECT TEST_PIPELINE +FROM @MockDatabase.MockSchema.dbt_TEST_PIPELINE_stage MAIN_FILE='@MockDatabase.MockSchema.dbt_TEST_PIPELINE_stage/dbt_project.yml' +DBT_VERSION='1.2.3' DBT_ADAPTER_VERSION='3.4.5' WAREHOUSE='XL'""" + ) + def test_raises_when_dbt_project_is_not_available( self, dbt_project_path, mock_connect, runner ): From 28b5424a0b62761cc990cc804532413803e30a4c Mon Sep 17 00:00:00 2001 From: Jakub Wilkowski Date: Thu, 27 Feb 2025 14:14:39 +0100 Subject: [PATCH 27/54] refactor: [SNOW-1890085] hide dbt app behind feature flag --- src/snowflake/cli/_plugins/dbt/commands.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/snowflake/cli/_plugins/dbt/commands.py b/src/snowflake/cli/_plugins/dbt/commands.py index ca4ba22029..bd6c5f4c8d 100644 --- a/src/snowflake/cli/_plugins/dbt/commands.py +++ b/src/snowflake/cli/_plugins/dbt/commands.py @@ -24,13 +24,14 @@ from snowflake.cli.api.commands.decorators import global_options_with_connection from snowflake.cli.api.commands.flags import identifier_argument from snowflake.cli.api.commands.snow_typer import SnowTyperFactory +from snowflake.cli.api.feature_flags import FeatureFlag from snowflake.cli.api.identifiers import FQN from snowflake.cli.api.output.types import CommandResult, QueryResult app = SnowTyperFactory( name="dbt", help="Manages dbt on Snowflake projects", - is_hidden=lambda: True, + is_hidden=FeatureFlag.ENABLE_DBT_POC.is_disabled, ) log = logging.getLogger(__name__) From 09d8b5eaa8340345adbb62372f85ff665b360bd1 Mon Sep 17 00:00:00 2001 From: Jakub Wilkowski Date: Mon, 10 Mar 2025 10:55:49 +0100 Subject: [PATCH 28/54] feat: [SNOW-1966187] update commands according to backend implementation --- src/snowflake/cli/_plugins/dbt/commands.py | 5 ++-- src/snowflake/cli/_plugins/dbt/manager.py | 20 +++++++--------- tests/dbt/test_dbt_commands.py | 28 ++++++++++++---------- 3 files changed, 27 insertions(+), 26 deletions(-) diff --git a/src/snowflake/cli/_plugins/dbt/commands.py b/src/snowflake/cli/_plugins/dbt/commands.py index bd6c5f4c8d..aed4c2413e 100644 --- a/src/snowflake/cli/_plugins/dbt/commands.py +++ b/src/snowflake/cli/_plugins/dbt/commands.py @@ -71,7 +71,8 @@ def deploy_dbt( None, help="Version of dbt tool to be used. Taken from dbt_project.yml if not provided.", ), - dbt_adapter_version: str = typer.Option( + dbt_adapter_version: Optional[str] = typer.Option( + None, help="dbt-snowflake adapter version to be used", ), execute_in_warehouse: Optional[str] = typer.Option( @@ -82,7 +83,6 @@ def deploy_dbt( """ Copy dbt files and create or update dbt on Snowflake project. """ - # TODO: options for DBT version? if source is None: path = Path.cwd() else: @@ -131,7 +131,6 @@ def before_callback( def _dbt_execute( ctx: typer.Context, ) -> CommandResult: - # TODO: figure out how to present logs to users dbt_cli_args = ctx.args dbt_command = ctx.command.name name = ctx.parent.params["name"] diff --git a/src/snowflake/cli/_plugins/dbt/manager.py b/src/snowflake/cli/_plugins/dbt/manager.py index f394c1adfd..8502f9585b 100644 --- a/src/snowflake/cli/_plugins/dbt/manager.py +++ b/src/snowflake/cli/_plugins/dbt/manager.py @@ -41,7 +41,7 @@ def execute_query(self, query, **kwargs): class DBTManager(StdoutExecutionMixin): def list(self) -> SnowflakeCursor: # noqa: A003 - query = "SHOW DBT PROJECT" + query = "SHOW DBT PROJECTS" return self.execute_query(query) def deploy( @@ -78,20 +78,18 @@ def deploy( cli_console.step(f"Copied {len(results)} files") with cli_console.phase("Creating DBT project"): - staged_dbt_project_path = self._get_dbt_project_stage_path(stage_name) query = f"""{'CREATE OR REPLACE' if force is True else 'CREATE'} DBT PROJECT {name} -FROM {stage_name} MAIN_FILE='{staged_dbt_project_path}' -DBT_VERSION='{dbt_version}' DBT_ADAPTER_VERSION='{dbt_adapter_version}'""" +FROM {stage_name} +DBT_VERSION='{dbt_version}'""" + + if dbt_adapter_version: + query += f"\nDBT_ADAPTER_VERSION='{dbt_adapter_version}'" if execute_in_warehouse: - query += f" WAREHOUSE='{execute_in_warehouse}'" + query += f"\nWAREHOUSE='{execute_in_warehouse}'" return self.execute_query(query) def execute(self, dbt_command: str, name: str, *dbt_cli_args): - query = f"EXECUTE DBT PROJECT {name} {dbt_command}" if dbt_cli_args: - query += " " + " ".join([arg for arg in dbt_cli_args]) + dbt_command = dbt_command + " " + " ".join([arg for arg in dbt_cli_args]) + query = f"EXECUTE DBT PROJECT {name} args='{dbt_command.strip()}'" return self.execute_query(query) - - @staticmethod - def _get_dbt_project_stage_path(stage_name): - return "/".join([stage_name, "dbt_project.yml"]) diff --git a/tests/dbt/test_dbt_commands.py b/tests/dbt/test_dbt_commands.py index 65e396064a..e64df06fb8 100644 --- a/tests/dbt/test_dbt_commands.py +++ b/tests/dbt/test_dbt_commands.py @@ -37,7 +37,7 @@ def test_dbt_list(self, mock_connect, runner): result = runner.invoke(["dbt", "list"]) assert result.exit_code == 0, result.output - assert mock_connect.mocked_ctx.get_query() == "SHOW DBT PROJECT" + assert mock_connect.mocked_ctx.get_query() == "SHOW DBT PROJECTS" class TestDBTDeploy: @@ -75,8 +75,9 @@ def test_deploys_project_from_source( assert ( mock_connect.mocked_ctx.get_query() == """CREATE DBT PROJECT TEST_PIPELINE -FROM @MockDatabase.MockSchema.dbt_TEST_PIPELINE_stage MAIN_FILE='@MockDatabase.MockSchema.dbt_TEST_PIPELINE_stage/dbt_project.yml' -DBT_VERSION='1.2.3' DBT_ADAPTER_VERSION='3.4.5'""" +FROM @MockDatabase.MockSchema.dbt_TEST_PIPELINE_stage +DBT_VERSION='1.2.3' +DBT_ADAPTER_VERSION='3.4.5'""" ) stage_fqn = FQN.from_string(f"dbt_TEST_PIPELINE_stage").using_context() mock_create.assert_called_once_with(stage_fqn, temporary=True) @@ -104,8 +105,9 @@ def test_dbt_version_from_option_has_precedence_over_file( assert ( mock_connect.mocked_ctx.get_query() == """CREATE DBT PROJECT TEST_PIPELINE -FROM @MockDatabase.MockSchema.dbt_TEST_PIPELINE_stage MAIN_FILE='@MockDatabase.MockSchema.dbt_TEST_PIPELINE_stage/dbt_project.yml' -DBT_VERSION='2.3.4' DBT_ADAPTER_VERSION='3.4.5'""" +FROM @MockDatabase.MockSchema.dbt_TEST_PIPELINE_stage +DBT_VERSION='2.3.4' +DBT_ADAPTER_VERSION='3.4.5'""" ) @mock.patch("snowflake.cli._plugins.dbt.manager.StageManager.put_recursive") @@ -149,8 +151,10 @@ def test_execute_in_warehouse( assert result.exit_code == 0, result.output assert mock_connect.mocked_ctx.get_query() == dedent( """CREATE DBT PROJECT TEST_PIPELINE -FROM @MockDatabase.MockSchema.dbt_TEST_PIPELINE_stage MAIN_FILE='@MockDatabase.MockSchema.dbt_TEST_PIPELINE_stage/dbt_project.yml' -DBT_VERSION='1.2.3' DBT_ADAPTER_VERSION='3.4.5' WAREHOUSE='XL'""" +FROM @MockDatabase.MockSchema.dbt_TEST_PIPELINE_stage +DBT_VERSION='1.2.3' +DBT_ADAPTER_VERSION='3.4.5' +WAREHOUSE='XL'""" ) def test_raises_when_dbt_project_is_not_available( @@ -209,7 +213,7 @@ class TestDBTExecute: "pipeline_name", "test", ], - "EXECUTE DBT PROJECT pipeline_name test", + "EXECUTE DBT PROJECT pipeline_name args='test'", id="simple-command", ), pytest.param( @@ -221,12 +225,12 @@ class TestDBTExecute: "-f", "--select @source:snowplow,tag:nightly models/export", ], - "EXECUTE DBT PROJECT pipeline_name run -f --select @source:snowplow,tag:nightly models/export", + "EXECUTE DBT PROJECT pipeline_name args='run -f --select @source:snowplow,tag:nightly models/export'", id="with-dbt-options", ), pytest.param( ["dbt", "execute", "pipeline_name", "compile", "--vars '{foo:bar}'"], - "EXECUTE DBT PROJECT pipeline_name compile --vars '{foo:bar}'", + "EXECUTE DBT PROJECT pipeline_name args='compile --vars '{foo:bar}''", id="with-dbt-vars", ), pytest.param( @@ -242,7 +246,7 @@ class TestDBTExecute: "--info", "--config-file=/", ], - "EXECUTE DBT PROJECT pipeline_name compile --format=TXT -v -h --debug --info --config-file=/", + "EXECUTE DBT PROJECT pipeline_name args='compile --format=TXT -v -h --debug --info --config-file=/'", id="with-dbt-conflicting-options", ), pytest.param( @@ -253,7 +257,7 @@ class TestDBTExecute: "pipeline_name", "compile", ], - "EXECUTE DBT PROJECT pipeline_name compile", + "EXECUTE DBT PROJECT pipeline_name args='compile'", id="with-cli-flag", ), ], From 9fb7e875eb136d4a064eb6b18515c1bc3d6b5814 Mon Sep 17 00:00:00 2001 From: Jakub Wilkowski Date: Wed, 12 Mar 2025 10:55:49 +0100 Subject: [PATCH 29/54] feat: [SNOW-1966187] remove StdoutExecutionMixin --- src/snowflake/cli/_plugins/dbt/manager.py | 15 +-------------- 1 file changed, 1 insertion(+), 14 deletions(-) diff --git a/src/snowflake/cli/_plugins/dbt/manager.py b/src/snowflake/cli/_plugins/dbt/manager.py index 8502f9585b..c42715aa41 100644 --- a/src/snowflake/cli/_plugins/dbt/manager.py +++ b/src/snowflake/cli/_plugins/dbt/manager.py @@ -21,25 +21,12 @@ from click import ClickException from snowflake.cli._plugins.stage.manager import StageManager from snowflake.cli.api.console import cli_console -from snowflake.cli.api.feature_flags import FeatureFlag from snowflake.cli.api.identifiers import FQN from snowflake.cli.api.sql_execution import SqlExecutionMixin from snowflake.connector.cursor import SnowflakeCursor -class StdoutExecutionMixin(SqlExecutionMixin): - def execute_query(self, query, **kwargs): - if FeatureFlag.ENABLE_DBT_POC.is_enabled(): - from unittest.mock import MagicMock - - cli_console.message(f"Sending query: {query}") - mock_cursor = MagicMock() - mock_cursor.description = [] - return mock_cursor - return super().execute_query(query, **kwargs) - - -class DBTManager(StdoutExecutionMixin): +class DBTManager(SqlExecutionMixin): def list(self) -> SnowflakeCursor: # noqa: A003 query = "SHOW DBT PROJECTS" return self.execute_query(query) From c14b07b91aeaaa5446ef40cd3a587287ffe65c88 Mon Sep 17 00:00:00 2001 From: Jakub Wilkowski Date: Mon, 17 Mar 2025 12:16:17 +0100 Subject: [PATCH 30/54] feat: [SNOW-1890085] update command help texts --- src/snowflake/cli/_plugins/dbt/commands.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/snowflake/cli/_plugins/dbt/commands.py b/src/snowflake/cli/_plugins/dbt/commands.py index aed4c2413e..4c1f8b0a27 100644 --- a/src/snowflake/cli/_plugins/dbt/commands.py +++ b/src/snowflake/cli/_plugins/dbt/commands.py @@ -103,7 +103,7 @@ def deploy_dbt( # suppressing usual CLI behaviour for displaying help or formatting options. dbt_execute_app = SnowTyperFactory( name="execute", - help="Execute a dbt command", + help="Execute a dbt command on Snowflake", ) app.add_typer(dbt_execute_app) @@ -125,7 +125,7 @@ def before_callback( requires_connection=False, requires_global_options=False, context_settings={"allow_extra_args": True, "ignore_unknown_options": True}, - help=f"Execute {cmd} command on dbt on Snowflake project.", + help=f"Execute {cmd} command on Snowflake.", add_help_option=False, ) def _dbt_execute( From 3ebaf8c518b5aed1235601a1b46c3e43aaa0c376 Mon Sep 17 00:00:00 2001 From: Jakub Wilkowski Date: Mon, 17 Mar 2025 13:29:53 +0100 Subject: [PATCH 31/54] feat: [SNOW-1890085] add run_async option --- src/snowflake/cli/_plugins/dbt/commands.py | 13 +++++++++++-- src/snowflake/cli/_plugins/dbt/manager.py | 4 ++-- tests/dbt/test_dbt_commands.py | 20 ++++++++++++++++++++ tests/testing_utils/fixtures.py | 0 4 files changed, 33 insertions(+), 4 deletions(-) create mode 100644 tests/testing_utils/fixtures.py diff --git a/src/snowflake/cli/_plugins/dbt/commands.py b/src/snowflake/cli/_plugins/dbt/commands.py index 4c1f8b0a27..4d3c8bf979 100644 --- a/src/snowflake/cli/_plugins/dbt/commands.py +++ b/src/snowflake/cli/_plugins/dbt/commands.py @@ -26,7 +26,7 @@ from snowflake.cli.api.commands.snow_typer import SnowTyperFactory from snowflake.cli.api.feature_flags import FeatureFlag from snowflake.cli.api.identifiers import FQN -from snowflake.cli.api.output.types import CommandResult, QueryResult +from snowflake.cli.api.output.types import CommandResult, MessageResult, QueryResult app = SnowTyperFactory( name="dbt", @@ -112,6 +112,9 @@ def deploy_dbt( @global_options_with_connection def before_callback( name: FQN = DBTNameArgument, + run_async: Optional[bool] = typer.Option( + False, help="Run dbt command asynchronously and check it's result later." + ), **options, ): """Handles global options passed before the command and takes pipeline name to be accessed through child context later""" @@ -134,4 +137,10 @@ def _dbt_execute( dbt_cli_args = ctx.args dbt_command = ctx.command.name name = ctx.parent.params["name"] - return QueryResult(DBTManager().execute(dbt_command, name, *dbt_cli_args)) + run_async = ctx.parent.params["run_async"] + result = DBTManager().execute(dbt_command, name, run_async, *dbt_cli_args) + if not run_async: + return QueryResult(result) + return MessageResult( + f"Command submitted. You can check the result with `snow sql -q \"select execution_status from table(information_schema.query_history_by_user()) where query_id in ('{result.sfqid}');\"`" + ) diff --git a/src/snowflake/cli/_plugins/dbt/manager.py b/src/snowflake/cli/_plugins/dbt/manager.py index c42715aa41..31791bbd58 100644 --- a/src/snowflake/cli/_plugins/dbt/manager.py +++ b/src/snowflake/cli/_plugins/dbt/manager.py @@ -75,8 +75,8 @@ def deploy( query += f"\nWAREHOUSE='{execute_in_warehouse}'" return self.execute_query(query) - def execute(self, dbt_command: str, name: str, *dbt_cli_args): + def execute(self, dbt_command: str, name: str, run_async: bool, *dbt_cli_args): if dbt_cli_args: dbt_command = dbt_command + " " + " ".join([arg for arg in dbt_cli_args]) query = f"EXECUTE DBT PROJECT {name} args='{dbt_command.strip()}'" - return self.execute_query(query) + return self.execute_query(query, _exec_async=run_async) diff --git a/tests/dbt/test_dbt_commands.py b/tests/dbt/test_dbt_commands.py index e64df06fb8..68cccb8ae5 100644 --- a/tests/dbt/test_dbt_commands.py +++ b/tests/dbt/test_dbt_commands.py @@ -267,4 +267,24 @@ def test_dbt_execute(self, mock_connect, runner, args, expected_query): result = runner.invoke(args) assert result.exit_code == 0, result.output + assert mock_connect.mocked_ctx.kwargs[0]["_exec_async"] is False assert mock_connect.mocked_ctx.get_query() == expected_query + + def test_execute_async(self, mock_connect, runner): + result = runner.invoke( + [ + "dbt", + "execute", + "--run-async", + "pipeline_name", + "compile", + ] + ) + + assert result.exit_code == 0, result.output + assert result.output.startswith("Command submitted") + assert mock_connect.mocked_ctx.kwargs[0]["_exec_async"] is True + assert ( + mock_connect.mocked_ctx.get_query() + == "EXECUTE DBT PROJECT pipeline_name args='compile'" + ) diff --git a/tests/testing_utils/fixtures.py b/tests/testing_utils/fixtures.py new file mode 100644 index 0000000000..e69de29bb2 From 5bf8f22185bbed0370a36c0617737ec296a3bedf Mon Sep 17 00:00:00 2001 From: Jakub Wilkowski Date: Mon, 17 Mar 2025 13:30:32 +0100 Subject: [PATCH 32/54] feat: [SNOW-1890085] use SecurePath --- src/snowflake/cli/_plugins/dbt/commands.py | 6 +++--- src/snowflake/cli/_plugins/dbt/manager.py | 11 ++++++----- src/snowflake/cli/api/secure_path.py | 4 ++++ 3 files changed, 13 insertions(+), 8 deletions(-) diff --git a/src/snowflake/cli/_plugins/dbt/commands.py b/src/snowflake/cli/_plugins/dbt/commands.py index 4d3c8bf979..65b88b80c2 100644 --- a/src/snowflake/cli/_plugins/dbt/commands.py +++ b/src/snowflake/cli/_plugins/dbt/commands.py @@ -15,7 +15,6 @@ from __future__ import annotations import logging -from pathlib import Path from typing import Optional import typer @@ -27,6 +26,7 @@ from snowflake.cli.api.feature_flags import FeatureFlag from snowflake.cli.api.identifiers import FQN from snowflake.cli.api.output.types import CommandResult, MessageResult, QueryResult +from snowflake.cli.api.secure_path import SecurePath app = SnowTyperFactory( name="dbt", @@ -84,9 +84,9 @@ def deploy_dbt( Copy dbt files and create or update dbt on Snowflake project. """ if source is None: - path = Path.cwd() + path = SecurePath.cwd() else: - path = Path(source) + path = SecurePath(source) return QueryResult( DBTManager().deploy( path.resolve(), diff --git a/src/snowflake/cli/_plugins/dbt/manager.py b/src/snowflake/cli/_plugins/dbt/manager.py index 31791bbd58..b9ed199e79 100644 --- a/src/snowflake/cli/_plugins/dbt/manager.py +++ b/src/snowflake/cli/_plugins/dbt/manager.py @@ -14,14 +14,15 @@ from __future__ import annotations -from pathlib import Path from typing import Optional import yaml from click import ClickException from snowflake.cli._plugins.stage.manager import StageManager from snowflake.cli.api.console import cli_console +from snowflake.cli.api.constants import DEFAULT_SIZE_LIMIT_MB from snowflake.cli.api.identifiers import FQN +from snowflake.cli.api.secure_path import SecurePath from snowflake.cli.api.sql_execution import SqlExecutionMixin from snowflake.connector.cursor import SnowflakeCursor @@ -33,19 +34,19 @@ def list(self) -> SnowflakeCursor: # noqa: A003 def deploy( self, - path: Path, + path: SecurePath, name: FQN, dbt_version: Optional[str], dbt_adapter_version: str, execute_in_warehouse: Optional[str], force: bool, ) -> SnowflakeCursor: - dbt_project_path = path.joinpath("dbt_project.yml") + dbt_project_path = path / "dbt_project.yml" if not dbt_project_path.exists(): raise ClickException(f"dbt_project.yml does not exist in provided path.") if dbt_version is None: - with dbt_project_path.open() as fd: + with dbt_project_path.open(read_file_limit_mb=DEFAULT_SIZE_LIMIT_MB) as fd: dbt_project_config = yaml.safe_load(fd) try: dbt_version = dbt_project_config["version"] @@ -61,7 +62,7 @@ def deploy( stage_manager.create(stage_fqn, temporary=True) with cli_console.phase("Copying project files to stage"): - results = list(stage_manager.put_recursive(path, stage_name)) + results = list(stage_manager.put_recursive(path.path, stage_name)) cli_console.step(f"Copied {len(results)} files") with cli_console.phase("Creating DBT project"): diff --git a/src/snowflake/cli/api/secure_path.py b/src/snowflake/cli/api/secure_path.py index f920fe194c..3891081189 100644 --- a/src/snowflake/cli/api/secure_path.py +++ b/src/snowflake/cli/api/secure_path.py @@ -72,6 +72,10 @@ def absolute(self): """ return SecurePath(self._path.absolute()) + @classmethod + def cwd(cls) -> SecurePath: + return cls(Path.cwd()) + def resolve(self): """ Make the path absolute, resolving symlinks From 9d041a10de8c5aef741525bd8ce426722101f109 Mon Sep 17 00:00:00 2001 From: Jakub Wilkowski Date: Mon, 17 Mar 2025 15:44:09 +0100 Subject: [PATCH 33/54] feat: [SNOW-1890085] check if dbt project exists before deploying --- src/snowflake/cli/_plugins/dbt/commands.py | 2 +- src/snowflake/cli/_plugins/dbt/manager.py | 14 ++++- src/snowflake/cli/api/constants.py | 2 + tests/dbt/test_dbt_commands.py | 64 ++++++++++++++++++++-- 4 files changed, 76 insertions(+), 6 deletions(-) diff --git a/src/snowflake/cli/_plugins/dbt/commands.py b/src/snowflake/cli/_plugins/dbt/commands.py index 65b88b80c2..2bc7645b39 100644 --- a/src/snowflake/cli/_plugins/dbt/commands.py +++ b/src/snowflake/cli/_plugins/dbt/commands.py @@ -36,7 +36,7 @@ log = logging.getLogger(__name__) -DBTNameArgument = identifier_argument(sf_object="DBT Object", example="my_pipeline") +DBTNameArgument = identifier_argument(sf_object="DBT Project", example="my_pipeline") @app.command( diff --git a/src/snowflake/cli/_plugins/dbt/manager.py b/src/snowflake/cli/_plugins/dbt/manager.py index b9ed199e79..67e7c89d0e 100644 --- a/src/snowflake/cli/_plugins/dbt/manager.py +++ b/src/snowflake/cli/_plugins/dbt/manager.py @@ -18,9 +18,10 @@ import yaml from click import ClickException +from snowflake.cli._plugins.object.manager import ObjectManager from snowflake.cli._plugins.stage.manager import StageManager from snowflake.cli.api.console import cli_console -from snowflake.cli.api.constants import DEFAULT_SIZE_LIMIT_MB +from snowflake.cli.api.constants import DEFAULT_SIZE_LIMIT_MB, ObjectType from snowflake.cli.api.identifiers import FQN from snowflake.cli.api.secure_path import SecurePath from snowflake.cli.api.sql_execution import SqlExecutionMixin @@ -32,6 +33,12 @@ def list(self) -> SnowflakeCursor: # noqa: A003 query = "SHOW DBT PROJECTS" return self.execute_query(query) + @staticmethod + def exists(name: FQN) -> bool: + return ObjectManager().object_exists( + object_type=ObjectType.DBT_PROJECT.value.cli_name, fqn=name + ) + def deploy( self, path: SecurePath, @@ -55,6 +62,11 @@ def deploy( f"dbt-version was not provided and is not available in dbt_project.yml" ) + if self.exists(name=name) and force is not True: + raise ClickException( + f"DBT project {name} already exists. Use --force flag to overwrite" + ) + with cli_console.phase("Creating temporary stage"): stage_manager = StageManager() stage_fqn = FQN.from_string(f"dbt_{name}_stage").using_context() diff --git a/src/snowflake/cli/api/constants.py b/src/snowflake/cli/api/constants.py index 0c0674a9ad..871bcd831c 100644 --- a/src/snowflake/cli/api/constants.py +++ b/src/snowflake/cli/api/constants.py @@ -35,6 +35,7 @@ def __str__(self): class ObjectType(Enum): COMPUTE_POOL = ObjectNames("compute-pool", "compute pool", "compute pools") + DBT_PROJECT = ObjectNames("dbt-project", "DBT project", "DBT projects") DATABASE = ObjectNames("database", "database", "databases") FUNCTION = ObjectNames("function", "function", "functions") INTEGRATION = ObjectNames("integration", "integration", "integrations") @@ -79,6 +80,7 @@ def __str__(self): ObjectType.APPLICATION.value.cli_name, ObjectType.APPLICATION_PACKAGE.value.cli_name, ObjectType.PROJECT.value.cli_name, + ObjectType.DBT_PROJECT.value.cli_name, } SUPPORTED_OBJECTS = sorted(OBJECT_TO_NAMES.keys() - UNSUPPORTED_OBJECTS) diff --git a/tests/dbt/test_dbt_commands.py b/tests/dbt/test_dbt_commands.py index 68cccb8ae5..8a6f4637d4 100644 --- a/tests/dbt/test_dbt_commands.py +++ b/tests/dbt/test_dbt_commands.py @@ -55,10 +55,23 @@ def mock_cli_console(self): with mock.patch("snowflake.cli.api.console") as _fixture: yield _fixture + @pytest.fixture + def mock_exists(self): + with mock.patch( + "snowflake.cli._plugins.dbt.manager.DBTManager.exists", return_value=False + ) as _fixture: + yield _fixture + @mock.patch("snowflake.cli._plugins.dbt.manager.StageManager.put_recursive") @mock.patch("snowflake.cli._plugins.dbt.manager.StageManager.create") def test_deploys_project_from_source( - self, mock_create, mock_put_recursive, mock_connect, runner, dbt_project_path + self, + mock_create, + mock_put_recursive, + mock_connect, + runner, + dbt_project_path, + mock_exists, ): result = runner.invoke( @@ -88,7 +101,13 @@ def test_deploys_project_from_source( @mock.patch("snowflake.cli._plugins.dbt.manager.StageManager.put_recursive") @mock.patch("snowflake.cli._plugins.dbt.manager.StageManager.create") def test_dbt_version_from_option_has_precedence_over_file( - self, _mock_create, _mock_put_recursive, mock_connect, runner, dbt_project_path + self, + _mock_create, + _mock_put_recursive, + mock_connect, + runner, + dbt_project_path, + mock_exists, ): result = runner.invoke( [ @@ -110,11 +129,21 @@ def test_dbt_version_from_option_has_precedence_over_file( DBT_ADAPTER_VERSION='3.4.5'""" ) + @pytest.mark.parametrize("exists", (True, False)) @mock.patch("snowflake.cli._plugins.dbt.manager.StageManager.put_recursive") @mock.patch("snowflake.cli._plugins.dbt.manager.StageManager.create") def test_force_flag_uses_create_or_replace( - self, _mock_create, _mock_put_recursive, mock_connect, runner, dbt_project_path + self, + _mock_create, + _mock_put_recursive, + exists, + mock_connect, + runner, + dbt_project_path, + mock_exists, ): + mock_exists.return_value = exists + result = runner.invoke( [ "dbt", @@ -134,7 +163,13 @@ def test_force_flag_uses_create_or_replace( @mock.patch("snowflake.cli._plugins.dbt.manager.StageManager.put_recursive") @mock.patch("snowflake.cli._plugins.dbt.manager.StageManager.create") def test_execute_in_warehouse( - self, _mock_create, _mock_put_recursive, mock_connect, runner, dbt_project_path + self, + _mock_create, + _mock_put_recursive, + mock_connect, + runner, + dbt_project_path, + mock_exists, ): result = runner.invoke( @@ -201,6 +236,27 @@ def test_raises_when_dbt_project_version_is_not_specified( ) assert mock_connect.mocked_ctx.get_query() == "" + def test_raises_when_dbt_project_exists_and_is_not_force( + self, dbt_project_path, mock_connect, runner, mock_exists + ): + mock_exists.return_value = True + + result = runner.invoke( + [ + "dbt", + "deploy", + "TEST_PIPELINE", + f"--source={dbt_project_path}", + ], + ) + + assert result.exit_code == 1, result.output + assert ( + "DBT project TEST_PIPELINE already exists. Use --force flag to overwrite" + in result.output + ) + assert mock_connect.mocked_ctx.get_query() == "" + class TestDBTExecute: @pytest.mark.parametrize( From 6d620c7f51b287968b4298a6ea369c5ca6d3d479 Mon Sep 17 00:00:00 2001 From: Jakub Wilkowski Date: Mon, 17 Mar 2025 15:51:55 +0100 Subject: [PATCH 34/54] refactor: [SNOW-1890085] use object plugin for listing dbt projects --- src/snowflake/cli/_plugins/dbt/commands.py | 24 +++++++++-------- src/snowflake/cli/api/constants.py | 2 +- tests/dbt/test_dbt_commands.py | 30 +++++++++++++++++++--- 3 files changed, 40 insertions(+), 16 deletions(-) diff --git a/src/snowflake/cli/_plugins/dbt/commands.py b/src/snowflake/cli/_plugins/dbt/commands.py index 2bc7645b39..db8c2aa0a2 100644 --- a/src/snowflake/cli/_plugins/dbt/commands.py +++ b/src/snowflake/cli/_plugins/dbt/commands.py @@ -20,9 +20,12 @@ import typer from snowflake.cli._plugins.dbt.constants import DBT_COMMANDS from snowflake.cli._plugins.dbt.manager import DBTManager +from snowflake.cli._plugins.object.command_aliases import add_object_command_aliases +from snowflake.cli._plugins.object.commands import scope_option from snowflake.cli.api.commands.decorators import global_options_with_connection -from snowflake.cli.api.commands.flags import identifier_argument +from snowflake.cli.api.commands.flags import identifier_argument, like_option from snowflake.cli.api.commands.snow_typer import SnowTyperFactory +from snowflake.cli.api.constants import ObjectType from snowflake.cli.api.feature_flags import FeatureFlag from snowflake.cli.api.identifiers import FQN from snowflake.cli.api.output.types import CommandResult, MessageResult, QueryResult @@ -39,17 +42,16 @@ DBTNameArgument = identifier_argument(sf_object="DBT Project", example="my_pipeline") -@app.command( - "list", - requires_connection=True, +add_object_command_aliases( + app=app, + object_type=ObjectType.DBT_PROJECT, + name_argument=DBTNameArgument, + like_option=like_option( + help_example='`list --like "my%"` lists all dbt projects that begin with “my”' + ), + scope_option=scope_option(help_example="`list --in database my_db`"), + ommit_commands=["drop", "create", "describe"], ) -def list_dbts( - **options, -) -> CommandResult: - """ - List all dbt on Snowflake projects. - """ - return QueryResult(DBTManager().list()) @app.command( diff --git a/src/snowflake/cli/api/constants.py b/src/snowflake/cli/api/constants.py index 871bcd831c..b15112a7cf 100644 --- a/src/snowflake/cli/api/constants.py +++ b/src/snowflake/cli/api/constants.py @@ -35,7 +35,7 @@ def __str__(self): class ObjectType(Enum): COMPUTE_POOL = ObjectNames("compute-pool", "compute pool", "compute pools") - DBT_PROJECT = ObjectNames("dbt-project", "DBT project", "DBT projects") + DBT_PROJECT = ObjectNames("dbt-project", "dbt project", "dbt projects") DATABASE = ObjectNames("database", "database", "databases") FUNCTION = ObjectNames("function", "function", "functions") INTEGRATION = ObjectNames("integration", "integration", "integrations") diff --git a/tests/dbt/test_dbt_commands.py b/tests/dbt/test_dbt_commands.py index 8a6f4637d4..f984933d0c 100644 --- a/tests/dbt/test_dbt_commands.py +++ b/tests/dbt/test_dbt_commands.py @@ -32,12 +32,34 @@ def mock_connect(mock_ctx): class TestDBTList: - def test_dbt_list(self, mock_connect, runner): - - result = runner.invoke(["dbt", "list"]) + def test_list_command_alias(self, mock_connect, runner): + result = runner.invoke( + [ + "object", + "list", + "dbt-project", + "--like", + "%PROJECT_NAME%", + "--in", + "database", + "my_db", + ] + ) assert result.exit_code == 0, result.output - assert mock_connect.mocked_ctx.get_query() == "SHOW DBT PROJECTS" + result = runner.invoke( + ["dbt", "list", "--like", "%PROJECT_NAME%", "--in", "database", "my_db"], + catch_exceptions=False, + ) + assert result.exit_code == 0, result.output + + queries = mock_connect.mocked_ctx.get_queries() + assert len(queries) == 2 + assert ( + queries[0] + == queries[1] + == "show dbt projects like '%PROJECT_NAME%' in database my_db" + ) class TestDBTDeploy: From cf5a477a04e43b05de2ce1143572066f9a74d303 Mon Sep 17 00:00:00 2001 From: Jakub Wilkowski Date: Mon, 17 Mar 2025 18:15:47 +0100 Subject: [PATCH 35/54] refactor: [SNOW-1890085] set dbt execute subcommand metavar --- src/snowflake/cli/_plugins/dbt/commands.py | 1 + src/snowflake/cli/api/commands/snow_typer.py | 3 +++ 2 files changed, 4 insertions(+) diff --git a/src/snowflake/cli/_plugins/dbt/commands.py b/src/snowflake/cli/_plugins/dbt/commands.py index db8c2aa0a2..4f92d7ef8e 100644 --- a/src/snowflake/cli/_plugins/dbt/commands.py +++ b/src/snowflake/cli/_plugins/dbt/commands.py @@ -106,6 +106,7 @@ def deploy_dbt( dbt_execute_app = SnowTyperFactory( name="execute", help="Execute a dbt command on Snowflake", + subcommand_metavar="DBT_COMMAND", ) app.add_typer(dbt_execute_app) diff --git a/src/snowflake/cli/api/commands/snow_typer.py b/src/snowflake/cli/api/commands/snow_typer.py index 786bb6e776..2bc4bdd807 100644 --- a/src/snowflake/cli/api/commands/snow_typer.py +++ b/src/snowflake/cli/api/commands/snow_typer.py @@ -228,6 +228,7 @@ def __init__( short_help: Optional[str] = None, is_hidden: Optional[Callable[[], bool]] = None, deprecated: bool = False, + subcommand_metavar: Optional[str] = None, ): self.name = name self.help = help @@ -237,6 +238,7 @@ def __init__( self.commands_to_register: List[SnowTyperCommandData] = [] self.subapps_to_register: List[SnowTyperFactory] = [] self.callbacks_to_register: List[Callable] = [] + self.subcommand_metavar = subcommand_metavar def create_instance(self) -> SnowTyper: app = SnowTyper( @@ -245,6 +247,7 @@ def create_instance(self) -> SnowTyper: short_help=self.short_help, hidden=self.is_hidden() if self.is_hidden else False, deprecated=self.deprecated, + subcommand_metavar=self.subcommand_metavar, ) # register commands for command in self.commands_to_register: From 2926a172a63c8739c9c7d5a5718a4ad814e69852 Mon Sep 17 00:00:00 2001 From: Jakub Wilkowski Date: Tue, 18 Mar 2025 08:05:41 +0100 Subject: [PATCH 36/54] feat: [SNOW-1890085] remove execute_in_warehouse option --- src/snowflake/cli/_plugins/dbt/commands.py | 4 --- src/snowflake/cli/_plugins/dbt/manager.py | 3 -- tests/dbt/test_dbt_commands.py | 33 ---------------------- 3 files changed, 40 deletions(-) diff --git a/src/snowflake/cli/_plugins/dbt/commands.py b/src/snowflake/cli/_plugins/dbt/commands.py index 4f92d7ef8e..3a81061daa 100644 --- a/src/snowflake/cli/_plugins/dbt/commands.py +++ b/src/snowflake/cli/_plugins/dbt/commands.py @@ -77,9 +77,6 @@ def deploy_dbt( None, help="dbt-snowflake adapter version to be used", ), - execute_in_warehouse: Optional[str] = typer.Option( - None, help="Warehouse to use when running `dbt execute` commands" - ), **options, ) -> CommandResult: """ @@ -95,7 +92,6 @@ def deploy_dbt( name, dbt_version, dbt_adapter_version, - execute_in_warehouse, force=force, ) ) diff --git a/src/snowflake/cli/_plugins/dbt/manager.py b/src/snowflake/cli/_plugins/dbt/manager.py index 67e7c89d0e..7c164d842d 100644 --- a/src/snowflake/cli/_plugins/dbt/manager.py +++ b/src/snowflake/cli/_plugins/dbt/manager.py @@ -45,7 +45,6 @@ def deploy( name: FQN, dbt_version: Optional[str], dbt_adapter_version: str, - execute_in_warehouse: Optional[str], force: bool, ) -> SnowflakeCursor: dbt_project_path = path / "dbt_project.yml" @@ -84,8 +83,6 @@ def deploy( if dbt_adapter_version: query += f"\nDBT_ADAPTER_VERSION='{dbt_adapter_version}'" - if execute_in_warehouse: - query += f"\nWAREHOUSE='{execute_in_warehouse}'" return self.execute_query(query) def execute(self, dbt_command: str, name: str, run_async: bool, *dbt_cli_args): diff --git a/tests/dbt/test_dbt_commands.py b/tests/dbt/test_dbt_commands.py index f984933d0c..fbe79cf8d7 100644 --- a/tests/dbt/test_dbt_commands.py +++ b/tests/dbt/test_dbt_commands.py @@ -14,7 +14,6 @@ from __future__ import annotations -from textwrap import dedent from unittest import mock import pytest @@ -182,38 +181,6 @@ def test_force_flag_uses_create_or_replace( "CREATE OR REPLACE DBT PROJECT" ) - @mock.patch("snowflake.cli._plugins.dbt.manager.StageManager.put_recursive") - @mock.patch("snowflake.cli._plugins.dbt.manager.StageManager.create") - def test_execute_in_warehouse( - self, - _mock_create, - _mock_put_recursive, - mock_connect, - runner, - dbt_project_path, - mock_exists, - ): - - result = runner.invoke( - [ - "dbt", - "deploy", - "TEST_PIPELINE", - f"--source={dbt_project_path}", - "--dbt-adapter-version=3.4.5", - "--execute-in-warehouse=XL", - ] - ) - - assert result.exit_code == 0, result.output - assert mock_connect.mocked_ctx.get_query() == dedent( - """CREATE DBT PROJECT TEST_PIPELINE -FROM @MockDatabase.MockSchema.dbt_TEST_PIPELINE_stage -DBT_VERSION='1.2.3' -DBT_ADAPTER_VERSION='3.4.5' -WAREHOUSE='XL'""" - ) - def test_raises_when_dbt_project_is_not_available( self, dbt_project_path, mock_connect, runner ): From 6071c8494942d24ec2de4bc12840e9d9e40d6fe5 Mon Sep 17 00:00:00 2001 From: Jakub Wilkowski Date: Tue, 18 Mar 2025 09:13:03 +0100 Subject: [PATCH 37/54] feat: [SNOW-1890085] add spinner to dbt execute --- src/snowflake/cli/_plugins/dbt/commands.py | 22 ++++++++++++++++------ 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/src/snowflake/cli/_plugins/dbt/commands.py b/src/snowflake/cli/_plugins/dbt/commands.py index 3a81061daa..bc1456fbdc 100644 --- a/src/snowflake/cli/_plugins/dbt/commands.py +++ b/src/snowflake/cli/_plugins/dbt/commands.py @@ -18,6 +18,7 @@ from typing import Optional import typer +from rich.progress import Progress, SpinnerColumn, TextColumn from snowflake.cli._plugins.dbt.constants import DBT_COMMANDS from snowflake.cli._plugins.dbt.manager import DBTManager from snowflake.cli._plugins.object.command_aliases import add_object_command_aliases @@ -137,9 +138,18 @@ def _dbt_execute( dbt_command = ctx.command.name name = ctx.parent.params["name"] run_async = ctx.parent.params["run_async"] - result = DBTManager().execute(dbt_command, name, run_async, *dbt_cli_args) - if not run_async: - return QueryResult(result) - return MessageResult( - f"Command submitted. You can check the result with `snow sql -q \"select execution_status from table(information_schema.query_history_by_user()) where query_id in ('{result.sfqid}');\"`" - ) + execute_args = (dbt_command, name, run_async, *dbt_cli_args) + + if run_async is True: + result = DBTManager().execute(*execute_args) + return MessageResult( + f"Command submitted. You can check the result with `snow sql -q \"select execution_status from table(information_schema.query_history_by_user()) where query_id in ('{result.sfqid}');\"`" + ) + + with Progress( + SpinnerColumn(), + TextColumn("[progress.description]{task.description}"), + transient=True, + ) as progress: + progress.add_task(description="Executing dbt command...", total=None) + return QueryResult(DBTManager().execute(*execute_args)) From 0a6f0d1146df5ec1fd7a3b963c6d961598b2737e Mon Sep 17 00:00:00 2001 From: Jakub Wilkowski Date: Mon, 24 Mar 2025 09:07:16 +0100 Subject: [PATCH 38/54] feat: [SNOW-1890085] limit supported dbt commands --- src/snowflake/cli/_plugins/dbt/constants.py | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/src/snowflake/cli/_plugins/dbt/constants.py b/src/snowflake/cli/_plugins/dbt/constants.py index 4085d7ad0d..ba249f183f 100644 --- a/src/snowflake/cli/_plugins/dbt/constants.py +++ b/src/snowflake/cli/_plugins/dbt/constants.py @@ -1,20 +1,22 @@ DBT_COMMANDS = [ "build", - "clean", - "clone", "compile", - "debug", "deps", - "docs", - "init", "list", "parse", - "retry", - "run", "run", "seed", "show", "snapshot", - "source", "test", ] + +UNSUPPORTED_COMMANDS = [ + "clean", + "clone", + "debug", + "docs", + "init", + "retry", + "source", +] From c8ecab7af31ef100e45f671cb98e7d4e45260ccc Mon Sep 17 00:00:00 2001 From: Jakub Wilkowski Date: Mon, 24 Mar 2025 09:15:03 +0100 Subject: [PATCH 39/54] feat: [SNOW-1890085] remove options for dbt_version and dbt_adapter_version --- src/snowflake/cli/_plugins/dbt/commands.py | 10 ----- src/snowflake/cli/_plugins/dbt/manager.py | 22 +--------- tests/dbt/test_dbt_commands.py | 49 +--------------------- 3 files changed, 4 insertions(+), 77 deletions(-) diff --git a/src/snowflake/cli/_plugins/dbt/commands.py b/src/snowflake/cli/_plugins/dbt/commands.py index bc1456fbdc..be3e574519 100644 --- a/src/snowflake/cli/_plugins/dbt/commands.py +++ b/src/snowflake/cli/_plugins/dbt/commands.py @@ -70,14 +70,6 @@ def deploy_dbt( False, help="Overwrites conflicting files in the project, if any.", ), - dbt_version: Optional[str] = typer.Option( - None, - help="Version of dbt tool to be used. Taken from dbt_project.yml if not provided.", - ), - dbt_adapter_version: Optional[str] = typer.Option( - None, - help="dbt-snowflake adapter version to be used", - ), **options, ) -> CommandResult: """ @@ -91,8 +83,6 @@ def deploy_dbt( DBTManager().deploy( path.resolve(), name, - dbt_version, - dbt_adapter_version, force=force, ) ) diff --git a/src/snowflake/cli/_plugins/dbt/manager.py b/src/snowflake/cli/_plugins/dbt/manager.py index 7c164d842d..adc60fe8b1 100644 --- a/src/snowflake/cli/_plugins/dbt/manager.py +++ b/src/snowflake/cli/_plugins/dbt/manager.py @@ -14,14 +14,11 @@ from __future__ import annotations -from typing import Optional - -import yaml from click import ClickException from snowflake.cli._plugins.object.manager import ObjectManager from snowflake.cli._plugins.stage.manager import StageManager from snowflake.cli.api.console import cli_console -from snowflake.cli.api.constants import DEFAULT_SIZE_LIMIT_MB, ObjectType +from snowflake.cli.api.constants import ObjectType from snowflake.cli.api.identifiers import FQN from snowflake.cli.api.secure_path import SecurePath from snowflake.cli.api.sql_execution import SqlExecutionMixin @@ -43,24 +40,12 @@ def deploy( self, path: SecurePath, name: FQN, - dbt_version: Optional[str], - dbt_adapter_version: str, force: bool, ) -> SnowflakeCursor: dbt_project_path = path / "dbt_project.yml" if not dbt_project_path.exists(): raise ClickException(f"dbt_project.yml does not exist in provided path.") - if dbt_version is None: - with dbt_project_path.open(read_file_limit_mb=DEFAULT_SIZE_LIMIT_MB) as fd: - dbt_project_config = yaml.safe_load(fd) - try: - dbt_version = dbt_project_config["version"] - except (KeyError, TypeError): - raise ClickException( - f"dbt-version was not provided and is not available in dbt_project.yml" - ) - if self.exists(name=name) and force is not True: raise ClickException( f"DBT project {name} already exists. Use --force flag to overwrite" @@ -78,11 +63,8 @@ def deploy( with cli_console.phase("Creating DBT project"): query = f"""{'CREATE OR REPLACE' if force is True else 'CREATE'} DBT PROJECT {name} -FROM {stage_name} -DBT_VERSION='{dbt_version}'""" +FROM {stage_name}""" - if dbt_adapter_version: - query += f"\nDBT_ADAPTER_VERSION='{dbt_adapter_version}'" return self.execute_query(query) def execute(self, dbt_command: str, name: str, run_async: bool, *dbt_cli_args): diff --git a/tests/dbt/test_dbt_commands.py b/tests/dbt/test_dbt_commands.py index fbe79cf8d7..05950173e1 100644 --- a/tests/dbt/test_dbt_commands.py +++ b/tests/dbt/test_dbt_commands.py @@ -17,19 +17,9 @@ from unittest import mock import pytest -import yaml from snowflake.cli.api.identifiers import FQN -@pytest.fixture -def mock_connect(mock_ctx): - with mock.patch("snowflake.connector.connect") as _fixture: - ctx = mock_ctx() - _fixture.return_value = ctx - _fixture.mocked_ctx = _fixture.return_value - yield _fixture - - class TestDBTList: def test_list_command_alias(self, mock_connect, runner): result = runner.invoke( @@ -67,8 +57,6 @@ def dbt_project_path(self, tmp_path_factory): source_path = tmp_path_factory.mktemp("dbt_project") dbt_file = source_path / "dbt_project.yml" dbt_file.touch() - with dbt_file.open(mode="w") as fd: - yaml.dump({"version": "1.2.3"}, fd) yield source_path @pytest.fixture @@ -101,7 +89,6 @@ def test_deploys_project_from_source( "deploy", "TEST_PIPELINE", f"--source={dbt_project_path}", - "--dbt-adapter-version=3.4.5", ] ) @@ -109,9 +96,7 @@ def test_deploys_project_from_source( assert ( mock_connect.mocked_ctx.get_query() == """CREATE DBT PROJECT TEST_PIPELINE -FROM @MockDatabase.MockSchema.dbt_TEST_PIPELINE_stage -DBT_VERSION='1.2.3' -DBT_ADAPTER_VERSION='3.4.5'""" +FROM @MockDatabase.MockSchema.dbt_TEST_PIPELINE_stage""" ) stage_fqn = FQN.from_string(f"dbt_TEST_PIPELINE_stage").using_context() mock_create.assert_called_once_with(stage_fqn, temporary=True) @@ -136,8 +121,6 @@ def test_dbt_version_from_option_has_precedence_over_file( "deploy", "TEST_PIPELINE", f"--source={dbt_project_path}", - "--dbt-version=2.3.4", - "--dbt-adapter-version=3.4.5", ] ) @@ -145,9 +128,7 @@ def test_dbt_version_from_option_has_precedence_over_file( assert ( mock_connect.mocked_ctx.get_query() == """CREATE DBT PROJECT TEST_PIPELINE -FROM @MockDatabase.MockSchema.dbt_TEST_PIPELINE_stage -DBT_VERSION='2.3.4' -DBT_ADAPTER_VERSION='3.4.5'""" +FROM @MockDatabase.MockSchema.dbt_TEST_PIPELINE_stage""" ) @pytest.mark.parametrize("exists", (True, False)) @@ -172,7 +153,6 @@ def test_force_flag_uses_create_or_replace( "TEST_PIPELINE", f"--source={dbt_project_path}", "--force", - "--dbt-adapter-version=3.4.5", ] ) @@ -193,7 +173,6 @@ def test_raises_when_dbt_project_is_not_available( "deploy", "TEST_PIPELINE", f"--source={dbt_project_path}", - "--dbt-adapter-version=3.4.5", ], ) @@ -201,30 +180,6 @@ def test_raises_when_dbt_project_is_not_available( assert "dbt_project.yml does not exist in provided path." in result.output assert mock_connect.mocked_ctx.get_query() == "" - def test_raises_when_dbt_project_version_is_not_specified( - self, dbt_project_path, mock_connect, runner - ): - dbt_file = dbt_project_path / "dbt_project.yml" - with dbt_file.open(mode="w") as fd: - yaml.dump({}, fd) - - result = runner.invoke( - [ - "dbt", - "deploy", - "TEST_PIPELINE", - f"--source={dbt_project_path}", - "--dbt-adapter-version=3.4.5", - ] - ) - - assert result.exit_code == 1, result.output - assert ( - "dbt-version was not provided and is not available in dbt_project.yml" - in result.output - ) - assert mock_connect.mocked_ctx.get_query() == "" - def test_raises_when_dbt_project_exists_and_is_not_force( self, dbt_project_path, mock_connect, runner, mock_exists ): From 0af447c603ef96568065ce6299c992cb0e94abca Mon Sep 17 00:00:00 2001 From: Jakub Wilkowski Date: Mon, 24 Mar 2025 09:24:07 +0100 Subject: [PATCH 40/54] fix: [SNOW-1890085] restore lost changes during rebasing --- tests/conftest.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/conftest.py b/tests/conftest.py index 7ac9811d19..655fc686f2 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -24,7 +24,7 @@ from io import StringIO from logging import FileHandler from pathlib import Path -from typing import Generator, List, NamedTuple, Optional, Union +from typing import Any, Dict, Generator, List, NamedTuple, Optional, Union from unittest import mock import pytest @@ -276,6 +276,7 @@ def __init__( self._checkout_count = 0 self._role = role self._warehouse = warehouse + self.kwargs: List[Dict[str, Any]] = [] def get_query(self): return "\n".join(self.queries) @@ -315,6 +316,7 @@ def execute_string(self, query: str, **kwargs): if self._checkout_count > 1: raise ProgrammingError("Checkout already exists") self.queries.append(query) + self.kwargs.append(kwargs) return (self.cs,) def execute_stream(self, query: StringIO, **kwargs): From 63959379ddaf959ff8ad06e9952b45855bc014df Mon Sep 17 00:00:00 2001 From: Jakub Wilkowski Date: Mon, 24 Mar 2025 09:34:41 +0100 Subject: [PATCH 41/54] test: [SNOW-1890085] unlock dbt help snapshots --- tests/__snapshots__/test_help_messages.ambr | 1532 +++++++++++++++++++ tests/test_help_messages.py | 2 +- 2 files changed, 1533 insertions(+), 1 deletion(-) diff --git a/tests/__snapshots__/test_help_messages.ambr b/tests/__snapshots__/test_help_messages.ambr index ebf0de778c..4a67a3ce64 100644 --- a/tests/__snapshots__/test_help_messages.ambr +++ b/tests/__snapshots__/test_help_messages.ambr @@ -3964,6 +3964,1411 @@ +------------------------------------------------------------------------------+ + ''' +# --- +# name: test_help_messages[dbt.deploy] + ''' + + Usage: root dbt deploy [OPTIONS] NAME + + Copy dbt files and create or update dbt on Snowflake project. + + +- Arguments ------------------------------------------------------------------+ + | * name TEXT Identifier of the DBT Project; for example: my_pipeline | + | [required] | + +------------------------------------------------------------------------------+ + +- Options --------------------------------------------------------------------+ + | --source TEXT Path to directory containing dbt files to | + | deploy. Defaults to current working | + | directory. | + | --force --no-force Overwrites conflicting files in the | + | project, if any. | + | [default: no-force] | + | --help -h Show this message and exit. | + +------------------------------------------------------------------------------+ + +- Connection configuration ---------------------------------------------------+ + | --connection,--environment -c TEXT Name of the connection, as | + | defined in your config.toml | + | file. Default: default. | + | --host TEXT Host address for the | + | connection. Overrides the | + | value specified for the | + | connection. | + | --port INTEGER Port for the connection. | + | Overrides the value | + | specified for the | + | connection. | + | --account,--accountname TEXT Name assigned to your | + | Snowflake account. Overrides | + | the value specified for the | + | connection. | + | --user,--username TEXT Username to connect to | + | Snowflake. Overrides the | + | value specified for the | + | connection. | + | --password TEXT Snowflake password. | + | Overrides the value | + | specified for the | + | connection. | + | --authenticator TEXT Snowflake authenticator. | + | Overrides the value | + | specified for the | + | connection. | + | --private-key-file,--private… TEXT Snowflake private key file | + | path. Overrides the value | + | specified for the | + | connection. | + | --token-file-path TEXT Path to file with an OAuth | + | token that should be used | + | when connecting to Snowflake | + | --database,--dbname TEXT Database to use. Overrides | + | the value specified for the | + | connection. | + | --schema,--schemaname TEXT Database schema to use. | + | Overrides the value | + | specified for the | + | connection. | + | --role,--rolename TEXT Role to use. Overrides the | + | value specified for the | + | connection. | + | --warehouse TEXT Warehouse to use. Overrides | + | the value specified for the | + | connection. | + | --temporary-connection -x Uses a connection defined | + | with command line | + | parameters, instead of one | + | defined in config | + | --mfa-passcode TEXT Token to use for | + | multi-factor authentication | + | (MFA) | + | --enable-diag Whether to generate a | + | connection diagnostic | + | report. | + | --diag-log-path TEXT Path for the generated | + | report. Defaults to system | + | temporary directory. | + | --diag-allowlist-path TEXT Path to a JSON file | + | containing allowlist | + | parameters. | + +------------------------------------------------------------------------------+ + +- Global configuration -------------------------------------------------------+ + | --format [TABLE|JSON] Specifies the output format. | + | [default: TABLE] | + | --verbose -v Displays log entries for log levels info | + | and higher. | + | --debug Displays log entries for log levels debug | + | and higher; debug logs contain additional | + | information. | + | --silent Turns off intermediate output to console. | + +------------------------------------------------------------------------------+ + + + ''' +# --- +# name: test_help_messages[dbt.execute.build] + ''' + + Usage: root dbt execute [OPTIONS] NAME DBT_COMMAND + + Execute a dbt command on Snowflake + + +- Arguments ------------------------------------------------------------------+ + | * name TEXT Identifier of the DBT Project; for example: my_pipeline | + | [required] | + +------------------------------------------------------------------------------+ + +- Options --------------------------------------------------------------------+ + | --run-async --no-run-async Run dbt command asynchronously and | + | check it's result later. | + | [default: no-run-async] | + | --help -h Show this message and exit. | + +------------------------------------------------------------------------------+ + +- Connection configuration ---------------------------------------------------+ + | --connection,--environment -c TEXT Name of the connection, as | + | defined in your config.toml | + | file. Default: default. | + | --host TEXT Host address for the | + | connection. Overrides the | + | value specified for the | + | connection. | + | --port INTEGER Port for the connection. | + | Overrides the value | + | specified for the | + | connection. | + | --account,--accountname TEXT Name assigned to your | + | Snowflake account. Overrides | + | the value specified for the | + | connection. | + | --user,--username TEXT Username to connect to | + | Snowflake. Overrides the | + | value specified for the | + | connection. | + | --password TEXT Snowflake password. | + | Overrides the value | + | specified for the | + | connection. | + | --authenticator TEXT Snowflake authenticator. | + | Overrides the value | + | specified for the | + | connection. | + | --private-key-file,--private… TEXT Snowflake private key file | + | path. Overrides the value | + | specified for the | + | connection. | + | --token-file-path TEXT Path to file with an OAuth | + | token that should be used | + | when connecting to Snowflake | + | --database,--dbname TEXT Database to use. Overrides | + | the value specified for the | + | connection. | + | --schema,--schemaname TEXT Database schema to use. | + | Overrides the value | + | specified for the | + | connection. | + | --role,--rolename TEXT Role to use. Overrides the | + | value specified for the | + | connection. | + | --warehouse TEXT Warehouse to use. Overrides | + | the value specified for the | + | connection. | + | --temporary-connection -x Uses a connection defined | + | with command line | + | parameters, instead of one | + | defined in config | + | --mfa-passcode TEXT Token to use for | + | multi-factor authentication | + | (MFA) | + | --enable-diag Whether to generate a | + | connection diagnostic | + | report. | + | --diag-log-path TEXT Path for the generated | + | report. Defaults to system | + | temporary directory. | + | --diag-allowlist-path TEXT Path to a JSON file | + | containing allowlist | + | parameters. | + +------------------------------------------------------------------------------+ + +- Global configuration -------------------------------------------------------+ + | --format [TABLE|JSON] Specifies the output format. | + | [default: TABLE] | + | --verbose -v Displays log entries for log levels info | + | and higher. | + | --debug Displays log entries for log levels debug | + | and higher; debug logs contain additional | + | information. | + | --silent Turns off intermediate output to console. | + +------------------------------------------------------------------------------+ + +- Commands -------------------------------------------------------------------+ + | build Execute build command on Snowflake. | + | compile Execute compile command on Snowflake. | + | deps Execute deps command on Snowflake. | + | list Execute list command on Snowflake. | + | parse Execute parse command on Snowflake. | + | run Execute run command on Snowflake. | + | seed Execute seed command on Snowflake. | + | show Execute show command on Snowflake. | + | snapshot Execute snapshot command on Snowflake. | + | test Execute test command on Snowflake. | + +------------------------------------------------------------------------------+ + + + ''' +# --- +# name: test_help_messages[dbt.execute.compile] + ''' + + Usage: root dbt execute [OPTIONS] NAME DBT_COMMAND + + Execute a dbt command on Snowflake + + +- Arguments ------------------------------------------------------------------+ + | * name TEXT Identifier of the DBT Project; for example: my_pipeline | + | [required] | + +------------------------------------------------------------------------------+ + +- Options --------------------------------------------------------------------+ + | --run-async --no-run-async Run dbt command asynchronously and | + | check it's result later. | + | [default: no-run-async] | + | --help -h Show this message and exit. | + +------------------------------------------------------------------------------+ + +- Connection configuration ---------------------------------------------------+ + | --connection,--environment -c TEXT Name of the connection, as | + | defined in your config.toml | + | file. Default: default. | + | --host TEXT Host address for the | + | connection. Overrides the | + | value specified for the | + | connection. | + | --port INTEGER Port for the connection. | + | Overrides the value | + | specified for the | + | connection. | + | --account,--accountname TEXT Name assigned to your | + | Snowflake account. Overrides | + | the value specified for the | + | connection. | + | --user,--username TEXT Username to connect to | + | Snowflake. Overrides the | + | value specified for the | + | connection. | + | --password TEXT Snowflake password. | + | Overrides the value | + | specified for the | + | connection. | + | --authenticator TEXT Snowflake authenticator. | + | Overrides the value | + | specified for the | + | connection. | + | --private-key-file,--private… TEXT Snowflake private key file | + | path. Overrides the value | + | specified for the | + | connection. | + | --token-file-path TEXT Path to file with an OAuth | + | token that should be used | + | when connecting to Snowflake | + | --database,--dbname TEXT Database to use. Overrides | + | the value specified for the | + | connection. | + | --schema,--schemaname TEXT Database schema to use. | + | Overrides the value | + | specified for the | + | connection. | + | --role,--rolename TEXT Role to use. Overrides the | + | value specified for the | + | connection. | + | --warehouse TEXT Warehouse to use. Overrides | + | the value specified for the | + | connection. | + | --temporary-connection -x Uses a connection defined | + | with command line | + | parameters, instead of one | + | defined in config | + | --mfa-passcode TEXT Token to use for | + | multi-factor authentication | + | (MFA) | + | --enable-diag Whether to generate a | + | connection diagnostic | + | report. | + | --diag-log-path TEXT Path for the generated | + | report. Defaults to system | + | temporary directory. | + | --diag-allowlist-path TEXT Path to a JSON file | + | containing allowlist | + | parameters. | + +------------------------------------------------------------------------------+ + +- Global configuration -------------------------------------------------------+ + | --format [TABLE|JSON] Specifies the output format. | + | [default: TABLE] | + | --verbose -v Displays log entries for log levels info | + | and higher. | + | --debug Displays log entries for log levels debug | + | and higher; debug logs contain additional | + | information. | + | --silent Turns off intermediate output to console. | + +------------------------------------------------------------------------------+ + +- Commands -------------------------------------------------------------------+ + | build Execute build command on Snowflake. | + | compile Execute compile command on Snowflake. | + | deps Execute deps command on Snowflake. | + | list Execute list command on Snowflake. | + | parse Execute parse command on Snowflake. | + | run Execute run command on Snowflake. | + | seed Execute seed command on Snowflake. | + | show Execute show command on Snowflake. | + | snapshot Execute snapshot command on Snowflake. | + | test Execute test command on Snowflake. | + +------------------------------------------------------------------------------+ + + + ''' +# --- +# name: test_help_messages[dbt.execute.deps] + ''' + + Usage: root dbt execute [OPTIONS] NAME DBT_COMMAND + + Execute a dbt command on Snowflake + + +- Arguments ------------------------------------------------------------------+ + | * name TEXT Identifier of the DBT Project; for example: my_pipeline | + | [required] | + +------------------------------------------------------------------------------+ + +- Options --------------------------------------------------------------------+ + | --run-async --no-run-async Run dbt command asynchronously and | + | check it's result later. | + | [default: no-run-async] | + | --help -h Show this message and exit. | + +------------------------------------------------------------------------------+ + +- Connection configuration ---------------------------------------------------+ + | --connection,--environment -c TEXT Name of the connection, as | + | defined in your config.toml | + | file. Default: default. | + | --host TEXT Host address for the | + | connection. Overrides the | + | value specified for the | + | connection. | + | --port INTEGER Port for the connection. | + | Overrides the value | + | specified for the | + | connection. | + | --account,--accountname TEXT Name assigned to your | + | Snowflake account. Overrides | + | the value specified for the | + | connection. | + | --user,--username TEXT Username to connect to | + | Snowflake. Overrides the | + | value specified for the | + | connection. | + | --password TEXT Snowflake password. | + | Overrides the value | + | specified for the | + | connection. | + | --authenticator TEXT Snowflake authenticator. | + | Overrides the value | + | specified for the | + | connection. | + | --private-key-file,--private… TEXT Snowflake private key file | + | path. Overrides the value | + | specified for the | + | connection. | + | --token-file-path TEXT Path to file with an OAuth | + | token that should be used | + | when connecting to Snowflake | + | --database,--dbname TEXT Database to use. Overrides | + | the value specified for the | + | connection. | + | --schema,--schemaname TEXT Database schema to use. | + | Overrides the value | + | specified for the | + | connection. | + | --role,--rolename TEXT Role to use. Overrides the | + | value specified for the | + | connection. | + | --warehouse TEXT Warehouse to use. Overrides | + | the value specified for the | + | connection. | + | --temporary-connection -x Uses a connection defined | + | with command line | + | parameters, instead of one | + | defined in config | + | --mfa-passcode TEXT Token to use for | + | multi-factor authentication | + | (MFA) | + | --enable-diag Whether to generate a | + | connection diagnostic | + | report. | + | --diag-log-path TEXT Path for the generated | + | report. Defaults to system | + | temporary directory. | + | --diag-allowlist-path TEXT Path to a JSON file | + | containing allowlist | + | parameters. | + +------------------------------------------------------------------------------+ + +- Global configuration -------------------------------------------------------+ + | --format [TABLE|JSON] Specifies the output format. | + | [default: TABLE] | + | --verbose -v Displays log entries for log levels info | + | and higher. | + | --debug Displays log entries for log levels debug | + | and higher; debug logs contain additional | + | information. | + | --silent Turns off intermediate output to console. | + +------------------------------------------------------------------------------+ + +- Commands -------------------------------------------------------------------+ + | build Execute build command on Snowflake. | + | compile Execute compile command on Snowflake. | + | deps Execute deps command on Snowflake. | + | list Execute list command on Snowflake. | + | parse Execute parse command on Snowflake. | + | run Execute run command on Snowflake. | + | seed Execute seed command on Snowflake. | + | show Execute show command on Snowflake. | + | snapshot Execute snapshot command on Snowflake. | + | test Execute test command on Snowflake. | + +------------------------------------------------------------------------------+ + + + ''' +# --- +# name: test_help_messages[dbt.execute.list] + ''' + + Usage: root dbt execute [OPTIONS] NAME DBT_COMMAND + + Execute a dbt command on Snowflake + + +- Arguments ------------------------------------------------------------------+ + | * name TEXT Identifier of the DBT Project; for example: my_pipeline | + | [required] | + +------------------------------------------------------------------------------+ + +- Options --------------------------------------------------------------------+ + | --run-async --no-run-async Run dbt command asynchronously and | + | check it's result later. | + | [default: no-run-async] | + | --help -h Show this message and exit. | + +------------------------------------------------------------------------------+ + +- Connection configuration ---------------------------------------------------+ + | --connection,--environment -c TEXT Name of the connection, as | + | defined in your config.toml | + | file. Default: default. | + | --host TEXT Host address for the | + | connection. Overrides the | + | value specified for the | + | connection. | + | --port INTEGER Port for the connection. | + | Overrides the value | + | specified for the | + | connection. | + | --account,--accountname TEXT Name assigned to your | + | Snowflake account. Overrides | + | the value specified for the | + | connection. | + | --user,--username TEXT Username to connect to | + | Snowflake. Overrides the | + | value specified for the | + | connection. | + | --password TEXT Snowflake password. | + | Overrides the value | + | specified for the | + | connection. | + | --authenticator TEXT Snowflake authenticator. | + | Overrides the value | + | specified for the | + | connection. | + | --private-key-file,--private… TEXT Snowflake private key file | + | path. Overrides the value | + | specified for the | + | connection. | + | --token-file-path TEXT Path to file with an OAuth | + | token that should be used | + | when connecting to Snowflake | + | --database,--dbname TEXT Database to use. Overrides | + | the value specified for the | + | connection. | + | --schema,--schemaname TEXT Database schema to use. | + | Overrides the value | + | specified for the | + | connection. | + | --role,--rolename TEXT Role to use. Overrides the | + | value specified for the | + | connection. | + | --warehouse TEXT Warehouse to use. Overrides | + | the value specified for the | + | connection. | + | --temporary-connection -x Uses a connection defined | + | with command line | + | parameters, instead of one | + | defined in config | + | --mfa-passcode TEXT Token to use for | + | multi-factor authentication | + | (MFA) | + | --enable-diag Whether to generate a | + | connection diagnostic | + | report. | + | --diag-log-path TEXT Path for the generated | + | report. Defaults to system | + | temporary directory. | + | --diag-allowlist-path TEXT Path to a JSON file | + | containing allowlist | + | parameters. | + +------------------------------------------------------------------------------+ + +- Global configuration -------------------------------------------------------+ + | --format [TABLE|JSON] Specifies the output format. | + | [default: TABLE] | + | --verbose -v Displays log entries for log levels info | + | and higher. | + | --debug Displays log entries for log levels debug | + | and higher; debug logs contain additional | + | information. | + | --silent Turns off intermediate output to console. | + +------------------------------------------------------------------------------+ + +- Commands -------------------------------------------------------------------+ + | build Execute build command on Snowflake. | + | compile Execute compile command on Snowflake. | + | deps Execute deps command on Snowflake. | + | list Execute list command on Snowflake. | + | parse Execute parse command on Snowflake. | + | run Execute run command on Snowflake. | + | seed Execute seed command on Snowflake. | + | show Execute show command on Snowflake. | + | snapshot Execute snapshot command on Snowflake. | + | test Execute test command on Snowflake. | + +------------------------------------------------------------------------------+ + + + ''' +# --- +# name: test_help_messages[dbt.execute.parse] + ''' + + Usage: root dbt execute [OPTIONS] NAME DBT_COMMAND + + Execute a dbt command on Snowflake + + +- Arguments ------------------------------------------------------------------+ + | * name TEXT Identifier of the DBT Project; for example: my_pipeline | + | [required] | + +------------------------------------------------------------------------------+ + +- Options --------------------------------------------------------------------+ + | --run-async --no-run-async Run dbt command asynchronously and | + | check it's result later. | + | [default: no-run-async] | + | --help -h Show this message and exit. | + +------------------------------------------------------------------------------+ + +- Connection configuration ---------------------------------------------------+ + | --connection,--environment -c TEXT Name of the connection, as | + | defined in your config.toml | + | file. Default: default. | + | --host TEXT Host address for the | + | connection. Overrides the | + | value specified for the | + | connection. | + | --port INTEGER Port for the connection. | + | Overrides the value | + | specified for the | + | connection. | + | --account,--accountname TEXT Name assigned to your | + | Snowflake account. Overrides | + | the value specified for the | + | connection. | + | --user,--username TEXT Username to connect to | + | Snowflake. Overrides the | + | value specified for the | + | connection. | + | --password TEXT Snowflake password. | + | Overrides the value | + | specified for the | + | connection. | + | --authenticator TEXT Snowflake authenticator. | + | Overrides the value | + | specified for the | + | connection. | + | --private-key-file,--private… TEXT Snowflake private key file | + | path. Overrides the value | + | specified for the | + | connection. | + | --token-file-path TEXT Path to file with an OAuth | + | token that should be used | + | when connecting to Snowflake | + | --database,--dbname TEXT Database to use. Overrides | + | the value specified for the | + | connection. | + | --schema,--schemaname TEXT Database schema to use. | + | Overrides the value | + | specified for the | + | connection. | + | --role,--rolename TEXT Role to use. Overrides the | + | value specified for the | + | connection. | + | --warehouse TEXT Warehouse to use. Overrides | + | the value specified for the | + | connection. | + | --temporary-connection -x Uses a connection defined | + | with command line | + | parameters, instead of one | + | defined in config | + | --mfa-passcode TEXT Token to use for | + | multi-factor authentication | + | (MFA) | + | --enable-diag Whether to generate a | + | connection diagnostic | + | report. | + | --diag-log-path TEXT Path for the generated | + | report. Defaults to system | + | temporary directory. | + | --diag-allowlist-path TEXT Path to a JSON file | + | containing allowlist | + | parameters. | + +------------------------------------------------------------------------------+ + +- Global configuration -------------------------------------------------------+ + | --format [TABLE|JSON] Specifies the output format. | + | [default: TABLE] | + | --verbose -v Displays log entries for log levels info | + | and higher. | + | --debug Displays log entries for log levels debug | + | and higher; debug logs contain additional | + | information. | + | --silent Turns off intermediate output to console. | + +------------------------------------------------------------------------------+ + +- Commands -------------------------------------------------------------------+ + | build Execute build command on Snowflake. | + | compile Execute compile command on Snowflake. | + | deps Execute deps command on Snowflake. | + | list Execute list command on Snowflake. | + | parse Execute parse command on Snowflake. | + | run Execute run command on Snowflake. | + | seed Execute seed command on Snowflake. | + | show Execute show command on Snowflake. | + | snapshot Execute snapshot command on Snowflake. | + | test Execute test command on Snowflake. | + +------------------------------------------------------------------------------+ + + + ''' +# --- +# name: test_help_messages[dbt.execute.run] + ''' + + Usage: root dbt execute [OPTIONS] NAME DBT_COMMAND + + Execute a dbt command on Snowflake + + +- Arguments ------------------------------------------------------------------+ + | * name TEXT Identifier of the DBT Project; for example: my_pipeline | + | [required] | + +------------------------------------------------------------------------------+ + +- Options --------------------------------------------------------------------+ + | --run-async --no-run-async Run dbt command asynchronously and | + | check it's result later. | + | [default: no-run-async] | + | --help -h Show this message and exit. | + +------------------------------------------------------------------------------+ + +- Connection configuration ---------------------------------------------------+ + | --connection,--environment -c TEXT Name of the connection, as | + | defined in your config.toml | + | file. Default: default. | + | --host TEXT Host address for the | + | connection. Overrides the | + | value specified for the | + | connection. | + | --port INTEGER Port for the connection. | + | Overrides the value | + | specified for the | + | connection. | + | --account,--accountname TEXT Name assigned to your | + | Snowflake account. Overrides | + | the value specified for the | + | connection. | + | --user,--username TEXT Username to connect to | + | Snowflake. Overrides the | + | value specified for the | + | connection. | + | --password TEXT Snowflake password. | + | Overrides the value | + | specified for the | + | connection. | + | --authenticator TEXT Snowflake authenticator. | + | Overrides the value | + | specified for the | + | connection. | + | --private-key-file,--private… TEXT Snowflake private key file | + | path. Overrides the value | + | specified for the | + | connection. | + | --token-file-path TEXT Path to file with an OAuth | + | token that should be used | + | when connecting to Snowflake | + | --database,--dbname TEXT Database to use. Overrides | + | the value specified for the | + | connection. | + | --schema,--schemaname TEXT Database schema to use. | + | Overrides the value | + | specified for the | + | connection. | + | --role,--rolename TEXT Role to use. Overrides the | + | value specified for the | + | connection. | + | --warehouse TEXT Warehouse to use. Overrides | + | the value specified for the | + | connection. | + | --temporary-connection -x Uses a connection defined | + | with command line | + | parameters, instead of one | + | defined in config | + | --mfa-passcode TEXT Token to use for | + | multi-factor authentication | + | (MFA) | + | --enable-diag Whether to generate a | + | connection diagnostic | + | report. | + | --diag-log-path TEXT Path for the generated | + | report. Defaults to system | + | temporary directory. | + | --diag-allowlist-path TEXT Path to a JSON file | + | containing allowlist | + | parameters. | + +------------------------------------------------------------------------------+ + +- Global configuration -------------------------------------------------------+ + | --format [TABLE|JSON] Specifies the output format. | + | [default: TABLE] | + | --verbose -v Displays log entries for log levels info | + | and higher. | + | --debug Displays log entries for log levels debug | + | and higher; debug logs contain additional | + | information. | + | --silent Turns off intermediate output to console. | + +------------------------------------------------------------------------------+ + +- Commands -------------------------------------------------------------------+ + | build Execute build command on Snowflake. | + | compile Execute compile command on Snowflake. | + | deps Execute deps command on Snowflake. | + | list Execute list command on Snowflake. | + | parse Execute parse command on Snowflake. | + | run Execute run command on Snowflake. | + | seed Execute seed command on Snowflake. | + | show Execute show command on Snowflake. | + | snapshot Execute snapshot command on Snowflake. | + | test Execute test command on Snowflake. | + +------------------------------------------------------------------------------+ + + + ''' +# --- +# name: test_help_messages[dbt.execute.seed] + ''' + + Usage: root dbt execute [OPTIONS] NAME DBT_COMMAND + + Execute a dbt command on Snowflake + + +- Arguments ------------------------------------------------------------------+ + | * name TEXT Identifier of the DBT Project; for example: my_pipeline | + | [required] | + +------------------------------------------------------------------------------+ + +- Options --------------------------------------------------------------------+ + | --run-async --no-run-async Run dbt command asynchronously and | + | check it's result later. | + | [default: no-run-async] | + | --help -h Show this message and exit. | + +------------------------------------------------------------------------------+ + +- Connection configuration ---------------------------------------------------+ + | --connection,--environment -c TEXT Name of the connection, as | + | defined in your config.toml | + | file. Default: default. | + | --host TEXT Host address for the | + | connection. Overrides the | + | value specified for the | + | connection. | + | --port INTEGER Port for the connection. | + | Overrides the value | + | specified for the | + | connection. | + | --account,--accountname TEXT Name assigned to your | + | Snowflake account. Overrides | + | the value specified for the | + | connection. | + | --user,--username TEXT Username to connect to | + | Snowflake. Overrides the | + | value specified for the | + | connection. | + | --password TEXT Snowflake password. | + | Overrides the value | + | specified for the | + | connection. | + | --authenticator TEXT Snowflake authenticator. | + | Overrides the value | + | specified for the | + | connection. | + | --private-key-file,--private… TEXT Snowflake private key file | + | path. Overrides the value | + | specified for the | + | connection. | + | --token-file-path TEXT Path to file with an OAuth | + | token that should be used | + | when connecting to Snowflake | + | --database,--dbname TEXT Database to use. Overrides | + | the value specified for the | + | connection. | + | --schema,--schemaname TEXT Database schema to use. | + | Overrides the value | + | specified for the | + | connection. | + | --role,--rolename TEXT Role to use. Overrides the | + | value specified for the | + | connection. | + | --warehouse TEXT Warehouse to use. Overrides | + | the value specified for the | + | connection. | + | --temporary-connection -x Uses a connection defined | + | with command line | + | parameters, instead of one | + | defined in config | + | --mfa-passcode TEXT Token to use for | + | multi-factor authentication | + | (MFA) | + | --enable-diag Whether to generate a | + | connection diagnostic | + | report. | + | --diag-log-path TEXT Path for the generated | + | report. Defaults to system | + | temporary directory. | + | --diag-allowlist-path TEXT Path to a JSON file | + | containing allowlist | + | parameters. | + +------------------------------------------------------------------------------+ + +- Global configuration -------------------------------------------------------+ + | --format [TABLE|JSON] Specifies the output format. | + | [default: TABLE] | + | --verbose -v Displays log entries for log levels info | + | and higher. | + | --debug Displays log entries for log levels debug | + | and higher; debug logs contain additional | + | information. | + | --silent Turns off intermediate output to console. | + +------------------------------------------------------------------------------+ + +- Commands -------------------------------------------------------------------+ + | build Execute build command on Snowflake. | + | compile Execute compile command on Snowflake. | + | deps Execute deps command on Snowflake. | + | list Execute list command on Snowflake. | + | parse Execute parse command on Snowflake. | + | run Execute run command on Snowflake. | + | seed Execute seed command on Snowflake. | + | show Execute show command on Snowflake. | + | snapshot Execute snapshot command on Snowflake. | + | test Execute test command on Snowflake. | + +------------------------------------------------------------------------------+ + + + ''' +# --- +# name: test_help_messages[dbt.execute.show] + ''' + + Usage: root dbt execute [OPTIONS] NAME DBT_COMMAND + + Execute a dbt command on Snowflake + + +- Arguments ------------------------------------------------------------------+ + | * name TEXT Identifier of the DBT Project; for example: my_pipeline | + | [required] | + +------------------------------------------------------------------------------+ + +- Options --------------------------------------------------------------------+ + | --run-async --no-run-async Run dbt command asynchronously and | + | check it's result later. | + | [default: no-run-async] | + | --help -h Show this message and exit. | + +------------------------------------------------------------------------------+ + +- Connection configuration ---------------------------------------------------+ + | --connection,--environment -c TEXT Name of the connection, as | + | defined in your config.toml | + | file. Default: default. | + | --host TEXT Host address for the | + | connection. Overrides the | + | value specified for the | + | connection. | + | --port INTEGER Port for the connection. | + | Overrides the value | + | specified for the | + | connection. | + | --account,--accountname TEXT Name assigned to your | + | Snowflake account. Overrides | + | the value specified for the | + | connection. | + | --user,--username TEXT Username to connect to | + | Snowflake. Overrides the | + | value specified for the | + | connection. | + | --password TEXT Snowflake password. | + | Overrides the value | + | specified for the | + | connection. | + | --authenticator TEXT Snowflake authenticator. | + | Overrides the value | + | specified for the | + | connection. | + | --private-key-file,--private… TEXT Snowflake private key file | + | path. Overrides the value | + | specified for the | + | connection. | + | --token-file-path TEXT Path to file with an OAuth | + | token that should be used | + | when connecting to Snowflake | + | --database,--dbname TEXT Database to use. Overrides | + | the value specified for the | + | connection. | + | --schema,--schemaname TEXT Database schema to use. | + | Overrides the value | + | specified for the | + | connection. | + | --role,--rolename TEXT Role to use. Overrides the | + | value specified for the | + | connection. | + | --warehouse TEXT Warehouse to use. Overrides | + | the value specified for the | + | connection. | + | --temporary-connection -x Uses a connection defined | + | with command line | + | parameters, instead of one | + | defined in config | + | --mfa-passcode TEXT Token to use for | + | multi-factor authentication | + | (MFA) | + | --enable-diag Whether to generate a | + | connection diagnostic | + | report. | + | --diag-log-path TEXT Path for the generated | + | report. Defaults to system | + | temporary directory. | + | --diag-allowlist-path TEXT Path to a JSON file | + | containing allowlist | + | parameters. | + +------------------------------------------------------------------------------+ + +- Global configuration -------------------------------------------------------+ + | --format [TABLE|JSON] Specifies the output format. | + | [default: TABLE] | + | --verbose -v Displays log entries for log levels info | + | and higher. | + | --debug Displays log entries for log levels debug | + | and higher; debug logs contain additional | + | information. | + | --silent Turns off intermediate output to console. | + +------------------------------------------------------------------------------+ + +- Commands -------------------------------------------------------------------+ + | build Execute build command on Snowflake. | + | compile Execute compile command on Snowflake. | + | deps Execute deps command on Snowflake. | + | list Execute list command on Snowflake. | + | parse Execute parse command on Snowflake. | + | run Execute run command on Snowflake. | + | seed Execute seed command on Snowflake. | + | show Execute show command on Snowflake. | + | snapshot Execute snapshot command on Snowflake. | + | test Execute test command on Snowflake. | + +------------------------------------------------------------------------------+ + + + ''' +# --- +# name: test_help_messages[dbt.execute.snapshot] + ''' + + Usage: root dbt execute [OPTIONS] NAME DBT_COMMAND + + Execute a dbt command on Snowflake + + +- Arguments ------------------------------------------------------------------+ + | * name TEXT Identifier of the DBT Project; for example: my_pipeline | + | [required] | + +------------------------------------------------------------------------------+ + +- Options --------------------------------------------------------------------+ + | --run-async --no-run-async Run dbt command asynchronously and | + | check it's result later. | + | [default: no-run-async] | + | --help -h Show this message and exit. | + +------------------------------------------------------------------------------+ + +- Connection configuration ---------------------------------------------------+ + | --connection,--environment -c TEXT Name of the connection, as | + | defined in your config.toml | + | file. Default: default. | + | --host TEXT Host address for the | + | connection. Overrides the | + | value specified for the | + | connection. | + | --port INTEGER Port for the connection. | + | Overrides the value | + | specified for the | + | connection. | + | --account,--accountname TEXT Name assigned to your | + | Snowflake account. Overrides | + | the value specified for the | + | connection. | + | --user,--username TEXT Username to connect to | + | Snowflake. Overrides the | + | value specified for the | + | connection. | + | --password TEXT Snowflake password. | + | Overrides the value | + | specified for the | + | connection. | + | --authenticator TEXT Snowflake authenticator. | + | Overrides the value | + | specified for the | + | connection. | + | --private-key-file,--private… TEXT Snowflake private key file | + | path. Overrides the value | + | specified for the | + | connection. | + | --token-file-path TEXT Path to file with an OAuth | + | token that should be used | + | when connecting to Snowflake | + | --database,--dbname TEXT Database to use. Overrides | + | the value specified for the | + | connection. | + | --schema,--schemaname TEXT Database schema to use. | + | Overrides the value | + | specified for the | + | connection. | + | --role,--rolename TEXT Role to use. Overrides the | + | value specified for the | + | connection. | + | --warehouse TEXT Warehouse to use. Overrides | + | the value specified for the | + | connection. | + | --temporary-connection -x Uses a connection defined | + | with command line | + | parameters, instead of one | + | defined in config | + | --mfa-passcode TEXT Token to use for | + | multi-factor authentication | + | (MFA) | + | --enable-diag Whether to generate a | + | connection diagnostic | + | report. | + | --diag-log-path TEXT Path for the generated | + | report. Defaults to system | + | temporary directory. | + | --diag-allowlist-path TEXT Path to a JSON file | + | containing allowlist | + | parameters. | + +------------------------------------------------------------------------------+ + +- Global configuration -------------------------------------------------------+ + | --format [TABLE|JSON] Specifies the output format. | + | [default: TABLE] | + | --verbose -v Displays log entries for log levels info | + | and higher. | + | --debug Displays log entries for log levels debug | + | and higher; debug logs contain additional | + | information. | + | --silent Turns off intermediate output to console. | + +------------------------------------------------------------------------------+ + +- Commands -------------------------------------------------------------------+ + | build Execute build command on Snowflake. | + | compile Execute compile command on Snowflake. | + | deps Execute deps command on Snowflake. | + | list Execute list command on Snowflake. | + | parse Execute parse command on Snowflake. | + | run Execute run command on Snowflake. | + | seed Execute seed command on Snowflake. | + | show Execute show command on Snowflake. | + | snapshot Execute snapshot command on Snowflake. | + | test Execute test command on Snowflake. | + +------------------------------------------------------------------------------+ + + + ''' +# --- +# name: test_help_messages[dbt.execute.test] + ''' + + Usage: root dbt execute [OPTIONS] NAME DBT_COMMAND + + Execute a dbt command on Snowflake + + +- Arguments ------------------------------------------------------------------+ + | * name TEXT Identifier of the DBT Project; for example: my_pipeline | + | [required] | + +------------------------------------------------------------------------------+ + +- Options --------------------------------------------------------------------+ + | --run-async --no-run-async Run dbt command asynchronously and | + | check it's result later. | + | [default: no-run-async] | + | --help -h Show this message and exit. | + +------------------------------------------------------------------------------+ + +- Connection configuration ---------------------------------------------------+ + | --connection,--environment -c TEXT Name of the connection, as | + | defined in your config.toml | + | file. Default: default. | + | --host TEXT Host address for the | + | connection. Overrides the | + | value specified for the | + | connection. | + | --port INTEGER Port for the connection. | + | Overrides the value | + | specified for the | + | connection. | + | --account,--accountname TEXT Name assigned to your | + | Snowflake account. Overrides | + | the value specified for the | + | connection. | + | --user,--username TEXT Username to connect to | + | Snowflake. Overrides the | + | value specified for the | + | connection. | + | --password TEXT Snowflake password. | + | Overrides the value | + | specified for the | + | connection. | + | --authenticator TEXT Snowflake authenticator. | + | Overrides the value | + | specified for the | + | connection. | + | --private-key-file,--private… TEXT Snowflake private key file | + | path. Overrides the value | + | specified for the | + | connection. | + | --token-file-path TEXT Path to file with an OAuth | + | token that should be used | + | when connecting to Snowflake | + | --database,--dbname TEXT Database to use. Overrides | + | the value specified for the | + | connection. | + | --schema,--schemaname TEXT Database schema to use. | + | Overrides the value | + | specified for the | + | connection. | + | --role,--rolename TEXT Role to use. Overrides the | + | value specified for the | + | connection. | + | --warehouse TEXT Warehouse to use. Overrides | + | the value specified for the | + | connection. | + | --temporary-connection -x Uses a connection defined | + | with command line | + | parameters, instead of one | + | defined in config | + | --mfa-passcode TEXT Token to use for | + | multi-factor authentication | + | (MFA) | + | --enable-diag Whether to generate a | + | connection diagnostic | + | report. | + | --diag-log-path TEXT Path for the generated | + | report. Defaults to system | + | temporary directory. | + | --diag-allowlist-path TEXT Path to a JSON file | + | containing allowlist | + | parameters. | + +------------------------------------------------------------------------------+ + +- Global configuration -------------------------------------------------------+ + | --format [TABLE|JSON] Specifies the output format. | + | [default: TABLE] | + | --verbose -v Displays log entries for log levels info | + | and higher. | + | --debug Displays log entries for log levels debug | + | and higher; debug logs contain additional | + | information. | + | --silent Turns off intermediate output to console. | + +------------------------------------------------------------------------------+ + +- Commands -------------------------------------------------------------------+ + | build Execute build command on Snowflake. | + | compile Execute compile command on Snowflake. | + | deps Execute deps command on Snowflake. | + | list Execute list command on Snowflake. | + | parse Execute parse command on Snowflake. | + | run Execute run command on Snowflake. | + | seed Execute seed command on Snowflake. | + | show Execute show command on Snowflake. | + | snapshot Execute snapshot command on Snowflake. | + | test Execute test command on Snowflake. | + +------------------------------------------------------------------------------+ + + + ''' +# --- +# name: test_help_messages[dbt.execute] + ''' + + Usage: root dbt execute [OPTIONS] NAME DBT_COMMAND + + Execute a dbt command on Snowflake + + +- Arguments ------------------------------------------------------------------+ + | * name TEXT Identifier of the DBT Project; for example: my_pipeline | + | [required] | + +------------------------------------------------------------------------------+ + +- Options --------------------------------------------------------------------+ + | --run-async --no-run-async Run dbt command asynchronously and | + | check it's result later. | + | [default: no-run-async] | + | --help -h Show this message and exit. | + +------------------------------------------------------------------------------+ + +- Connection configuration ---------------------------------------------------+ + | --connection,--environment -c TEXT Name of the connection, as | + | defined in your config.toml | + | file. Default: default. | + | --host TEXT Host address for the | + | connection. Overrides the | + | value specified for the | + | connection. | + | --port INTEGER Port for the connection. | + | Overrides the value | + | specified for the | + | connection. | + | --account,--accountname TEXT Name assigned to your | + | Snowflake account. Overrides | + | the value specified for the | + | connection. | + | --user,--username TEXT Username to connect to | + | Snowflake. Overrides the | + | value specified for the | + | connection. | + | --password TEXT Snowflake password. | + | Overrides the value | + | specified for the | + | connection. | + | --authenticator TEXT Snowflake authenticator. | + | Overrides the value | + | specified for the | + | connection. | + | --private-key-file,--private… TEXT Snowflake private key file | + | path. Overrides the value | + | specified for the | + | connection. | + | --token-file-path TEXT Path to file with an OAuth | + | token that should be used | + | when connecting to Snowflake | + | --database,--dbname TEXT Database to use. Overrides | + | the value specified for the | + | connection. | + | --schema,--schemaname TEXT Database schema to use. | + | Overrides the value | + | specified for the | + | connection. | + | --role,--rolename TEXT Role to use. Overrides the | + | value specified for the | + | connection. | + | --warehouse TEXT Warehouse to use. Overrides | + | the value specified for the | + | connection. | + | --temporary-connection -x Uses a connection defined | + | with command line | + | parameters, instead of one | + | defined in config | + | --mfa-passcode TEXT Token to use for | + | multi-factor authentication | + | (MFA) | + | --enable-diag Whether to generate a | + | connection diagnostic | + | report. | + | --diag-log-path TEXT Path for the generated | + | report. Defaults to system | + | temporary directory. | + | --diag-allowlist-path TEXT Path to a JSON file | + | containing allowlist | + | parameters. | + +------------------------------------------------------------------------------+ + +- Global configuration -------------------------------------------------------+ + | --format [TABLE|JSON] Specifies the output format. | + | [default: TABLE] | + | --verbose -v Displays log entries for log levels info | + | and higher. | + | --debug Displays log entries for log levels debug | + | and higher; debug logs contain additional | + | information. | + | --silent Turns off intermediate output to console. | + +------------------------------------------------------------------------------+ + +- Commands -------------------------------------------------------------------+ + | build Execute build command on Snowflake. | + | compile Execute compile command on Snowflake. | + | deps Execute deps command on Snowflake. | + | list Execute list command on Snowflake. | + | parse Execute parse command on Snowflake. | + | run Execute run command on Snowflake. | + | seed Execute seed command on Snowflake. | + | show Execute show command on Snowflake. | + | snapshot Execute snapshot command on Snowflake. | + | test Execute test command on Snowflake. | + +------------------------------------------------------------------------------+ + + + ''' +# --- +# name: test_help_messages[dbt.list] + ''' + + Usage: root dbt list [OPTIONS] + + Lists all available dbt projects. + + +- Options --------------------------------------------------------------------+ + | --like -l TEXT SQL LIKE pattern for filtering objects by | + | name. For example, list --like "my%" lists | + | all dbt projects that begin with “my”. | + | [default: %%] | + | --in ... | + | | + | Specifies the scope of this command using | + | '--in ', for example list --in database | + | my_db. | + | [default: None, None] | + | --help -h Show this message and exit. | + +------------------------------------------------------------------------------+ + +- Connection configuration ---------------------------------------------------+ + | --connection,--environment -c TEXT Name of the connection, as | + | defined in your config.toml | + | file. Default: default. | + | --host TEXT Host address for the | + | connection. Overrides the | + | value specified for the | + | connection. | + | --port INTEGER Port for the connection. | + | Overrides the value | + | specified for the | + | connection. | + | --account,--accountname TEXT Name assigned to your | + | Snowflake account. Overrides | + | the value specified for the | + | connection. | + | --user,--username TEXT Username to connect to | + | Snowflake. Overrides the | + | value specified for the | + | connection. | + | --password TEXT Snowflake password. | + | Overrides the value | + | specified for the | + | connection. | + | --authenticator TEXT Snowflake authenticator. | + | Overrides the value | + | specified for the | + | connection. | + | --private-key-file,--private… TEXT Snowflake private key file | + | path. Overrides the value | + | specified for the | + | connection. | + | --token-file-path TEXT Path to file with an OAuth | + | token that should be used | + | when connecting to Snowflake | + | --database,--dbname TEXT Database to use. Overrides | + | the value specified for the | + | connection. | + | --schema,--schemaname TEXT Database schema to use. | + | Overrides the value | + | specified for the | + | connection. | + | --role,--rolename TEXT Role to use. Overrides the | + | value specified for the | + | connection. | + | --warehouse TEXT Warehouse to use. Overrides | + | the value specified for the | + | connection. | + | --temporary-connection -x Uses a connection defined | + | with command line | + | parameters, instead of one | + | defined in config | + | --mfa-passcode TEXT Token to use for | + | multi-factor authentication | + | (MFA) | + | --enable-diag Whether to generate a | + | connection diagnostic | + | report. | + | --diag-log-path TEXT Path for the generated | + | report. Defaults to system | + | temporary directory. | + | --diag-allowlist-path TEXT Path to a JSON file | + | containing allowlist | + | parameters. | + +------------------------------------------------------------------------------+ + +- Global configuration -------------------------------------------------------+ + | --format [TABLE|JSON] Specifies the output format. | + | [default: TABLE] | + | --verbose -v Displays log entries for log levels info | + | and higher. | + | --debug Displays log entries for log levels debug | + | and higher; debug logs contain additional | + | information. | + | --silent Turns off intermediate output to console. | + +------------------------------------------------------------------------------+ + + + ''' +# --- +# name: test_help_messages[dbt] + ''' + + Usage: root dbt [OPTIONS] COMMAND [ARGS]... + + Manages dbt on Snowflake projects + + +- Options --------------------------------------------------------------------+ + | --help -h Show this message and exit. | + +------------------------------------------------------------------------------+ + +- Commands -------------------------------------------------------------------+ + | deploy Copy dbt files and create or update dbt on Snowflake project. | + | execute Execute a dbt command on Snowflake | + | list Lists all available dbt projects. | + +------------------------------------------------------------------------------+ + + ''' # --- # name: test_help_messages[git.copy] @@ -14394,6 +15799,133 @@ +------------------------------------------------------------------------------+ + ''' +# --- +# name: test_help_messages_no_help_flag[dbt.execute] + ''' + + Usage: root dbt execute [OPTIONS] NAME DBT_COMMAND + + Execute a dbt command on Snowflake + + +- Arguments ------------------------------------------------------------------+ + | * name TEXT Identifier of the DBT Project; for example: my_pipeline | + | [required] | + +------------------------------------------------------------------------------+ + +- Options --------------------------------------------------------------------+ + | --run-async --no-run-async Run dbt command asynchronously and | + | check it's result later. | + | [default: no-run-async] | + | --help -h Show this message and exit. | + +------------------------------------------------------------------------------+ + +- Connection configuration ---------------------------------------------------+ + | --connection,--environment -c TEXT Name of the connection, as | + | defined in your config.toml | + | file. Default: default. | + | --host TEXT Host address for the | + | connection. Overrides the | + | value specified for the | + | connection. | + | --port INTEGER Port for the connection. | + | Overrides the value | + | specified for the | + | connection. | + | --account,--accountname TEXT Name assigned to your | + | Snowflake account. Overrides | + | the value specified for the | + | connection. | + | --user,--username TEXT Username to connect to | + | Snowflake. Overrides the | + | value specified for the | + | connection. | + | --password TEXT Snowflake password. | + | Overrides the value | + | specified for the | + | connection. | + | --authenticator TEXT Snowflake authenticator. | + | Overrides the value | + | specified for the | + | connection. | + | --private-key-file,--private… TEXT Snowflake private key file | + | path. Overrides the value | + | specified for the | + | connection. | + | --token-file-path TEXT Path to file with an OAuth | + | token that should be used | + | when connecting to Snowflake | + | --database,--dbname TEXT Database to use. Overrides | + | the value specified for the | + | connection. | + | --schema,--schemaname TEXT Database schema to use. | + | Overrides the value | + | specified for the | + | connection. | + | --role,--rolename TEXT Role to use. Overrides the | + | value specified for the | + | connection. | + | --warehouse TEXT Warehouse to use. Overrides | + | the value specified for the | + | connection. | + | --temporary-connection -x Uses a connection defined | + | with command line | + | parameters, instead of one | + | defined in config | + | --mfa-passcode TEXT Token to use for | + | multi-factor authentication | + | (MFA) | + | --enable-diag Whether to generate a | + | connection diagnostic | + | report. | + | --diag-log-path TEXT Path for the generated | + | report. Defaults to system | + | temporary directory. | + | --diag-allowlist-path TEXT Path to a JSON file | + | containing allowlist | + | parameters. | + +------------------------------------------------------------------------------+ + +- Global configuration -------------------------------------------------------+ + | --format [TABLE|JSON] Specifies the output format. | + | [default: TABLE] | + | --verbose -v Displays log entries for log levels info | + | and higher. | + | --debug Displays log entries for log levels debug | + | and higher; debug logs contain additional | + | information. | + | --silent Turns off intermediate output to console. | + +------------------------------------------------------------------------------+ + +- Commands -------------------------------------------------------------------+ + | build Execute build command on Snowflake. | + | compile Execute compile command on Snowflake. | + | deps Execute deps command on Snowflake. | + | list Execute list command on Snowflake. | + | parse Execute parse command on Snowflake. | + | run Execute run command on Snowflake. | + | seed Execute seed command on Snowflake. | + | show Execute show command on Snowflake. | + | snapshot Execute snapshot command on Snowflake. | + | test Execute test command on Snowflake. | + +------------------------------------------------------------------------------+ + + + ''' +# --- +# name: test_help_messages_no_help_flag[dbt] + ''' + + Usage: root dbt [OPTIONS] COMMAND [ARGS]... + + Manages dbt on Snowflake projects + + +- Options --------------------------------------------------------------------+ + | --help -h Show this message and exit. | + +------------------------------------------------------------------------------+ + +- Commands -------------------------------------------------------------------+ + | deploy Copy dbt files and create or update dbt on Snowflake project. | + | execute Execute a dbt command on Snowflake | + | list Lists all available dbt projects. | + +------------------------------------------------------------------------------+ + + ''' # --- # name: test_help_messages_no_help_flag[git] diff --git a/tests/test_help_messages.py b/tests/test_help_messages.py index d18d46d184..f7f1ed2074 100644 --- a/tests/test_help_messages.py +++ b/tests/test_help_messages.py @@ -34,7 +34,7 @@ def iter_through_all_commands(command_groups_only: bool = False): Generator iterating through all commands. Paths are yielded as List[str] """ - ignore_plugins = ["helpers", "cortex", "workspace", "dbt"] + ignore_plugins = ["helpers", "cortex", "workspace"] no_command: List[str] = [] yield no_command From 6ad78b5bc562fe8a0fb588395cdbdff2cb9dd4be Mon Sep 17 00:00:00 2001 From: Jakub Wilkowski Date: Mon, 24 Mar 2025 10:22:30 +0100 Subject: [PATCH 42/54] chore: [SNOW-1890085] cleanup --- src/snowflake/cli/_plugins/dbt/constants.py | 14 ++++++++++++++ tests/testing_utils/fixtures.py | 0 2 files changed, 14 insertions(+) delete mode 100644 tests/testing_utils/fixtures.py diff --git a/src/snowflake/cli/_plugins/dbt/constants.py b/src/snowflake/cli/_plugins/dbt/constants.py index ba249f183f..1a71541dd8 100644 --- a/src/snowflake/cli/_plugins/dbt/constants.py +++ b/src/snowflake/cli/_plugins/dbt/constants.py @@ -1,3 +1,17 @@ +# Copyright (c) 2025 Snowflake Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + DBT_COMMANDS = [ "build", "compile", diff --git a/tests/testing_utils/fixtures.py b/tests/testing_utils/fixtures.py deleted file mode 100644 index e69de29bb2..0000000000 From f7b09f2c5e83b2fd4ae20572c8b3ae7bc6b85207 Mon Sep 17 00:00:00 2001 From: Jakub Wilkowski Date: Mon, 24 Mar 2025 12:42:45 +0100 Subject: [PATCH 43/54] test: [SNOW-1890085] add integration test for simple dbt workflow --- tests_integration/conftest.py | 13 ++++ .../projects/dbt_project/dbt_project.yml | 20 ++++++ .../models/example/my_first_dbt_model.sql | 11 +++ .../models/example/my_second_dbt_model.sql | 3 + .../dbt_project/models/example/schema.yml | 21 ++++++ tests_integration/test_dbt.py | 67 +++++++++++++++++++ 6 files changed, 135 insertions(+) create mode 100644 tests_integration/test_data/projects/dbt_project/dbt_project.yml create mode 100644 tests_integration/test_data/projects/dbt_project/models/example/my_first_dbt_model.sql create mode 100644 tests_integration/test_data/projects/dbt_project/models/example/my_second_dbt_model.sql create mode 100644 tests_integration/test_data/projects/dbt_project/models/example/schema.yml create mode 100644 tests_integration/test_dbt.py diff --git a/tests_integration/conftest.py b/tests_integration/conftest.py index b9bb16b0a2..6e22ea8701 100644 --- a/tests_integration/conftest.py +++ b/tests_integration/conftest.py @@ -185,6 +185,19 @@ def invoke_with_connection( ) -> CommandResult: return self.invoke_with_config([*args, "-c", connection], **kwargs) + def invoke_passthrough_with_connection( + self, + args, + connection: str = "integration", + passthrough_args: Optional[list[str]] = None, + **kwargs, + ) -> CommandResult: + if passthrough_args is None: + passthrough_args = list() + return self.invoke_with_config( + [*args, "-c", connection, *passthrough_args], **kwargs + ) + @pytest.fixture def runner(test_snowcli_config_provider, default_username, resource_suffix): diff --git a/tests_integration/test_data/projects/dbt_project/dbt_project.yml b/tests_integration/test_data/projects/dbt_project/dbt_project.yml new file mode 100644 index 0000000000..55a4e79ef0 --- /dev/null +++ b/tests_integration/test_data/projects/dbt_project/dbt_project.yml @@ -0,0 +1,20 @@ +name: 'dbt_integration_project' +version: '1.0.0' + +profile: 'dbt_integration_project' + +model-paths: ["models"] +analysis-paths: ["analyses"] +test-paths: ["tests"] +seed-paths: ["seeds"] +macro-paths: ["macros"] +snapshot-paths: ["snapshots"] + +clean-targets: + - "target" + - "dbt_packages" + +models: + dbt_integration_project: + example: + +materialized: view diff --git a/tests_integration/test_data/projects/dbt_project/models/example/my_first_dbt_model.sql b/tests_integration/test_data/projects/dbt_project/models/example/my_first_dbt_model.sql new file mode 100644 index 0000000000..983588f0eb --- /dev/null +++ b/tests_integration/test_data/projects/dbt_project/models/example/my_first_dbt_model.sql @@ -0,0 +1,11 @@ +{{ config(materialized='table') }} + +with source_data as ( + select 1 as id + union all + select null as id +) + +select * +from source_data +where id is not null diff --git a/tests_integration/test_data/projects/dbt_project/models/example/my_second_dbt_model.sql b/tests_integration/test_data/projects/dbt_project/models/example/my_second_dbt_model.sql new file mode 100644 index 0000000000..7e2c031f11 --- /dev/null +++ b/tests_integration/test_data/projects/dbt_project/models/example/my_second_dbt_model.sql @@ -0,0 +1,3 @@ +select * +from {{ ref('my_first_dbt_model') }} +where id = 1 diff --git a/tests_integration/test_data/projects/dbt_project/models/example/schema.yml b/tests_integration/test_data/projects/dbt_project/models/example/schema.yml new file mode 100644 index 0000000000..9730b7071b --- /dev/null +++ b/tests_integration/test_data/projects/dbt_project/models/example/schema.yml @@ -0,0 +1,21 @@ + +version: 2 + +models: + - name: my_first_dbt_model + description: "A starter dbt model" + columns: + - name: id + description: "The primary key for this table" + data_tests: + - unique + - not_null + + - name: my_second_dbt_model + description: "A starter dbt model" + columns: + - name: id + description: "The primary key for this table" + data_tests: + - unique + - not_null diff --git a/tests_integration/test_dbt.py b/tests_integration/test_dbt.py new file mode 100644 index 0000000000..4eec850179 --- /dev/null +++ b/tests_integration/test_dbt.py @@ -0,0 +1,67 @@ +# Copyright (c) 2025 Snowflake Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import datetime + +import pytest + + +@pytest.mark.integration +@pytest.mark.qa_only +def test_dbt_deploy( + runner, + snowflake_session, + test_database, + project_directory, +): + with project_directory("dbt_project"): + # Given a local dbt project + ts = int(datetime.datetime.now().timestamp()) + name = f"dbt_project_{ts}" + + # When it's deployed + result = runner.invoke_with_connection_json(["dbt", "deploy", name]) + assert result.exit_code == 0, result.output + + # Then it can be listed + result = runner.invoke_with_connection_json( + [ + "dbt", + "list", + "--like", + name, + ] + ) + assert result.exit_code == 0, result.output + assert len(result.json) == 1 + dbt_object = result.json[0] + assert dbt_object["name"].lower() == name.lower() + + # And when dbt run gets called on it + result = runner.invoke_passthrough_with_connection( + args=[ + "dbt", + "execute", + ], + passthrough_args=[name, "run"], + ) + + # Then is succeeds and models get populated according to expectations + assert result.exit_code == 0, result.output + assert "Done. PASS=2 WARN=0 ERROR=0 SKIP=0 TOTAL=2" in result.output + + result = runner.invoke_with_connection_json( + ["sql", "-q", "select count(*) as COUNT from my_second_dbt_model;"] + ) + assert len(result.json) == 1, result.json + assert result.json[0]["COUNT"] == 1, result.json[0] From 382855a731dcfee54f1305a03d1c2b0c884a78a5 Mon Sep 17 00:00:00 2001 From: Jakub Wilkowski Date: Tue, 25 Mar 2025 17:32:28 +0100 Subject: [PATCH 44/54] chore: [SNOW-1890085] rename feature flag --- src/snowflake/cli/_plugins/dbt/commands.py | 2 +- src/snowflake/cli/api/feature_flags.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/snowflake/cli/_plugins/dbt/commands.py b/src/snowflake/cli/_plugins/dbt/commands.py index be3e574519..8cd37c4e99 100644 --- a/src/snowflake/cli/_plugins/dbt/commands.py +++ b/src/snowflake/cli/_plugins/dbt/commands.py @@ -35,7 +35,7 @@ app = SnowTyperFactory( name="dbt", help="Manages dbt on Snowflake projects", - is_hidden=FeatureFlag.ENABLE_DBT_POC.is_disabled, + is_hidden=FeatureFlag.ENABLE_DBT.is_disabled, ) log = logging.getLogger(__name__) diff --git a/src/snowflake/cli/api/feature_flags.py b/src/snowflake/cli/api/feature_flags.py index adbc3846d3..75080444ec 100644 --- a/src/snowflake/cli/api/feature_flags.py +++ b/src/snowflake/cli/api/feature_flags.py @@ -68,5 +68,5 @@ class FeatureFlag(FeatureFlagMixin): ) ENABLE_SNOWPARK_GLOB_SUPPORT = BooleanFlag("ENABLE_SNOWPARK_GLOB_SUPPORT", False) ENABLE_SPCS_SERVICE_EVENTS = BooleanFlag("ENABLE_SPCS_SERVICE_EVENTS", False) + ENABLE_DBT = BooleanFlag("ENABLE_DBT", False) ENABLE_AUTH_KEYPAIR = BooleanFlag("ENABLE_AUTH_KEYPAIR", False) - ENABLE_DBT_POC = BooleanFlag("ENABLE_DBT_POC", False) From 3bc96c1bcb154e28dc5d5e5cdbae1c81fb311c22 Mon Sep 17 00:00:00 2001 From: Jakub Wilkowski Date: Wed, 26 Mar 2025 09:58:24 +0100 Subject: [PATCH 45/54] refactor: [SNOW-1890085] post code review fixes --- src/snowflake/cli/_plugins/dbt/commands.py | 15 +++++++++++---- src/snowflake/cli/_plugins/dbt/manager.py | 8 +++++--- src/snowflake/cli/api/sql_execution.py | 8 +++----- tests/dbt/test_dbt_commands.py | 2 +- 4 files changed, 20 insertions(+), 13 deletions(-) diff --git a/src/snowflake/cli/_plugins/dbt/commands.py b/src/snowflake/cli/_plugins/dbt/commands.py index 8cd37c4e99..1cb622b3ba 100644 --- a/src/snowflake/cli/_plugins/dbt/commands.py +++ b/src/snowflake/cli/_plugins/dbt/commands.py @@ -29,7 +29,12 @@ from snowflake.cli.api.constants import ObjectType from snowflake.cli.api.feature_flags import FeatureFlag from snowflake.cli.api.identifiers import FQN -from snowflake.cli.api.output.types import CommandResult, MessageResult, QueryResult +from snowflake.cli.api.output.types import ( + CommandResult, + MessageResult, + QueryResult, + SingleQueryResult, +) from snowflake.cli.api.secure_path import SecurePath app = SnowTyperFactory( @@ -129,9 +134,10 @@ def _dbt_execute( name = ctx.parent.params["name"] run_async = ctx.parent.params["run_async"] execute_args = (dbt_command, name, run_async, *dbt_cli_args) + dbt_manager = DBTManager() if run_async is True: - result = DBTManager().execute(*execute_args) + result = dbt_manager.execute(*execute_args) return MessageResult( f"Command submitted. You can check the result with `snow sql -q \"select execution_status from table(information_schema.query_history_by_user()) where query_id in ('{result.sfqid}');\"`" ) @@ -141,5 +147,6 @@ def _dbt_execute( TextColumn("[progress.description]{task.description}"), transient=True, ) as progress: - progress.add_task(description="Executing dbt command...", total=None) - return QueryResult(DBTManager().execute(*execute_args)) + progress.add_task(description=f"Executing 'dbt {dbt_command}'", total=None) + result = dbt_manager.execute(*execute_args) + return SingleQueryResult(result) diff --git a/src/snowflake/cli/_plugins/dbt/manager.py b/src/snowflake/cli/_plugins/dbt/manager.py index adc60fe8b1..090085894a 100644 --- a/src/snowflake/cli/_plugins/dbt/manager.py +++ b/src/snowflake/cli/_plugins/dbt/manager.py @@ -44,7 +44,9 @@ def deploy( ) -> SnowflakeCursor: dbt_project_path = path / "dbt_project.yml" if not dbt_project_path.exists(): - raise ClickException(f"dbt_project.yml does not exist in provided path.") + raise ClickException( + f"dbt_project.yml does not exist in directory {path.path.absolute()}." + ) if self.exists(name=name) and force is not True: raise ClickException( @@ -69,6 +71,6 @@ def deploy( def execute(self, dbt_command: str, name: str, run_async: bool, *dbt_cli_args): if dbt_cli_args: - dbt_command = dbt_command + " " + " ".join([arg for arg in dbt_cli_args]) - query = f"EXECUTE DBT PROJECT {name} args='{dbt_command.strip()}'" + dbt_command = " ".join([dbt_command, *dbt_cli_args]).strip() + query = f"EXECUTE DBT PROJECT {name} args='{dbt_command}'" return self.execute_query(query, _exec_async=run_async) diff --git a/src/snowflake/cli/api/sql_execution.py b/src/snowflake/cli/api/sql_execution.py index d5038a9312..c533e3f176 100644 --- a/src/snowflake/cli/api/sql_execution.py +++ b/src/snowflake/cli/api/sql_execution.py @@ -87,20 +87,18 @@ def _execute_string( def execute_string(self, query: str, **kwargs) -> Iterable[SnowflakeCursor]: """Executes a single SQL query and returns the results""" - return self._execute_string(query, **kwargs) + return self._execute_string(dedent(query), **kwargs) def execute_query(self, query: str, **kwargs) -> SnowflakeCursor: """Executes a single SQL query and returns the last result""" - *_, last_result = list(self.execute_string(dedent(query), **kwargs)) + *_, last_result = list(self.execute_string(query, **kwargs)) return last_result def execute_queries(self, queries: str, **kwargs): """Executes multiple SQL queries (passed as one string) and returns the results as a list""" # Without remove_comments=True, connectors might throw an error if there is a comment at the end of the file - return list( - self.execute_string(dedent(queries), remove_comments=True, **kwargs) - ) + return list(self.execute_string(queries, remove_comments=True, **kwargs)) class SqlExecutor(BaseSqlExecutor): diff --git a/tests/dbt/test_dbt_commands.py b/tests/dbt/test_dbt_commands.py index 05950173e1..b9eb2d1691 100644 --- a/tests/dbt/test_dbt_commands.py +++ b/tests/dbt/test_dbt_commands.py @@ -177,7 +177,7 @@ def test_raises_when_dbt_project_is_not_available( ) assert result.exit_code == 1, result.output - assert "dbt_project.yml does not exist in provided path." in result.output + assert f"dbt_project.yml does not exist in directory" in result.output assert mock_connect.mocked_ctx.get_query() == "" def test_raises_when_dbt_project_exists_and_is_not_force( From b58a86de77e4ed8865429c686bf874dfe8948404 Mon Sep 17 00:00:00 2001 From: Jakub Wilkowski Date: Tue, 1 Apr 2025 15:40:44 +0200 Subject: [PATCH 46/54] feat: [SNOW-1890085] wip: capture success code from server --- src/snowflake/cli/_plugins/dbt/commands.py | 15 +++++++++++++-- src/snowflake/cli/_plugins/dbt/manager.py | 4 +++- 2 files changed, 16 insertions(+), 3 deletions(-) diff --git a/src/snowflake/cli/_plugins/dbt/commands.py b/src/snowflake/cli/_plugins/dbt/commands.py index 1cb622b3ba..6e89a942ab 100644 --- a/src/snowflake/cli/_plugins/dbt/commands.py +++ b/src/snowflake/cli/_plugins/dbt/commands.py @@ -18,6 +18,7 @@ from typing import Optional import typer +from click import ClickException from rich.progress import Progress, SpinnerColumn, TextColumn from snowflake.cli._plugins.dbt.constants import DBT_COMMANDS from snowflake.cli._plugins.dbt.manager import DBTManager @@ -33,7 +34,6 @@ CommandResult, MessageResult, QueryResult, - SingleQueryResult, ) from snowflake.cli.api.secure_path import SecurePath @@ -148,5 +148,16 @@ def _dbt_execute( transient=True, ) as progress: progress.add_task(description=f"Executing 'dbt {dbt_command}'", total=None) + result = dbt_manager.execute(*execute_args) - return SingleQueryResult(result) + columns = [column.name for column in result.description] + success_column_index = columns.index("SUCCESS") + stdout_column_index = columns.index("STDOUT") + is_success, output = [ + (row[success_column_index], row[stdout_column_index]) for row in result + ][-1] + + if is_success is True: + return MessageResult(output) + else: + raise ClickException(output) diff --git a/src/snowflake/cli/_plugins/dbt/manager.py b/src/snowflake/cli/_plugins/dbt/manager.py index 090085894a..513209d4d3 100644 --- a/src/snowflake/cli/_plugins/dbt/manager.py +++ b/src/snowflake/cli/_plugins/dbt/manager.py @@ -69,7 +69,9 @@ def deploy( return self.execute_query(query) - def execute(self, dbt_command: str, name: str, run_async: bool, *dbt_cli_args): + def execute( + self, dbt_command: str, name: str, run_async: bool, *dbt_cli_args + ) -> SnowflakeCursor: if dbt_cli_args: dbt_command = " ".join([dbt_command, *dbt_cli_args]).strip() query = f"EXECUTE DBT PROJECT {name} args='{dbt_command}'" From 86dcf644dafbe2694585271cf67a6037404a33aa Mon Sep 17 00:00:00 2001 From: Jakub Wilkowski Date: Wed, 2 Apr 2025 17:33:13 +0200 Subject: [PATCH 47/54] feat: [SNOW-1890085] process data from server --- src/snowflake/cli/_plugins/dbt/commands.py | 26 +++-- src/snowflake/cli/_plugins/dbt/constants.py | 3 + tests/__snapshots__/test_help_messages.ambr | 112 ++++++++++---------- tests/dbt/test_dbt_commands.py | 69 +++++++++++- 4 files changed, 146 insertions(+), 64 deletions(-) diff --git a/src/snowflake/cli/_plugins/dbt/commands.py b/src/snowflake/cli/_plugins/dbt/commands.py index 6e89a942ab..fb1b1def42 100644 --- a/src/snowflake/cli/_plugins/dbt/commands.py +++ b/src/snowflake/cli/_plugins/dbt/commands.py @@ -20,7 +20,11 @@ import typer from click import ClickException from rich.progress import Progress, SpinnerColumn, TextColumn -from snowflake.cli._plugins.dbt.constants import DBT_COMMANDS +from snowflake.cli._plugins.dbt.constants import ( + DBT_COMMANDS, + OUTPUT_COLUMN_NAME, + RESULT_COLUMN_NAME, +) from snowflake.cli._plugins.dbt.manager import DBTManager from snowflake.cli._plugins.object.command_aliases import add_object_command_aliases from snowflake.cli._plugins.object.commands import scope_option @@ -150,12 +154,20 @@ def _dbt_execute( progress.add_task(description=f"Executing 'dbt {dbt_command}'", total=None) result = dbt_manager.execute(*execute_args) - columns = [column.name for column in result.description] - success_column_index = columns.index("SUCCESS") - stdout_column_index = columns.index("STDOUT") - is_success, output = [ - (row[success_column_index], row[stdout_column_index]) for row in result - ][-1] + + try: + columns = [column.name for column in result.description] + success_column_index = columns.index(RESULT_COLUMN_NAME) + stdout_column_index = columns.index(OUTPUT_COLUMN_NAME) + except ValueError: + raise ClickException("Malformed server response") + try: + is_success, output = [ + (row[success_column_index], row[stdout_column_index]) + for row in result + ][-1] + except IndexError: + raise ClickException("No data returned from server") if is_success is True: return MessageResult(output) diff --git a/src/snowflake/cli/_plugins/dbt/constants.py b/src/snowflake/cli/_plugins/dbt/constants.py index 1a71541dd8..4c79dfa81c 100644 --- a/src/snowflake/cli/_plugins/dbt/constants.py +++ b/src/snowflake/cli/_plugins/dbt/constants.py @@ -12,6 +12,9 @@ # See the License for the specific language governing permissions and # limitations under the License. +RESULT_COLUMN_NAME = "SUCCESS" +OUTPUT_COLUMN_NAME = "STDOUT" + DBT_COMMANDS = [ "build", "compile", diff --git a/tests/__snapshots__/test_help_messages.ambr b/tests/__snapshots__/test_help_messages.ambr index 4a67a3ce64..46b2e8d87d 100644 --- a/tests/__snapshots__/test_help_messages.ambr +++ b/tests/__snapshots__/test_help_messages.ambr @@ -4019,8 +4019,8 @@ | specified for the | | connection. | | --token-file-path TEXT Path to file with an OAuth | - | token that should be used | - | when connecting to Snowflake | + | token to use when connecting | + | to Snowflake. | | --database,--dbname TEXT Database to use. Overrides | | the value specified for the | | connection. | @@ -4047,8 +4047,8 @@ | --diag-log-path TEXT Path for the generated | | report. Defaults to system | | temporary directory. | - | --diag-allowlist-path TEXT Path to a JSON file | - | containing allowlist | + | --diag-allowlist-path TEXT Path to a JSON file that | + | contains allowlist | | parameters. | +------------------------------------------------------------------------------+ +- Global configuration -------------------------------------------------------+ @@ -4115,8 +4115,8 @@ | specified for the | | connection. | | --token-file-path TEXT Path to file with an OAuth | - | token that should be used | - | when connecting to Snowflake | + | token to use when connecting | + | to Snowflake. | | --database,--dbname TEXT Database to use. Overrides | | the value specified for the | | connection. | @@ -4143,8 +4143,8 @@ | --diag-log-path TEXT Path for the generated | | report. Defaults to system | | temporary directory. | - | --diag-allowlist-path TEXT Path to a JSON file | - | containing allowlist | + | --diag-allowlist-path TEXT Path to a JSON file that | + | contains allowlist | | parameters. | +------------------------------------------------------------------------------+ +- Global configuration -------------------------------------------------------+ @@ -4223,8 +4223,8 @@ | specified for the | | connection. | | --token-file-path TEXT Path to file with an OAuth | - | token that should be used | - | when connecting to Snowflake | + | token to use when connecting | + | to Snowflake. | | --database,--dbname TEXT Database to use. Overrides | | the value specified for the | | connection. | @@ -4251,8 +4251,8 @@ | --diag-log-path TEXT Path for the generated | | report. Defaults to system | | temporary directory. | - | --diag-allowlist-path TEXT Path to a JSON file | - | containing allowlist | + | --diag-allowlist-path TEXT Path to a JSON file that | + | contains allowlist | | parameters. | +------------------------------------------------------------------------------+ +- Global configuration -------------------------------------------------------+ @@ -4331,8 +4331,8 @@ | specified for the | | connection. | | --token-file-path TEXT Path to file with an OAuth | - | token that should be used | - | when connecting to Snowflake | + | token to use when connecting | + | to Snowflake. | | --database,--dbname TEXT Database to use. Overrides | | the value specified for the | | connection. | @@ -4359,8 +4359,8 @@ | --diag-log-path TEXT Path for the generated | | report. Defaults to system | | temporary directory. | - | --diag-allowlist-path TEXT Path to a JSON file | - | containing allowlist | + | --diag-allowlist-path TEXT Path to a JSON file that | + | contains allowlist | | parameters. | +------------------------------------------------------------------------------+ +- Global configuration -------------------------------------------------------+ @@ -4439,8 +4439,8 @@ | specified for the | | connection. | | --token-file-path TEXT Path to file with an OAuth | - | token that should be used | - | when connecting to Snowflake | + | token to use when connecting | + | to Snowflake. | | --database,--dbname TEXT Database to use. Overrides | | the value specified for the | | connection. | @@ -4467,8 +4467,8 @@ | --diag-log-path TEXT Path for the generated | | report. Defaults to system | | temporary directory. | - | --diag-allowlist-path TEXT Path to a JSON file | - | containing allowlist | + | --diag-allowlist-path TEXT Path to a JSON file that | + | contains allowlist | | parameters. | +------------------------------------------------------------------------------+ +- Global configuration -------------------------------------------------------+ @@ -4547,8 +4547,8 @@ | specified for the | | connection. | | --token-file-path TEXT Path to file with an OAuth | - | token that should be used | - | when connecting to Snowflake | + | token to use when connecting | + | to Snowflake. | | --database,--dbname TEXT Database to use. Overrides | | the value specified for the | | connection. | @@ -4575,8 +4575,8 @@ | --diag-log-path TEXT Path for the generated | | report. Defaults to system | | temporary directory. | - | --diag-allowlist-path TEXT Path to a JSON file | - | containing allowlist | + | --diag-allowlist-path TEXT Path to a JSON file that | + | contains allowlist | | parameters. | +------------------------------------------------------------------------------+ +- Global configuration -------------------------------------------------------+ @@ -4655,8 +4655,8 @@ | specified for the | | connection. | | --token-file-path TEXT Path to file with an OAuth | - | token that should be used | - | when connecting to Snowflake | + | token to use when connecting | + | to Snowflake. | | --database,--dbname TEXT Database to use. Overrides | | the value specified for the | | connection. | @@ -4683,8 +4683,8 @@ | --diag-log-path TEXT Path for the generated | | report. Defaults to system | | temporary directory. | - | --diag-allowlist-path TEXT Path to a JSON file | - | containing allowlist | + | --diag-allowlist-path TEXT Path to a JSON file that | + | contains allowlist | | parameters. | +------------------------------------------------------------------------------+ +- Global configuration -------------------------------------------------------+ @@ -4763,8 +4763,8 @@ | specified for the | | connection. | | --token-file-path TEXT Path to file with an OAuth | - | token that should be used | - | when connecting to Snowflake | + | token to use when connecting | + | to Snowflake. | | --database,--dbname TEXT Database to use. Overrides | | the value specified for the | | connection. | @@ -4791,8 +4791,8 @@ | --diag-log-path TEXT Path for the generated | | report. Defaults to system | | temporary directory. | - | --diag-allowlist-path TEXT Path to a JSON file | - | containing allowlist | + | --diag-allowlist-path TEXT Path to a JSON file that | + | contains allowlist | | parameters. | +------------------------------------------------------------------------------+ +- Global configuration -------------------------------------------------------+ @@ -4871,8 +4871,8 @@ | specified for the | | connection. | | --token-file-path TEXT Path to file with an OAuth | - | token that should be used | - | when connecting to Snowflake | + | token to use when connecting | + | to Snowflake. | | --database,--dbname TEXT Database to use. Overrides | | the value specified for the | | connection. | @@ -4899,8 +4899,8 @@ | --diag-log-path TEXT Path for the generated | | report. Defaults to system | | temporary directory. | - | --diag-allowlist-path TEXT Path to a JSON file | - | containing allowlist | + | --diag-allowlist-path TEXT Path to a JSON file that | + | contains allowlist | | parameters. | +------------------------------------------------------------------------------+ +- Global configuration -------------------------------------------------------+ @@ -4979,8 +4979,8 @@ | specified for the | | connection. | | --token-file-path TEXT Path to file with an OAuth | - | token that should be used | - | when connecting to Snowflake | + | token to use when connecting | + | to Snowflake. | | --database,--dbname TEXT Database to use. Overrides | | the value specified for the | | connection. | @@ -5007,8 +5007,8 @@ | --diag-log-path TEXT Path for the generated | | report. Defaults to system | | temporary directory. | - | --diag-allowlist-path TEXT Path to a JSON file | - | containing allowlist | + | --diag-allowlist-path TEXT Path to a JSON file that | + | contains allowlist | | parameters. | +------------------------------------------------------------------------------+ +- Global configuration -------------------------------------------------------+ @@ -5087,8 +5087,8 @@ | specified for the | | connection. | | --token-file-path TEXT Path to file with an OAuth | - | token that should be used | - | when connecting to Snowflake | + | token to use when connecting | + | to Snowflake. | | --database,--dbname TEXT Database to use. Overrides | | the value specified for the | | connection. | @@ -5115,8 +5115,8 @@ | --diag-log-path TEXT Path for the generated | | report. Defaults to system | | temporary directory. | - | --diag-allowlist-path TEXT Path to a JSON file | - | containing allowlist | + | --diag-allowlist-path TEXT Path to a JSON file that | + | contains allowlist | | parameters. | +------------------------------------------------------------------------------+ +- Global configuration -------------------------------------------------------+ @@ -5195,8 +5195,8 @@ | specified for the | | connection. | | --token-file-path TEXT Path to file with an OAuth | - | token that should be used | - | when connecting to Snowflake | + | token to use when connecting | + | to Snowflake. | | --database,--dbname TEXT Database to use. Overrides | | the value specified for the | | connection. | @@ -5223,8 +5223,8 @@ | --diag-log-path TEXT Path for the generated | | report. Defaults to system | | temporary directory. | - | --diag-allowlist-path TEXT Path to a JSON file | - | containing allowlist | + | --diag-allowlist-path TEXT Path to a JSON file that | + | contains allowlist | | parameters. | +------------------------------------------------------------------------------+ +- Global configuration -------------------------------------------------------+ @@ -5306,8 +5306,8 @@ | specified for the | | connection. | | --token-file-path TEXT Path to file with an OAuth | - | token that should be used | - | when connecting to Snowflake | + | token to use when connecting | + | to Snowflake. | | --database,--dbname TEXT Database to use. Overrides | | the value specified for the | | connection. | @@ -5334,8 +5334,8 @@ | --diag-log-path TEXT Path for the generated | | report. Defaults to system | | temporary directory. | - | --diag-allowlist-path TEXT Path to a JSON file | - | containing allowlist | + | --diag-allowlist-path TEXT Path to a JSON file that | + | contains allowlist | | parameters. | +------------------------------------------------------------------------------+ +- Global configuration -------------------------------------------------------+ @@ -15851,8 +15851,8 @@ | specified for the | | connection. | | --token-file-path TEXT Path to file with an OAuth | - | token that should be used | - | when connecting to Snowflake | + | token to use when connecting | + | to Snowflake. | | --database,--dbname TEXT Database to use. Overrides | | the value specified for the | | connection. | @@ -15879,8 +15879,8 @@ | --diag-log-path TEXT Path for the generated | | report. Defaults to system | | temporary directory. | - | --diag-allowlist-path TEXT Path to a JSON file | - | containing allowlist | + | --diag-allowlist-path TEXT Path to a JSON file that | + | contains allowlist | | parameters. | +------------------------------------------------------------------------------+ +- Global configuration -------------------------------------------------------+ diff --git a/tests/dbt/test_dbt_commands.py b/tests/dbt/test_dbt_commands.py index b9eb2d1691..74eae77e6f 100644 --- a/tests/dbt/test_dbt_commands.py +++ b/tests/dbt/test_dbt_commands.py @@ -17,6 +17,7 @@ from unittest import mock import pytest +from snowflake.cli._plugins.dbt.constants import OUTPUT_COLUMN_NAME, RESULT_COLUMN_NAME from snowflake.cli.api.identifiers import FQN @@ -262,7 +263,12 @@ class TestDBTExecute: ), ], ) - def test_dbt_execute(self, mock_connect, runner, args, expected_query): + def test_dbt_execute(self, mock_connect, mock_cursor, runner, args, expected_query): + cursor = mock_cursor( + rows=[(True, "very detailed logs")], + columns=[RESULT_COLUMN_NAME, OUTPUT_COLUMN_NAME], + ) + mock_connect.mocked_ctx.cs = cursor result = runner.invoke(args) @@ -288,3 +294,64 @@ def test_execute_async(self, mock_connect, runner): mock_connect.mocked_ctx.get_query() == "EXECUTE DBT PROJECT pipeline_name args='compile'" ) + + def test_dbt_execute_dbt_failure_returns_non_0_code( + self, mock_connect, mock_cursor, runner + ): + cursor = mock_cursor( + rows=[(False, "1 of 4 FAIL 1 not_null_my_first_dbt_model_id")], + columns=[RESULT_COLUMN_NAME, OUTPUT_COLUMN_NAME], + ) + mock_connect.mocked_ctx.cs = cursor + + result = runner.invoke( + [ + "dbt", + "execute", + "pipeline_name", + "test", + ] + ) + + assert result.exit_code == 1, result.output + assert "1 of 4 FAIL 1 not_null_my_first_dbt_model_id" in result.output + + def test_dbt_execute_malformed_server_response( + self, mock_connect, mock_cursor, runner + ): + cursor = mock_cursor( + rows=[(True, "very detailed logs")], + columns=["foo", "bar"], + ) + mock_connect.mocked_ctx.cs = cursor + + result = runner.invoke( + [ + "dbt", + "execute", + "pipeline_name", + "test", + ] + ) + + assert result.exit_code == 1, result.output + assert "Malformed server response" in result.output + + def test_dbt_execute_no_rows_in_response(self, mock_connect, mock_cursor, runner): + cursor = mock_cursor( + rows=[], + columns=[RESULT_COLUMN_NAME, OUTPUT_COLUMN_NAME], + ) + mock_connect.mocked_ctx.cs = cursor + + result = runner.invoke( + [ + "dbt", + "execute", + "pipeline_name", + "test", + ] + ) + + assert result.exit_code == 1, result.output + assert "No data returned from server" in result.output From 57de90dc9a7ba0f5e86c3faaa0542d2fa6ead597 Mon Sep 17 00:00:00 2001 From: Jakub Wilkowski Date: Fri, 4 Apr 2025 10:52:04 +0200 Subject: [PATCH 48/54] feat: [SNOW-1890085] custom version for dbt branch --- src/snowflake/cli/__about__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/snowflake/cli/__about__.py b/src/snowflake/cli/__about__.py index 2ba4191de3..82769b09fe 100644 --- a/src/snowflake/cli/__about__.py +++ b/src/snowflake/cli/__about__.py @@ -16,7 +16,7 @@ from enum import Enum, unique -VERSION = "3.8.0.dev0" +VERSION = "3.7.0.dev+dbt0" @unique From a928c36d8d0581d067041586cecf083ee64f0f1a Mon Sep 17 00:00:00 2001 From: Jakub Wilkowski Date: Thu, 10 Apr 2025 14:24:24 +0200 Subject: [PATCH 49/54] chore: [SNOW-1890085] update snapshots --- tests/__snapshots__/test_help_messages.ambr | 280 ++++++++++++-------- 1 file changed, 168 insertions(+), 112 deletions(-) diff --git a/tests/__snapshots__/test_help_messages.ambr b/tests/__snapshots__/test_help_messages.ambr index 46b2e8d87d..1d2c643dae 100644 --- a/tests/__snapshots__/test_help_messages.ambr +++ b/tests/__snapshots__/test_help_messages.ambr @@ -4052,14 +4052,18 @@ | parameters. | +------------------------------------------------------------------------------+ +- Global configuration -------------------------------------------------------+ - | --format [TABLE|JSON] Specifies the output format. | - | [default: TABLE] | - | --verbose -v Displays log entries for log levels info | - | and higher. | - | --debug Displays log entries for log levels debug | - | and higher; debug logs contain additional | - | information. | - | --silent Turns off intermediate output to console. | + | --format [TABLE|JSON] Specifies the output format. | + | [default: TABLE] | + | --verbose -v Displays log entries for log | + | levels info and higher. | + | --debug Displays log entries for log | + | levels debug and higher; debug | + | logs contain additional | + | information. | + | --silent Turns off intermediate output | + | to console. | + | --enhanced-exit-codes Differentiate exit error codes | + | based on failure type. | +------------------------------------------------------------------------------+ @@ -4148,14 +4152,18 @@ | parameters. | +------------------------------------------------------------------------------+ +- Global configuration -------------------------------------------------------+ - | --format [TABLE|JSON] Specifies the output format. | - | [default: TABLE] | - | --verbose -v Displays log entries for log levels info | - | and higher. | - | --debug Displays log entries for log levels debug | - | and higher; debug logs contain additional | - | information. | - | --silent Turns off intermediate output to console. | + | --format [TABLE|JSON] Specifies the output format. | + | [default: TABLE] | + | --verbose -v Displays log entries for log | + | levels info and higher. | + | --debug Displays log entries for log | + | levels debug and higher; debug | + | logs contain additional | + | information. | + | --silent Turns off intermediate output | + | to console. | + | --enhanced-exit-codes Differentiate exit error codes | + | based on failure type. | +------------------------------------------------------------------------------+ +- Commands -------------------------------------------------------------------+ | build Execute build command on Snowflake. | @@ -4256,14 +4264,18 @@ | parameters. | +------------------------------------------------------------------------------+ +- Global configuration -------------------------------------------------------+ - | --format [TABLE|JSON] Specifies the output format. | - | [default: TABLE] | - | --verbose -v Displays log entries for log levels info | - | and higher. | - | --debug Displays log entries for log levels debug | - | and higher; debug logs contain additional | - | information. | - | --silent Turns off intermediate output to console. | + | --format [TABLE|JSON] Specifies the output format. | + | [default: TABLE] | + | --verbose -v Displays log entries for log | + | levels info and higher. | + | --debug Displays log entries for log | + | levels debug and higher; debug | + | logs contain additional | + | information. | + | --silent Turns off intermediate output | + | to console. | + | --enhanced-exit-codes Differentiate exit error codes | + | based on failure type. | +------------------------------------------------------------------------------+ +- Commands -------------------------------------------------------------------+ | build Execute build command on Snowflake. | @@ -4364,14 +4376,18 @@ | parameters. | +------------------------------------------------------------------------------+ +- Global configuration -------------------------------------------------------+ - | --format [TABLE|JSON] Specifies the output format. | - | [default: TABLE] | - | --verbose -v Displays log entries for log levels info | - | and higher. | - | --debug Displays log entries for log levels debug | - | and higher; debug logs contain additional | - | information. | - | --silent Turns off intermediate output to console. | + | --format [TABLE|JSON] Specifies the output format. | + | [default: TABLE] | + | --verbose -v Displays log entries for log | + | levels info and higher. | + | --debug Displays log entries for log | + | levels debug and higher; debug | + | logs contain additional | + | information. | + | --silent Turns off intermediate output | + | to console. | + | --enhanced-exit-codes Differentiate exit error codes | + | based on failure type. | +------------------------------------------------------------------------------+ +- Commands -------------------------------------------------------------------+ | build Execute build command on Snowflake. | @@ -4472,14 +4488,18 @@ | parameters. | +------------------------------------------------------------------------------+ +- Global configuration -------------------------------------------------------+ - | --format [TABLE|JSON] Specifies the output format. | - | [default: TABLE] | - | --verbose -v Displays log entries for log levels info | - | and higher. | - | --debug Displays log entries for log levels debug | - | and higher; debug logs contain additional | - | information. | - | --silent Turns off intermediate output to console. | + | --format [TABLE|JSON] Specifies the output format. | + | [default: TABLE] | + | --verbose -v Displays log entries for log | + | levels info and higher. | + | --debug Displays log entries for log | + | levels debug and higher; debug | + | logs contain additional | + | information. | + | --silent Turns off intermediate output | + | to console. | + | --enhanced-exit-codes Differentiate exit error codes | + | based on failure type. | +------------------------------------------------------------------------------+ +- Commands -------------------------------------------------------------------+ | build Execute build command on Snowflake. | @@ -4580,14 +4600,18 @@ | parameters. | +------------------------------------------------------------------------------+ +- Global configuration -------------------------------------------------------+ - | --format [TABLE|JSON] Specifies the output format. | - | [default: TABLE] | - | --verbose -v Displays log entries for log levels info | - | and higher. | - | --debug Displays log entries for log levels debug | - | and higher; debug logs contain additional | - | information. | - | --silent Turns off intermediate output to console. | + | --format [TABLE|JSON] Specifies the output format. | + | [default: TABLE] | + | --verbose -v Displays log entries for log | + | levels info and higher. | + | --debug Displays log entries for log | + | levels debug and higher; debug | + | logs contain additional | + | information. | + | --silent Turns off intermediate output | + | to console. | + | --enhanced-exit-codes Differentiate exit error codes | + | based on failure type. | +------------------------------------------------------------------------------+ +- Commands -------------------------------------------------------------------+ | build Execute build command on Snowflake. | @@ -4688,14 +4712,18 @@ | parameters. | +------------------------------------------------------------------------------+ +- Global configuration -------------------------------------------------------+ - | --format [TABLE|JSON] Specifies the output format. | - | [default: TABLE] | - | --verbose -v Displays log entries for log levels info | - | and higher. | - | --debug Displays log entries for log levels debug | - | and higher; debug logs contain additional | - | information. | - | --silent Turns off intermediate output to console. | + | --format [TABLE|JSON] Specifies the output format. | + | [default: TABLE] | + | --verbose -v Displays log entries for log | + | levels info and higher. | + | --debug Displays log entries for log | + | levels debug and higher; debug | + | logs contain additional | + | information. | + | --silent Turns off intermediate output | + | to console. | + | --enhanced-exit-codes Differentiate exit error codes | + | based on failure type. | +------------------------------------------------------------------------------+ +- Commands -------------------------------------------------------------------+ | build Execute build command on Snowflake. | @@ -4796,14 +4824,18 @@ | parameters. | +------------------------------------------------------------------------------+ +- Global configuration -------------------------------------------------------+ - | --format [TABLE|JSON] Specifies the output format. | - | [default: TABLE] | - | --verbose -v Displays log entries for log levels info | - | and higher. | - | --debug Displays log entries for log levels debug | - | and higher; debug logs contain additional | - | information. | - | --silent Turns off intermediate output to console. | + | --format [TABLE|JSON] Specifies the output format. | + | [default: TABLE] | + | --verbose -v Displays log entries for log | + | levels info and higher. | + | --debug Displays log entries for log | + | levels debug and higher; debug | + | logs contain additional | + | information. | + | --silent Turns off intermediate output | + | to console. | + | --enhanced-exit-codes Differentiate exit error codes | + | based on failure type. | +------------------------------------------------------------------------------+ +- Commands -------------------------------------------------------------------+ | build Execute build command on Snowflake. | @@ -4904,14 +4936,18 @@ | parameters. | +------------------------------------------------------------------------------+ +- Global configuration -------------------------------------------------------+ - | --format [TABLE|JSON] Specifies the output format. | - | [default: TABLE] | - | --verbose -v Displays log entries for log levels info | - | and higher. | - | --debug Displays log entries for log levels debug | - | and higher; debug logs contain additional | - | information. | - | --silent Turns off intermediate output to console. | + | --format [TABLE|JSON] Specifies the output format. | + | [default: TABLE] | + | --verbose -v Displays log entries for log | + | levels info and higher. | + | --debug Displays log entries for log | + | levels debug and higher; debug | + | logs contain additional | + | information. | + | --silent Turns off intermediate output | + | to console. | + | --enhanced-exit-codes Differentiate exit error codes | + | based on failure type. | +------------------------------------------------------------------------------+ +- Commands -------------------------------------------------------------------+ | build Execute build command on Snowflake. | @@ -5012,14 +5048,18 @@ | parameters. | +------------------------------------------------------------------------------+ +- Global configuration -------------------------------------------------------+ - | --format [TABLE|JSON] Specifies the output format. | - | [default: TABLE] | - | --verbose -v Displays log entries for log levels info | - | and higher. | - | --debug Displays log entries for log levels debug | - | and higher; debug logs contain additional | - | information. | - | --silent Turns off intermediate output to console. | + | --format [TABLE|JSON] Specifies the output format. | + | [default: TABLE] | + | --verbose -v Displays log entries for log | + | levels info and higher. | + | --debug Displays log entries for log | + | levels debug and higher; debug | + | logs contain additional | + | information. | + | --silent Turns off intermediate output | + | to console. | + | --enhanced-exit-codes Differentiate exit error codes | + | based on failure type. | +------------------------------------------------------------------------------+ +- Commands -------------------------------------------------------------------+ | build Execute build command on Snowflake. | @@ -5120,14 +5160,18 @@ | parameters. | +------------------------------------------------------------------------------+ +- Global configuration -------------------------------------------------------+ - | --format [TABLE|JSON] Specifies the output format. | - | [default: TABLE] | - | --verbose -v Displays log entries for log levels info | - | and higher. | - | --debug Displays log entries for log levels debug | - | and higher; debug logs contain additional | - | information. | - | --silent Turns off intermediate output to console. | + | --format [TABLE|JSON] Specifies the output format. | + | [default: TABLE] | + | --verbose -v Displays log entries for log | + | levels info and higher. | + | --debug Displays log entries for log | + | levels debug and higher; debug | + | logs contain additional | + | information. | + | --silent Turns off intermediate output | + | to console. | + | --enhanced-exit-codes Differentiate exit error codes | + | based on failure type. | +------------------------------------------------------------------------------+ +- Commands -------------------------------------------------------------------+ | build Execute build command on Snowflake. | @@ -5228,14 +5272,18 @@ | parameters. | +------------------------------------------------------------------------------+ +- Global configuration -------------------------------------------------------+ - | --format [TABLE|JSON] Specifies the output format. | - | [default: TABLE] | - | --verbose -v Displays log entries for log levels info | - | and higher. | - | --debug Displays log entries for log levels debug | - | and higher; debug logs contain additional | - | information. | - | --silent Turns off intermediate output to console. | + | --format [TABLE|JSON] Specifies the output format. | + | [default: TABLE] | + | --verbose -v Displays log entries for log | + | levels info and higher. | + | --debug Displays log entries for log | + | levels debug and higher; debug | + | logs contain additional | + | information. | + | --silent Turns off intermediate output | + | to console. | + | --enhanced-exit-codes Differentiate exit error codes | + | based on failure type. | +------------------------------------------------------------------------------+ +- Commands -------------------------------------------------------------------+ | build Execute build command on Snowflake. | @@ -5339,14 +5387,18 @@ | parameters. | +------------------------------------------------------------------------------+ +- Global configuration -------------------------------------------------------+ - | --format [TABLE|JSON] Specifies the output format. | - | [default: TABLE] | - | --verbose -v Displays log entries for log levels info | - | and higher. | - | --debug Displays log entries for log levels debug | - | and higher; debug logs contain additional | - | information. | - | --silent Turns off intermediate output to console. | + | --format [TABLE|JSON] Specifies the output format. | + | [default: TABLE] | + | --verbose -v Displays log entries for log | + | levels info and higher. | + | --debug Displays log entries for log | + | levels debug and higher; debug | + | logs contain additional | + | information. | + | --silent Turns off intermediate output | + | to console. | + | --enhanced-exit-codes Differentiate exit error codes | + | based on failure type. | +------------------------------------------------------------------------------+ @@ -15884,14 +15936,18 @@ | parameters. | +------------------------------------------------------------------------------+ +- Global configuration -------------------------------------------------------+ - | --format [TABLE|JSON] Specifies the output format. | - | [default: TABLE] | - | --verbose -v Displays log entries for log levels info | - | and higher. | - | --debug Displays log entries for log levels debug | - | and higher; debug logs contain additional | - | information. | - | --silent Turns off intermediate output to console. | + | --format [TABLE|JSON] Specifies the output format. | + | [default: TABLE] | + | --verbose -v Displays log entries for log | + | levels info and higher. | + | --debug Displays log entries for log | + | levels debug and higher; debug | + | logs contain additional | + | information. | + | --silent Turns off intermediate output | + | to console. | + | --enhanced-exit-codes Differentiate exit error codes | + | based on failure type. | +------------------------------------------------------------------------------+ +- Commands -------------------------------------------------------------------+ | build Execute build command on Snowflake. | From af36b8a4d18878adb304d40614b58dc46bbab0e6 Mon Sep 17 00:00:00 2001 From: Jakub Wilkowski Date: Fri, 11 Apr 2025 12:46:38 +0200 Subject: [PATCH 50/54] Jw/snow 2036324 include dbt profile (#2196) * feat: [SNOW-2036324] validate dbt profile * refactor: [SNOW-2036324] PR fixes --- src/snowflake/cli/_plugins/dbt/manager.py | 27 ++++- tests/dbt/test_dbt_commands.py | 102 ++++++++++++------ .../projects/dbt_project/profiles.yml | 5 + tests_integration/test_dbt.py | 19 +++- 4 files changed, 117 insertions(+), 36 deletions(-) create mode 100644 tests_integration/test_data/projects/dbt_project/profiles.yml diff --git a/src/snowflake/cli/_plugins/dbt/manager.py b/src/snowflake/cli/_plugins/dbt/manager.py index 513209d4d3..305f02b2d6 100644 --- a/src/snowflake/cli/_plugins/dbt/manager.py +++ b/src/snowflake/cli/_plugins/dbt/manager.py @@ -14,11 +14,12 @@ from __future__ import annotations -from click import ClickException +import yaml from snowflake.cli._plugins.object.manager import ObjectManager from snowflake.cli._plugins.stage.manager import StageManager from snowflake.cli.api.console import cli_console -from snowflake.cli.api.constants import ObjectType +from snowflake.cli.api.constants import DEFAULT_SIZE_LIMIT_MB, ObjectType +from snowflake.cli.api.exceptions import CliError from snowflake.cli.api.identifiers import FQN from snowflake.cli.api.secure_path import SecurePath from snowflake.cli.api.sql_execution import SqlExecutionMixin @@ -44,12 +45,30 @@ def deploy( ) -> SnowflakeCursor: dbt_project_path = path / "dbt_project.yml" if not dbt_project_path.exists(): - raise ClickException( + raise CliError( f"dbt_project.yml does not exist in directory {path.path.absolute()}." ) + with dbt_project_path.open(read_file_limit_mb=DEFAULT_SIZE_LIMIT_MB) as fd: + dbt_project = yaml.safe_load(fd) + try: + profile = dbt_project["profile"] + except KeyError: + raise CliError("`profile` is not defined in dbt_project.yml") + + dbt_profiles_path = path / "profiles.yml" + if not dbt_profiles_path.exists(): + raise CliError( + f"profiles.yml does not exist in directory {path.path.absolute()}." + ) + + with dbt_profiles_path.open(read_file_limit_mb=DEFAULT_SIZE_LIMIT_MB) as fd: + profiles = yaml.safe_load(fd) + if profile not in profiles: + raise CliError(f"profile {profile} is not defined in profiles.yml") + if self.exists(name=name) and force is not True: - raise ClickException( + raise CliError( f"DBT project {name} already exists. Use --force flag to overwrite" ) diff --git a/tests/dbt/test_dbt_commands.py b/tests/dbt/test_dbt_commands.py index 74eae77e6f..2c01fd6f53 100644 --- a/tests/dbt/test_dbt_commands.py +++ b/tests/dbt/test_dbt_commands.py @@ -17,6 +17,7 @@ from unittest import mock import pytest +import yaml from snowflake.cli._plugins.dbt.constants import OUTPUT_COLUMN_NAME, RESULT_COLUMN_NAME from snowflake.cli.api.identifiers import FQN @@ -56,8 +57,20 @@ class TestDBTDeploy: @pytest.fixture def dbt_project_path(self, tmp_path_factory): source_path = tmp_path_factory.mktemp("dbt_project") - dbt_file = source_path / "dbt_project.yml" - dbt_file.touch() + dbt_project_file = source_path / "dbt_project.yml" + dbt_project_file.write_text(yaml.dump({"profile": "dev"})) + dbt_profiles_file = source_path / "profiles.yml" + dbt_profiles_file.write_text( + yaml.dump( + { + "dev": { + "outputs": { + "local": {"account": "test_account", "database": "testdb"} + } + } + }, + ) + ) yield source_path @pytest.fixture @@ -105,47 +118,41 @@ def test_deploys_project_from_source( dbt_project_path, "@MockDatabase.MockSchema.dbt_TEST_PIPELINE_stage" ) + @pytest.mark.parametrize("exists", (True, False)) @mock.patch("snowflake.cli._plugins.dbt.manager.StageManager.put_recursive") @mock.patch("snowflake.cli._plugins.dbt.manager.StageManager.create") - def test_dbt_version_from_option_has_precedence_over_file( + def test_force_flag_uses_create_or_replace( self, _mock_create, _mock_put_recursive, + exists, mock_connect, runner, dbt_project_path, mock_exists, ): + mock_exists.return_value = exists + result = runner.invoke( [ "dbt", "deploy", "TEST_PIPELINE", f"--source={dbt_project_path}", + "--force", ] ) assert result.exit_code == 0, result.output - assert ( - mock_connect.mocked_ctx.get_query() - == """CREATE DBT PROJECT TEST_PIPELINE -FROM @MockDatabase.MockSchema.dbt_TEST_PIPELINE_stage""" + assert mock_connect.mocked_ctx.get_query().startswith( + "CREATE OR REPLACE DBT PROJECT" ) - @pytest.mark.parametrize("exists", (True, False)) - @mock.patch("snowflake.cli._plugins.dbt.manager.StageManager.put_recursive") - @mock.patch("snowflake.cli._plugins.dbt.manager.StageManager.create") - def test_force_flag_uses_create_or_replace( - self, - _mock_create, - _mock_put_recursive, - exists, - mock_connect, - runner, - dbt_project_path, - mock_exists, + def test_raises_when_dbt_project_yml_is_not_available( + self, dbt_project_path, mock_connect, runner ): - mock_exists.return_value = exists + dbt_file = dbt_project_path / "dbt_project.yml" + dbt_file.unlink() result = runner.invoke( [ @@ -153,20 +160,36 @@ def test_force_flag_uses_create_or_replace( "deploy", "TEST_PIPELINE", f"--source={dbt_project_path}", - "--force", - ] + ], ) - assert result.exit_code == 0, result.output - assert mock_connect.mocked_ctx.get_query().startswith( - "CREATE OR REPLACE DBT PROJECT" + assert result.exit_code == 1, result.output + assert f"dbt_project.yml does not exist in directory" in result.output + assert mock_connect.mocked_ctx.get_query() == "" + + def test_raises_when_dbt_project_yml_does_not_specify_profile( + self, dbt_project_path, mock_connect, runner + ): + with open((dbt_project_path / "dbt_project.yml"), "w") as f: + yaml.dump({}, f) + + result = runner.invoke( + [ + "dbt", + "deploy", + "TEST_PIPELINE", + f"--source={dbt_project_path}", + ], ) - def test_raises_when_dbt_project_is_not_available( + assert result.exit_code == 1, result.output + assert "`profile` is not defined in dbt_project.yml" in result.output + assert mock_connect.mocked_ctx.get_query() == "" + + def test_raises_when_profiles_yml_is_not_available( self, dbt_project_path, mock_connect, runner ): - dbt_file = dbt_project_path / "dbt_project.yml" - dbt_file.unlink() + (dbt_project_path / "profiles.yml").unlink() result = runner.invoke( [ @@ -178,10 +201,29 @@ def test_raises_when_dbt_project_is_not_available( ) assert result.exit_code == 1, result.output - assert f"dbt_project.yml does not exist in directory" in result.output + assert f"profiles.yml does not exist in directory" in result.output + assert mock_connect.mocked_ctx.get_query() == "" + + def test_raises_when_profiles_yml_does_not_contain_selected_profile( + self, dbt_project_path, mock_connect, runner + ): + with open((dbt_project_path / "profiles.yml"), "w") as f: + yaml.dump({}, f) + + result = runner.invoke( + [ + "dbt", + "deploy", + "TEST_PIPELINE", + f"--source={dbt_project_path}", + ], + ) + + assert result.exit_code == 1, result.output + assert "profile dev is not defined in profiles.yml" in result.output assert mock_connect.mocked_ctx.get_query() == "" - def test_raises_when_dbt_project_exists_and_is_not_force( + def test_raises_when_dbt_object_exists_and_is_not_force( self, dbt_project_path, mock_connect, runner, mock_exists ): mock_exists.return_value = True diff --git a/tests_integration/test_data/projects/dbt_project/profiles.yml b/tests_integration/test_data/projects/dbt_project/profiles.yml new file mode 100644 index 0000000000..77a42e03e4 --- /dev/null +++ b/tests_integration/test_data/projects/dbt_project/profiles.yml @@ -0,0 +1,5 @@ +dbt_integration_project: + target: dev + outputs: + dev: + type: snowflake diff --git a/tests_integration/test_dbt.py b/tests_integration/test_dbt.py index 4eec850179..0d33f8bd82 100644 --- a/tests_integration/test_dbt.py +++ b/tests_integration/test_dbt.py @@ -14,20 +14,22 @@ import datetime import pytest +import yaml @pytest.mark.integration @pytest.mark.qa_only -def test_dbt_deploy( +def test_dbt( runner, snowflake_session, test_database, project_directory, ): - with project_directory("dbt_project"): + with project_directory("dbt_project") as root_dir: # Given a local dbt project ts = int(datetime.datetime.now().timestamp()) name = f"dbt_project_{ts}" + _setup_dbt_profile(root_dir, snowflake_session) # When it's deployed result = runner.invoke_with_connection_json(["dbt", "deploy", name]) @@ -65,3 +67,16 @@ def test_dbt_deploy( ) assert len(result.json) == 1, result.json assert result.json[0]["COUNT"] == 1, result.json[0] + + +def _setup_dbt_profile(root_dir, snowflake_session): + with open((root_dir / "profiles.yml"), "r") as f: + profiles = yaml.safe_load(f) + dev_profile = profiles["dbt_integration_project"]["outputs"]["dev"] + dev_profile["database"] = snowflake_session.database + dev_profile["account"] = snowflake_session.account + dev_profile["user"] = snowflake_session.user + dev_profile["role"] = snowflake_session.role + dev_profile["warehouse"] = snowflake_session.warehouse + dev_profile["schema"] = snowflake_session.schema + (root_dir / "profiles.yml").write_text(yaml.dump(profiles)) From ef916e661ed9c192943a613b1f3ae8c5f6bb9042 Mon Sep 17 00:00:00 2001 From: Jakub Wilkowski Date: Thu, 17 Apr 2025 17:02:24 +0200 Subject: [PATCH 51/54] Jw/snow 2045471 add flag for selecting profile directory (#2210) feat: [SNOW-2045471] add --profile-dir flag and more strict validations for profiles.yml --- src/snowflake/cli/__about__.py | 2 +- src/snowflake/cli/_plugins/dbt/commands.py | 14 +- src/snowflake/cli/_plugins/dbt/manager.py | 81 +++++++++-- .../cli/_plugins/snowpark/snowpark_entity.py | 2 +- src/snowflake/cli/api/secure_path.py | 5 + tests/__snapshots__/test_help_messages.ambr | 46 ++++++- tests/dbt/test_dbt_commands.py | 45 ++++++- tests/dbt/test_manager.py | 127 ++++++++++++++++++ tests_integration/__snapshots__/test_dbt.ambr | 11 ++ tests_integration/test_dbt.py | 42 +++++- 10 files changed, 341 insertions(+), 34 deletions(-) create mode 100644 tests/dbt/test_manager.py create mode 100644 tests_integration/__snapshots__/test_dbt.ambr diff --git a/src/snowflake/cli/__about__.py b/src/snowflake/cli/__about__.py index 82769b09fe..dd44e4c5b4 100644 --- a/src/snowflake/cli/__about__.py +++ b/src/snowflake/cli/__about__.py @@ -16,7 +16,7 @@ from enum import Enum, unique -VERSION = "3.7.0.dev+dbt0" +VERSION = "3.8.0.dev+dbt0" @unique diff --git a/src/snowflake/cli/_plugins/dbt/commands.py b/src/snowflake/cli/_plugins/dbt/commands.py index fb1b1def42..8043dfd705 100644 --- a/src/snowflake/cli/_plugins/dbt/commands.py +++ b/src/snowflake/cli/_plugins/dbt/commands.py @@ -75,6 +75,11 @@ def deploy_dbt( show_default=False, default=None, ), + profiles_dir: Optional[str] = typer.Option( + help="Path to directory containing profiles.yml. Defaults to directory provided in --source or current working directory", + show_default=False, + default=None, + ), force: Optional[bool] = typer.Option( False, help="Overwrites conflicting files in the project, if any.", @@ -84,14 +89,13 @@ def deploy_dbt( """ Copy dbt files and create or update dbt on Snowflake project. """ - if source is None: - path = SecurePath.cwd() - else: - path = SecurePath(source) + project_path = SecurePath(source) if source is not None else SecurePath.cwd() + profiles_dir_path = SecurePath(profiles_dir) if profiles_dir else project_path return QueryResult( DBTManager().deploy( - path.resolve(), name, + project_path.resolve(), + profiles_dir_path.resolve(), force=force, ) ) diff --git a/src/snowflake/cli/_plugins/dbt/manager.py b/src/snowflake/cli/_plugins/dbt/manager.py index 305f02b2d6..e06cd161b3 100644 --- a/src/snowflake/cli/_plugins/dbt/manager.py +++ b/src/snowflake/cli/_plugins/dbt/manager.py @@ -14,6 +14,8 @@ from __future__ import annotations +from collections import defaultdict + import yaml from snowflake.cli._plugins.object.manager import ObjectManager from snowflake.cli._plugins.stage.manager import StageManager @@ -39,8 +41,9 @@ def exists(name: FQN) -> bool: def deploy( self, - path: SecurePath, name: FQN, + path: SecurePath, + profiles_path: SecurePath, force: bool, ) -> SnowflakeCursor: dbt_project_path = path / "dbt_project.yml" @@ -56,16 +59,7 @@ def deploy( except KeyError: raise CliError("`profile` is not defined in dbt_project.yml") - dbt_profiles_path = path / "profiles.yml" - if not dbt_profiles_path.exists(): - raise CliError( - f"profiles.yml does not exist in directory {path.path.absolute()}." - ) - - with dbt_profiles_path.open(read_file_limit_mb=DEFAULT_SIZE_LIMIT_MB) as fd: - profiles = yaml.safe_load(fd) - if profile not in profiles: - raise CliError(f"profile {profile} is not defined in profiles.yml") + self._validate_profiles(profiles_path, profile) if self.exists(name=name) and force is not True: raise CliError( @@ -79,8 +73,13 @@ def deploy( stage_manager.create(stage_fqn, temporary=True) with cli_console.phase("Copying project files to stage"): - results = list(stage_manager.put_recursive(path.path, stage_name)) - cli_console.step(f"Copied {len(results)} files") + result_count = len(list(stage_manager.put_recursive(path.path, stage_name))) + if profiles_path != path: + stage_manager.put( + str((profiles_path.path / "profiles.yml").absolute()), stage_name + ) + result_count += 1 + cli_console.step(f"Copied {result_count} files") with cli_console.phase("Creating DBT project"): query = f"""{'CREATE OR REPLACE' if force is True else 'CREATE'} DBT PROJECT {name} @@ -88,6 +87,62 @@ def deploy( return self.execute_query(query) + @staticmethod + def _validate_profiles(profiles_path: SecurePath, target_profile: str) -> None: + """ + Validates that: + * profiles.yml exists + * contain profile specified in dbt_project.yml + * no other profiles are defined there + * does not contain any confidential data like passwords + """ + profiles_file = profiles_path / "profiles.yml" + if not profiles_file.exists(): + raise CliError( + f"profiles.yml does not exist in directory {profiles_path.path.absolute()}." + ) + with profiles_file.open(read_file_limit_mb=DEFAULT_SIZE_LIMIT_MB) as fd: + profiles = yaml.safe_load(fd) + + if target_profile not in profiles: + raise CliError(f"profile {target_profile} is not defined in profiles.yml") + + errors = defaultdict(list) + if len(profiles.keys()) > 1: + for profile_name in profiles.keys(): + if profile_name.lower() != target_profile.lower(): + errors[profile_name].append("Remove unnecessary profiles") + + supported_keys = { + "database", + "account", + "type", + "user", + "role", + "warehouse", + "schema", + } + for target_name, target in profiles[target_profile]["outputs"].items(): + if missing_keys := supported_keys - set(target.keys()): + errors[target_profile].append( + f"Missing required fields: {', '.join(sorted(missing_keys))} in target {target_name}" + ) + if unsupported_keys := set(target.keys()) - supported_keys: + errors[target_profile].append( + f"Unsupported fields found: {', '.join(sorted(unsupported_keys))} in target {target_name}" + ) + if "type" in target and target["type"].lower() != "snowflake": + errors[target_profile].append( + f"Value for type field is invalid. Should be set to `snowflake` in target {target_name}" + ) + + if errors: + message = "Found following errors in profiles.yml. Please fix them before proceeding:" + for target, issues in errors.items(): + message += f"\n{target}" + message += "\n * " + "\n * ".join(issues) + raise CliError(message) + def execute( self, dbt_command: str, name: str, run_async: bool, *dbt_cli_args ) -> SnowflakeCursor: diff --git a/src/snowflake/cli/_plugins/snowpark/snowpark_entity.py b/src/snowflake/cli/_plugins/snowpark/snowpark_entity.py index 7ff04062cb..e1f9c25da1 100644 --- a/src/snowflake/cli/_plugins/snowpark/snowpark_entity.py +++ b/src/snowflake/cli/_plugins/snowpark/snowpark_entity.py @@ -231,7 +231,7 @@ def _process_requirements( # TODO: maybe leave all the logic with requirements ) zip_dir( - source=tmp_dir, + source=tmp_dir.path, dest_zip=bundle_dir / archive_name, ) diff --git a/src/snowflake/cli/api/secure_path.py b/src/snowflake/cli/api/secure_path.py index 3891081189..0ffeada1bf 100644 --- a/src/snowflake/cli/api/secure_path.py +++ b/src/snowflake/cli/api/secure_path.py @@ -46,6 +46,11 @@ def __repr__(self): def __truediv__(self, key): return SecurePath(self._path / key) + def __eq__(self, other): + if isinstance(other, Path): + return self.path == other + return self.path == other.path + @property def path(self) -> Path: """ diff --git a/tests/__snapshots__/test_help_messages.ambr b/tests/__snapshots__/test_help_messages.ambr index 1d2c643dae..d1d6ae4c6b 100644 --- a/tests/__snapshots__/test_help_messages.ambr +++ b/tests/__snapshots__/test_help_messages.ambr @@ -3978,13 +3978,17 @@ | [required] | +------------------------------------------------------------------------------+ +- Options --------------------------------------------------------------------+ - | --source TEXT Path to directory containing dbt files to | - | deploy. Defaults to current working | - | directory. | - | --force --no-force Overwrites conflicting files in the | - | project, if any. | - | [default: no-force] | - | --help -h Show this message and exit. | + | --source TEXT Path to directory containing dbt | + | files to deploy. Defaults to current | + | working directory. | + | --profiles-dir TEXT Path to directory containing | + | profiles.yml. Defaults to directory | + | provided in --source or current | + | working directory | + | --force --no-force Overwrites conflicting files in the | + | project, if any. | + | [default: no-force] | + | --help -h Show this message and exit. | +------------------------------------------------------------------------------+ +- Connection configuration ---------------------------------------------------+ | --connection,--environment -c TEXT Name of the connection, as | @@ -4064,6 +4068,8 @@ | to console. | | --enhanced-exit-codes Differentiate exit error codes | | based on failure type. | + | [env var: | + | SNOWFLAKE_ENHANCED_EXIT_CODES] | +------------------------------------------------------------------------------+ @@ -4164,6 +4170,8 @@ | to console. | | --enhanced-exit-codes Differentiate exit error codes | | based on failure type. | + | [env var: | + | SNOWFLAKE_ENHANCED_EXIT_CODES] | +------------------------------------------------------------------------------+ +- Commands -------------------------------------------------------------------+ | build Execute build command on Snowflake. | @@ -4276,6 +4284,8 @@ | to console. | | --enhanced-exit-codes Differentiate exit error codes | | based on failure type. | + | [env var: | + | SNOWFLAKE_ENHANCED_EXIT_CODES] | +------------------------------------------------------------------------------+ +- Commands -------------------------------------------------------------------+ | build Execute build command on Snowflake. | @@ -4388,6 +4398,8 @@ | to console. | | --enhanced-exit-codes Differentiate exit error codes | | based on failure type. | + | [env var: | + | SNOWFLAKE_ENHANCED_EXIT_CODES] | +------------------------------------------------------------------------------+ +- Commands -------------------------------------------------------------------+ | build Execute build command on Snowflake. | @@ -4500,6 +4512,8 @@ | to console. | | --enhanced-exit-codes Differentiate exit error codes | | based on failure type. | + | [env var: | + | SNOWFLAKE_ENHANCED_EXIT_CODES] | +------------------------------------------------------------------------------+ +- Commands -------------------------------------------------------------------+ | build Execute build command on Snowflake. | @@ -4612,6 +4626,8 @@ | to console. | | --enhanced-exit-codes Differentiate exit error codes | | based on failure type. | + | [env var: | + | SNOWFLAKE_ENHANCED_EXIT_CODES] | +------------------------------------------------------------------------------+ +- Commands -------------------------------------------------------------------+ | build Execute build command on Snowflake. | @@ -4724,6 +4740,8 @@ | to console. | | --enhanced-exit-codes Differentiate exit error codes | | based on failure type. | + | [env var: | + | SNOWFLAKE_ENHANCED_EXIT_CODES] | +------------------------------------------------------------------------------+ +- Commands -------------------------------------------------------------------+ | build Execute build command on Snowflake. | @@ -4836,6 +4854,8 @@ | to console. | | --enhanced-exit-codes Differentiate exit error codes | | based on failure type. | + | [env var: | + | SNOWFLAKE_ENHANCED_EXIT_CODES] | +------------------------------------------------------------------------------+ +- Commands -------------------------------------------------------------------+ | build Execute build command on Snowflake. | @@ -4948,6 +4968,8 @@ | to console. | | --enhanced-exit-codes Differentiate exit error codes | | based on failure type. | + | [env var: | + | SNOWFLAKE_ENHANCED_EXIT_CODES] | +------------------------------------------------------------------------------+ +- Commands -------------------------------------------------------------------+ | build Execute build command on Snowflake. | @@ -5060,6 +5082,8 @@ | to console. | | --enhanced-exit-codes Differentiate exit error codes | | based on failure type. | + | [env var: | + | SNOWFLAKE_ENHANCED_EXIT_CODES] | +------------------------------------------------------------------------------+ +- Commands -------------------------------------------------------------------+ | build Execute build command on Snowflake. | @@ -5172,6 +5196,8 @@ | to console. | | --enhanced-exit-codes Differentiate exit error codes | | based on failure type. | + | [env var: | + | SNOWFLAKE_ENHANCED_EXIT_CODES] | +------------------------------------------------------------------------------+ +- Commands -------------------------------------------------------------------+ | build Execute build command on Snowflake. | @@ -5284,6 +5310,8 @@ | to console. | | --enhanced-exit-codes Differentiate exit error codes | | based on failure type. | + | [env var: | + | SNOWFLAKE_ENHANCED_EXIT_CODES] | +------------------------------------------------------------------------------+ +- Commands -------------------------------------------------------------------+ | build Execute build command on Snowflake. | @@ -5399,6 +5427,8 @@ | to console. | | --enhanced-exit-codes Differentiate exit error codes | | based on failure type. | + | [env var: | + | SNOWFLAKE_ENHANCED_EXIT_CODES] | +------------------------------------------------------------------------------+ @@ -15948,6 +15978,8 @@ | to console. | | --enhanced-exit-codes Differentiate exit error codes | | based on failure type. | + | [env var: | + | SNOWFLAKE_ENHANCED_EXIT_CODES] | +------------------------------------------------------------------------------+ +- Commands -------------------------------------------------------------------+ | build Execute build command on Snowflake. | diff --git a/tests/dbt/test_dbt_commands.py b/tests/dbt/test_dbt_commands.py index 2c01fd6f53..d0c81bacd4 100644 --- a/tests/dbt/test_dbt_commands.py +++ b/tests/dbt/test_dbt_commands.py @@ -14,6 +14,7 @@ from __future__ import annotations +from pathlib import Path from unittest import mock import pytest @@ -65,7 +66,15 @@ def dbt_project_path(self, tmp_path_factory): { "dev": { "outputs": { - "local": {"account": "test_account", "database": "testdb"} + "local": { + "account": "test_account", + "database": "testdb", + "role": "test_role", + "schema": "test_schema", + "type": "snowflake", + "user": "test_user", + "warehouse": "test_warehouse", + } } } }, @@ -148,6 +157,40 @@ def test_force_flag_uses_create_or_replace( "CREATE OR REPLACE DBT PROJECT" ) + @mock.patch("snowflake.cli._plugins.dbt.manager.StageManager.put_recursive") + @mock.patch("snowflake.cli._plugins.dbt.manager.StageManager.put") + @mock.patch("snowflake.cli._plugins.dbt.manager.StageManager.create") + def test_dbt_deploy_with_custom_profiles_dir( + self, + _mock_create, + mock_put, + _mock_put_recursive, + mock_connect, + runner, + dbt_project_path, + mock_exists, + ): + new_profiles_directory = Path(dbt_project_path) / "dbt_profiles" + new_profiles_directory.mkdir(parents=True, exist_ok=True) + profiles_file = dbt_project_path / "profiles.yml" + profiles_file.rename(new_profiles_directory / "profiles.yml") + + result = runner.invoke( + [ + "dbt", + "deploy", + "TEST_PIPELINE", + f"--source={dbt_project_path}", + f"--profiles-dir={new_profiles_directory}", + ] + ) + + assert result.exit_code == 0, result.output + mock_put.assert_called_once_with( + str(new_profiles_directory / "profiles.yml"), + "@MockDatabase.MockSchema.dbt_TEST_PIPELINE_stage", + ) + def test_raises_when_dbt_project_yml_is_not_available( self, dbt_project_path, mock_connect, runner ): diff --git a/tests/dbt/test_manager.py b/tests/dbt/test_manager.py new file mode 100644 index 0000000000..84c479faee --- /dev/null +++ b/tests/dbt/test_manager.py @@ -0,0 +1,127 @@ +from textwrap import dedent + +import pytest +import yaml +from snowflake.cli._plugins.dbt.manager import DBTManager +from snowflake.cli.api.exceptions import CliError +from snowflake.cli.api.secure_path import SecurePath + + +class TestDeploy: + @pytest.fixture() + def profile(self): + return { + "dev": { + "outputs": { + "local": { + "account": "test_account", + "database": "testdb", + "role": "test_role", + "schema": "test_schema", + "type": "snowflake", + "user": "test_user", + "warehouse": "test_warehouse", + } + } + } + } + + @pytest.fixture + def project_path(self, tmp_path_factory): + source_path = tmp_path_factory.mktemp("dbt_project") + yield source_path + + def _generate_profile(self, project_path, profile): + dbt_profiles_file = project_path / "profiles.yml" + dbt_profiles_file.write_text(yaml.dump(profile)) + + def test_validate_profiles_raises_when_file_does_not_exist(self, project_path): + + with pytest.raises(CliError) as exc_info: + DBTManager._validate_profiles( # noqa: SLF001 + SecurePath(project_path), "dev" + ) + + assert ( + exc_info.value.message + == f"profiles.yml does not exist in directory {project_path.absolute()}." + ) + + def test_validate_profiles_raises_when_profile_is_not_in_the_file( + self, project_path, profile + ): + self._generate_profile(project_path, profile) + + with pytest.raises(CliError) as exc_info: + DBTManager._validate_profiles( # noqa: SLF001 + SecurePath(project_path), "another_profile_name" + ) + + assert ( + exc_info.value.message + == "profile another_profile_name is not defined in profiles.yml" + ) + + def test_validate_profiles_raises_when_extra_profiles_are_defined( + self, project_path, profile + ): + profile["another_profile"] = {} + self._generate_profile(project_path, profile) + + with pytest.raises(CliError) as exc_info: + DBTManager._validate_profiles( # noqa: SLF001 + SecurePath(project_path), "dev" + ) + + expected_error_message = """Found following errors in profiles.yml. Please fix them before proceeding: +another_profile + * Remove unnecessary profiles""" + assert exc_info.value.message == dedent(expected_error_message) + + def test_validate_profiles_raises_when_required_fields_are_missing( + self, project_path, profile + ): + profile["dev"]["outputs"]["local"].pop("warehouse", None) + profile["dev"]["outputs"]["local"].pop("role", None) + self._generate_profile(project_path, profile) + + with pytest.raises(CliError) as exc_info: + DBTManager._validate_profiles( # noqa: SLF001 + SecurePath(project_path), "dev" + ) + + expected_error_message = """Found following errors in profiles.yml. Please fix them before proceeding: +dev + * Missing required fields: role, warehouse in target local""" + assert exc_info.value.message == dedent(expected_error_message) + + def test_validate_profiles_raises_when_unsupported_fields_are_provided( + self, project_path, profile + ): + profile["dev"]["outputs"]["local"]["password"] = "very secret password" + self._generate_profile(project_path, profile) + + with pytest.raises(CliError) as exc_info: + DBTManager._validate_profiles( # noqa: SLF001 + SecurePath(project_path), "dev" + ) + + expected_error_message = """Found following errors in profiles.yml. Please fix them before proceeding: +dev + * Unsupported fields found: password in target local""" + assert exc_info.value.message == dedent(expected_error_message) + assert "very secret password" not in exc_info.value.message + + def test_validate_profiles_raises_when_type_is_wrong(self, project_path, profile): + profile["dev"]["outputs"]["local"]["type"] = "sqlite" + self._generate_profile(project_path, profile) + + with pytest.raises(CliError) as exc_info: + DBTManager._validate_profiles( # noqa: SLF001 + SecurePath(project_path), "dev" + ) + + expected_error_message = """Found following errors in profiles.yml. Please fix them before proceeding: +dev + * Value for type field is invalid. Should be set to `snowflake` in target local""" + assert exc_info.value.message == dedent(expected_error_message) diff --git a/tests_integration/__snapshots__/test_dbt.ambr b/tests_integration/__snapshots__/test_dbt.ambr new file mode 100644 index 0000000000..24f56f7e38 --- /dev/null +++ b/tests_integration/__snapshots__/test_dbt.ambr @@ -0,0 +1,11 @@ +# serializer version: 1 +# name: test_dbt + ''' + ╭─ Error ──────────────────────────────────────────────────────────────────────╮ + │ Found following errors in profiles.yml. Please fix them before proceeding: │ + │ dbt_integration_project │ + │ * Unsupported fields found: password in target dev │ + ╰──────────────────────────────────────────────────────────────────────────────╯ + + ''' +# --- diff --git a/tests_integration/test_dbt.py b/tests_integration/test_dbt.py index 0d33f8bd82..1d220f252f 100644 --- a/tests_integration/test_dbt.py +++ b/tests_integration/test_dbt.py @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. import datetime +from pathlib import Path import pytest import yaml @@ -24,18 +25,43 @@ def test_dbt( snowflake_session, test_database, project_directory, + snapshot, ): with project_directory("dbt_project") as root_dir: # Given a local dbt project ts = int(datetime.datetime.now().timestamp()) name = f"dbt_project_{ts}" - _setup_dbt_profile(root_dir, snowflake_session) - # When it's deployed + # try to deploy, but fail since profiles.yml contains a password + _setup_dbt_profile(root_dir, snowflake_session, include_password=True) result = runner.invoke_with_connection_json(["dbt", "deploy", name]) + assert result.exit_code == 1, result.output + assert result.output == snapshot + + # deploy for the first time + _setup_dbt_profile(root_dir, snowflake_session, include_password=False) + result = runner.invoke_with_connection_json(["dbt", "deploy", name]) + assert result.exit_code == 0, result.output + + # change location of profiles.yml and redeploy + new_profiles_directory = Path(root_dir) / "dbt_profiles" + new_profiles_directory.mkdir(parents=True, exist_ok=True) + profiles_file = root_dir / "profiles.yml" + profiles_file.rename(new_profiles_directory / "profiles.yml") + + result = runner.invoke_with_connection_json( + [ + "dbt", + "deploy", + name, + "--force", + "--profiles-dir", + str(new_profiles_directory.resolve()), + ] + ) assert result.exit_code == 0, result.output - # Then it can be listed + # list all dbt objects result = runner.invoke_with_connection_json( [ "dbt", @@ -49,7 +75,7 @@ def test_dbt( dbt_object = result.json[0] assert dbt_object["name"].lower() == name.lower() - # And when dbt run gets called on it + # call `run` on dbt object result = runner.invoke_passthrough_with_connection( args=[ "dbt", @@ -58,7 +84,7 @@ def test_dbt( passthrough_args=[name, "run"], ) - # Then is succeeds and models get populated according to expectations + # a successful execution should produce data in my_second_dbt_model and assert result.exit_code == 0, result.output assert "Done. PASS=2 WARN=0 ERROR=0 SKIP=0 TOTAL=2" in result.output @@ -69,7 +95,7 @@ def test_dbt( assert result.json[0]["COUNT"] == 1, result.json[0] -def _setup_dbt_profile(root_dir, snowflake_session): +def _setup_dbt_profile(root_dir: Path, snowflake_session, include_password: bool): with open((root_dir / "profiles.yml"), "r") as f: profiles = yaml.safe_load(f) dev_profile = profiles["dbt_integration_project"]["outputs"]["dev"] @@ -79,4 +105,8 @@ def _setup_dbt_profile(root_dir, snowflake_session): dev_profile["role"] = snowflake_session.role dev_profile["warehouse"] = snowflake_session.warehouse dev_profile["schema"] = snowflake_session.schema + if include_password: + dev_profile["password"] = "secret_phrase" + else: + dev_profile.pop("password", None) (root_dir / "profiles.yml").write_text(yaml.dump(profiles)) From 2f1a7c45615fd4ddefa9afeb2706f99d7c3dcd35 Mon Sep 17 00:00:00 2001 From: Jakub Wilkowski Date: Tue, 13 May 2025 09:03:25 +0200 Subject: [PATCH 52/54] feat: [SNOW-2048251] enable dbt run-operation (#2261) --- src/snowflake/cli/_plugins/dbt/commands.py | 11 +- src/snowflake/cli/_plugins/dbt/constants.py | 1 + tests/__snapshots__/test_help_messages.ambr | 367 +++++++++++++------- 3 files changed, 256 insertions(+), 123 deletions(-) diff --git a/src/snowflake/cli/_plugins/dbt/commands.py b/src/snowflake/cli/_plugins/dbt/commands.py index 8043dfd705..e8a03b97be 100644 --- a/src/snowflake/cli/_plugins/dbt/commands.py +++ b/src/snowflake/cli/_plugins/dbt/commands.py @@ -18,7 +18,7 @@ from typing import Optional import typer -from click import ClickException +from click import ClickException, types from rich.progress import Progress, SpinnerColumn, TextColumn from snowflake.cli._plugins.dbt.constants import ( DBT_COMMANDS, @@ -51,6 +51,11 @@ DBTNameArgument = identifier_argument(sf_object="DBT Project", example="my_pipeline") +# in passthrough commands we need to support that user would either provide the name of dbt object or name of dbt +# command, in which case FQN validation could fail +DBTNameOrCommandArgument = identifier_argument( + sf_object="DBT Project", example="my_pipeline", click_type=types.StringParamType() +) add_object_command_aliases( app=app, @@ -114,7 +119,7 @@ def deploy_dbt( @dbt_execute_app.callback() @global_options_with_connection def before_callback( - name: FQN = DBTNameArgument, + name: str = DBTNameOrCommandArgument, run_async: Optional[bool] = typer.Option( False, help="Run dbt command asynchronously and check it's result later." ), @@ -139,7 +144,7 @@ def _dbt_execute( ) -> CommandResult: dbt_cli_args = ctx.args dbt_command = ctx.command.name - name = ctx.parent.params["name"] + name = FQN.from_string(ctx.parent.params["name"]) run_async = ctx.parent.params["run_async"] execute_args = (dbt_command, name, run_async, *dbt_cli_args) dbt_manager = DBTManager() diff --git a/src/snowflake/cli/_plugins/dbt/constants.py b/src/snowflake/cli/_plugins/dbt/constants.py index 4c79dfa81c..b530a2f19e 100644 --- a/src/snowflake/cli/_plugins/dbt/constants.py +++ b/src/snowflake/cli/_plugins/dbt/constants.py @@ -22,6 +22,7 @@ "list", "parse", "run", + "run-operation", "seed", "show", "snapshot", diff --git a/tests/__snapshots__/test_help_messages.ambr b/tests/__snapshots__/test_help_messages.ambr index d1d6ae4c6b..5406935407 100644 --- a/tests/__snapshots__/test_help_messages.ambr +++ b/tests/__snapshots__/test_help_messages.ambr @@ -4174,16 +4174,17 @@ | SNOWFLAKE_ENHANCED_EXIT_CODES] | +------------------------------------------------------------------------------+ +- Commands -------------------------------------------------------------------+ - | build Execute build command on Snowflake. | - | compile Execute compile command on Snowflake. | - | deps Execute deps command on Snowflake. | - | list Execute list command on Snowflake. | - | parse Execute parse command on Snowflake. | - | run Execute run command on Snowflake. | - | seed Execute seed command on Snowflake. | - | show Execute show command on Snowflake. | - | snapshot Execute snapshot command on Snowflake. | - | test Execute test command on Snowflake. | + | build Execute build command on Snowflake. | + | compile Execute compile command on Snowflake. | + | deps Execute deps command on Snowflake. | + | list Execute list command on Snowflake. | + | parse Execute parse command on Snowflake. | + | run Execute run command on Snowflake. | + | run-operation Execute run-operation command on Snowflake. | + | seed Execute seed command on Snowflake. | + | show Execute show command on Snowflake. | + | snapshot Execute snapshot command on Snowflake. | + | test Execute test command on Snowflake. | +------------------------------------------------------------------------------+ @@ -4288,16 +4289,17 @@ | SNOWFLAKE_ENHANCED_EXIT_CODES] | +------------------------------------------------------------------------------+ +- Commands -------------------------------------------------------------------+ - | build Execute build command on Snowflake. | - | compile Execute compile command on Snowflake. | - | deps Execute deps command on Snowflake. | - | list Execute list command on Snowflake. | - | parse Execute parse command on Snowflake. | - | run Execute run command on Snowflake. | - | seed Execute seed command on Snowflake. | - | show Execute show command on Snowflake. | - | snapshot Execute snapshot command on Snowflake. | - | test Execute test command on Snowflake. | + | build Execute build command on Snowflake. | + | compile Execute compile command on Snowflake. | + | deps Execute deps command on Snowflake. | + | list Execute list command on Snowflake. | + | parse Execute parse command on Snowflake. | + | run Execute run command on Snowflake. | + | run-operation Execute run-operation command on Snowflake. | + | seed Execute seed command on Snowflake. | + | show Execute show command on Snowflake. | + | snapshot Execute snapshot command on Snowflake. | + | test Execute test command on Snowflake. | +------------------------------------------------------------------------------+ @@ -4402,16 +4404,17 @@ | SNOWFLAKE_ENHANCED_EXIT_CODES] | +------------------------------------------------------------------------------+ +- Commands -------------------------------------------------------------------+ - | build Execute build command on Snowflake. | - | compile Execute compile command on Snowflake. | - | deps Execute deps command on Snowflake. | - | list Execute list command on Snowflake. | - | parse Execute parse command on Snowflake. | - | run Execute run command on Snowflake. | - | seed Execute seed command on Snowflake. | - | show Execute show command on Snowflake. | - | snapshot Execute snapshot command on Snowflake. | - | test Execute test command on Snowflake. | + | build Execute build command on Snowflake. | + | compile Execute compile command on Snowflake. | + | deps Execute deps command on Snowflake. | + | list Execute list command on Snowflake. | + | parse Execute parse command on Snowflake. | + | run Execute run command on Snowflake. | + | run-operation Execute run-operation command on Snowflake. | + | seed Execute seed command on Snowflake. | + | show Execute show command on Snowflake. | + | snapshot Execute snapshot command on Snowflake. | + | test Execute test command on Snowflake. | +------------------------------------------------------------------------------+ @@ -4516,16 +4519,17 @@ | SNOWFLAKE_ENHANCED_EXIT_CODES] | +------------------------------------------------------------------------------+ +- Commands -------------------------------------------------------------------+ - | build Execute build command on Snowflake. | - | compile Execute compile command on Snowflake. | - | deps Execute deps command on Snowflake. | - | list Execute list command on Snowflake. | - | parse Execute parse command on Snowflake. | - | run Execute run command on Snowflake. | - | seed Execute seed command on Snowflake. | - | show Execute show command on Snowflake. | - | snapshot Execute snapshot command on Snowflake. | - | test Execute test command on Snowflake. | + | build Execute build command on Snowflake. | + | compile Execute compile command on Snowflake. | + | deps Execute deps command on Snowflake. | + | list Execute list command on Snowflake. | + | parse Execute parse command on Snowflake. | + | run Execute run command on Snowflake. | + | run-operation Execute run-operation command on Snowflake. | + | seed Execute seed command on Snowflake. | + | show Execute show command on Snowflake. | + | snapshot Execute snapshot command on Snowflake. | + | test Execute test command on Snowflake. | +------------------------------------------------------------------------------+ @@ -4630,16 +4634,132 @@ | SNOWFLAKE_ENHANCED_EXIT_CODES] | +------------------------------------------------------------------------------+ +- Commands -------------------------------------------------------------------+ - | build Execute build command on Snowflake. | - | compile Execute compile command on Snowflake. | - | deps Execute deps command on Snowflake. | - | list Execute list command on Snowflake. | - | parse Execute parse command on Snowflake. | - | run Execute run command on Snowflake. | - | seed Execute seed command on Snowflake. | - | show Execute show command on Snowflake. | - | snapshot Execute snapshot command on Snowflake. | - | test Execute test command on Snowflake. | + | build Execute build command on Snowflake. | + | compile Execute compile command on Snowflake. | + | deps Execute deps command on Snowflake. | + | list Execute list command on Snowflake. | + | parse Execute parse command on Snowflake. | + | run Execute run command on Snowflake. | + | run-operation Execute run-operation command on Snowflake. | + | seed Execute seed command on Snowflake. | + | show Execute show command on Snowflake. | + | snapshot Execute snapshot command on Snowflake. | + | test Execute test command on Snowflake. | + +------------------------------------------------------------------------------+ + + + ''' +# --- +# name: test_help_messages[dbt.execute.run-operation] + ''' + + Usage: root dbt execute [OPTIONS] NAME DBT_COMMAND + + Execute a dbt command on Snowflake + + +- Arguments ------------------------------------------------------------------+ + | * name TEXT Identifier of the DBT Project; for example: my_pipeline | + | [required] | + +------------------------------------------------------------------------------+ + +- Options --------------------------------------------------------------------+ + | --run-async --no-run-async Run dbt command asynchronously and | + | check it's result later. | + | [default: no-run-async] | + | --help -h Show this message and exit. | + +------------------------------------------------------------------------------+ + +- Connection configuration ---------------------------------------------------+ + | --connection,--environment -c TEXT Name of the connection, as | + | defined in your config.toml | + | file. Default: default. | + | --host TEXT Host address for the | + | connection. Overrides the | + | value specified for the | + | connection. | + | --port INTEGER Port for the connection. | + | Overrides the value | + | specified for the | + | connection. | + | --account,--accountname TEXT Name assigned to your | + | Snowflake account. Overrides | + | the value specified for the | + | connection. | + | --user,--username TEXT Username to connect to | + | Snowflake. Overrides the | + | value specified for the | + | connection. | + | --password TEXT Snowflake password. | + | Overrides the value | + | specified for the | + | connection. | + | --authenticator TEXT Snowflake authenticator. | + | Overrides the value | + | specified for the | + | connection. | + | --private-key-file,--private… TEXT Snowflake private key file | + | path. Overrides the value | + | specified for the | + | connection. | + | --token-file-path TEXT Path to file with an OAuth | + | token to use when connecting | + | to Snowflake. | + | --database,--dbname TEXT Database to use. Overrides | + | the value specified for the | + | connection. | + | --schema,--schemaname TEXT Database schema to use. | + | Overrides the value | + | specified for the | + | connection. | + | --role,--rolename TEXT Role to use. Overrides the | + | value specified for the | + | connection. | + | --warehouse TEXT Warehouse to use. Overrides | + | the value specified for the | + | connection. | + | --temporary-connection -x Uses a connection defined | + | with command line | + | parameters, instead of one | + | defined in config | + | --mfa-passcode TEXT Token to use for | + | multi-factor authentication | + | (MFA) | + | --enable-diag Whether to generate a | + | connection diagnostic | + | report. | + | --diag-log-path TEXT Path for the generated | + | report. Defaults to system | + | temporary directory. | + | --diag-allowlist-path TEXT Path to a JSON file that | + | contains allowlist | + | parameters. | + +------------------------------------------------------------------------------+ + +- Global configuration -------------------------------------------------------+ + | --format [TABLE|JSON] Specifies the output format. | + | [default: TABLE] | + | --verbose -v Displays log entries for log | + | levels info and higher. | + | --debug Displays log entries for log | + | levels debug and higher; debug | + | logs contain additional | + | information. | + | --silent Turns off intermediate output | + | to console. | + | --enhanced-exit-codes Differentiate exit error codes | + | based on failure type. | + | [env var: | + | SNOWFLAKE_ENHANCED_EXIT_CODES] | + +------------------------------------------------------------------------------+ + +- Commands -------------------------------------------------------------------+ + | build Execute build command on Snowflake. | + | compile Execute compile command on Snowflake. | + | deps Execute deps command on Snowflake. | + | list Execute list command on Snowflake. | + | parse Execute parse command on Snowflake. | + | run Execute run command on Snowflake. | + | run-operation Execute run-operation command on Snowflake. | + | seed Execute seed command on Snowflake. | + | show Execute show command on Snowflake. | + | snapshot Execute snapshot command on Snowflake. | + | test Execute test command on Snowflake. | +------------------------------------------------------------------------------+ @@ -4744,16 +4864,17 @@ | SNOWFLAKE_ENHANCED_EXIT_CODES] | +------------------------------------------------------------------------------+ +- Commands -------------------------------------------------------------------+ - | build Execute build command on Snowflake. | - | compile Execute compile command on Snowflake. | - | deps Execute deps command on Snowflake. | - | list Execute list command on Snowflake. | - | parse Execute parse command on Snowflake. | - | run Execute run command on Snowflake. | - | seed Execute seed command on Snowflake. | - | show Execute show command on Snowflake. | - | snapshot Execute snapshot command on Snowflake. | - | test Execute test command on Snowflake. | + | build Execute build command on Snowflake. | + | compile Execute compile command on Snowflake. | + | deps Execute deps command on Snowflake. | + | list Execute list command on Snowflake. | + | parse Execute parse command on Snowflake. | + | run Execute run command on Snowflake. | + | run-operation Execute run-operation command on Snowflake. | + | seed Execute seed command on Snowflake. | + | show Execute show command on Snowflake. | + | snapshot Execute snapshot command on Snowflake. | + | test Execute test command on Snowflake. | +------------------------------------------------------------------------------+ @@ -4858,16 +4979,17 @@ | SNOWFLAKE_ENHANCED_EXIT_CODES] | +------------------------------------------------------------------------------+ +- Commands -------------------------------------------------------------------+ - | build Execute build command on Snowflake. | - | compile Execute compile command on Snowflake. | - | deps Execute deps command on Snowflake. | - | list Execute list command on Snowflake. | - | parse Execute parse command on Snowflake. | - | run Execute run command on Snowflake. | - | seed Execute seed command on Snowflake. | - | show Execute show command on Snowflake. | - | snapshot Execute snapshot command on Snowflake. | - | test Execute test command on Snowflake. | + | build Execute build command on Snowflake. | + | compile Execute compile command on Snowflake. | + | deps Execute deps command on Snowflake. | + | list Execute list command on Snowflake. | + | parse Execute parse command on Snowflake. | + | run Execute run command on Snowflake. | + | run-operation Execute run-operation command on Snowflake. | + | seed Execute seed command on Snowflake. | + | show Execute show command on Snowflake. | + | snapshot Execute snapshot command on Snowflake. | + | test Execute test command on Snowflake. | +------------------------------------------------------------------------------+ @@ -4972,16 +5094,17 @@ | SNOWFLAKE_ENHANCED_EXIT_CODES] | +------------------------------------------------------------------------------+ +- Commands -------------------------------------------------------------------+ - | build Execute build command on Snowflake. | - | compile Execute compile command on Snowflake. | - | deps Execute deps command on Snowflake. | - | list Execute list command on Snowflake. | - | parse Execute parse command on Snowflake. | - | run Execute run command on Snowflake. | - | seed Execute seed command on Snowflake. | - | show Execute show command on Snowflake. | - | snapshot Execute snapshot command on Snowflake. | - | test Execute test command on Snowflake. | + | build Execute build command on Snowflake. | + | compile Execute compile command on Snowflake. | + | deps Execute deps command on Snowflake. | + | list Execute list command on Snowflake. | + | parse Execute parse command on Snowflake. | + | run Execute run command on Snowflake. | + | run-operation Execute run-operation command on Snowflake. | + | seed Execute seed command on Snowflake. | + | show Execute show command on Snowflake. | + | snapshot Execute snapshot command on Snowflake. | + | test Execute test command on Snowflake. | +------------------------------------------------------------------------------+ @@ -5086,16 +5209,17 @@ | SNOWFLAKE_ENHANCED_EXIT_CODES] | +------------------------------------------------------------------------------+ +- Commands -------------------------------------------------------------------+ - | build Execute build command on Snowflake. | - | compile Execute compile command on Snowflake. | - | deps Execute deps command on Snowflake. | - | list Execute list command on Snowflake. | - | parse Execute parse command on Snowflake. | - | run Execute run command on Snowflake. | - | seed Execute seed command on Snowflake. | - | show Execute show command on Snowflake. | - | snapshot Execute snapshot command on Snowflake. | - | test Execute test command on Snowflake. | + | build Execute build command on Snowflake. | + | compile Execute compile command on Snowflake. | + | deps Execute deps command on Snowflake. | + | list Execute list command on Snowflake. | + | parse Execute parse command on Snowflake. | + | run Execute run command on Snowflake. | + | run-operation Execute run-operation command on Snowflake. | + | seed Execute seed command on Snowflake. | + | show Execute show command on Snowflake. | + | snapshot Execute snapshot command on Snowflake. | + | test Execute test command on Snowflake. | +------------------------------------------------------------------------------+ @@ -5200,16 +5324,17 @@ | SNOWFLAKE_ENHANCED_EXIT_CODES] | +------------------------------------------------------------------------------+ +- Commands -------------------------------------------------------------------+ - | build Execute build command on Snowflake. | - | compile Execute compile command on Snowflake. | - | deps Execute deps command on Snowflake. | - | list Execute list command on Snowflake. | - | parse Execute parse command on Snowflake. | - | run Execute run command on Snowflake. | - | seed Execute seed command on Snowflake. | - | show Execute show command on Snowflake. | - | snapshot Execute snapshot command on Snowflake. | - | test Execute test command on Snowflake. | + | build Execute build command on Snowflake. | + | compile Execute compile command on Snowflake. | + | deps Execute deps command on Snowflake. | + | list Execute list command on Snowflake. | + | parse Execute parse command on Snowflake. | + | run Execute run command on Snowflake. | + | run-operation Execute run-operation command on Snowflake. | + | seed Execute seed command on Snowflake. | + | show Execute show command on Snowflake. | + | snapshot Execute snapshot command on Snowflake. | + | test Execute test command on Snowflake. | +------------------------------------------------------------------------------+ @@ -5314,16 +5439,17 @@ | SNOWFLAKE_ENHANCED_EXIT_CODES] | +------------------------------------------------------------------------------+ +- Commands -------------------------------------------------------------------+ - | build Execute build command on Snowflake. | - | compile Execute compile command on Snowflake. | - | deps Execute deps command on Snowflake. | - | list Execute list command on Snowflake. | - | parse Execute parse command on Snowflake. | - | run Execute run command on Snowflake. | - | seed Execute seed command on Snowflake. | - | show Execute show command on Snowflake. | - | snapshot Execute snapshot command on Snowflake. | - | test Execute test command on Snowflake. | + | build Execute build command on Snowflake. | + | compile Execute compile command on Snowflake. | + | deps Execute deps command on Snowflake. | + | list Execute list command on Snowflake. | + | parse Execute parse command on Snowflake. | + | run Execute run command on Snowflake. | + | run-operation Execute run-operation command on Snowflake. | + | seed Execute seed command on Snowflake. | + | show Execute show command on Snowflake. | + | snapshot Execute snapshot command on Snowflake. | + | test Execute test command on Snowflake. | +------------------------------------------------------------------------------+ @@ -15982,16 +16108,17 @@ | SNOWFLAKE_ENHANCED_EXIT_CODES] | +------------------------------------------------------------------------------+ +- Commands -------------------------------------------------------------------+ - | build Execute build command on Snowflake. | - | compile Execute compile command on Snowflake. | - | deps Execute deps command on Snowflake. | - | list Execute list command on Snowflake. | - | parse Execute parse command on Snowflake. | - | run Execute run command on Snowflake. | - | seed Execute seed command on Snowflake. | - | show Execute show command on Snowflake. | - | snapshot Execute snapshot command on Snowflake. | - | test Execute test command on Snowflake. | + | build Execute build command on Snowflake. | + | compile Execute compile command on Snowflake. | + | deps Execute deps command on Snowflake. | + | list Execute list command on Snowflake. | + | parse Execute parse command on Snowflake. | + | run Execute run command on Snowflake. | + | run-operation Execute run-operation command on Snowflake. | + | seed Execute seed command on Snowflake. | + | show Execute show command on Snowflake. | + | snapshot Execute snapshot command on Snowflake. | + | test Execute test command on Snowflake. | +------------------------------------------------------------------------------+ From fc3f4d2df4b2f9e9dd4b0db94137e9a5d721fa52 Mon Sep 17 00:00:00 2001 From: Jakub Wilkowski Date: Wed, 14 May 2025 10:22:49 +0200 Subject: [PATCH 53/54] feat: [SNOW-2102782] support threads in profiles.yaml (#2291) --- src/snowflake/cli/_plugins/dbt/manager.py | 19 +++++++++++++------ tests/dbt/test_dbt_commands.py | 1 + tests/dbt/test_manager.py | 1 + .../projects/dbt_project/profiles.yml | 1 + 4 files changed, 16 insertions(+), 6 deletions(-) diff --git a/src/snowflake/cli/_plugins/dbt/manager.py b/src/snowflake/cli/_plugins/dbt/manager.py index e06cd161b3..ccd3c9e1c2 100644 --- a/src/snowflake/cli/_plugins/dbt/manager.py +++ b/src/snowflake/cli/_plugins/dbt/manager.py @@ -113,21 +113,28 @@ def _validate_profiles(profiles_path: SecurePath, target_profile: str) -> None: if profile_name.lower() != target_profile.lower(): errors[profile_name].append("Remove unnecessary profiles") - supported_keys = { - "database", + required_fields = { "account", + "database", + "role", + "schema", "type", "user", - "role", "warehouse", - "schema", + } + supported_fields = { + "threads", } for target_name, target in profiles[target_profile]["outputs"].items(): - if missing_keys := supported_keys - set(target.keys()): + if missing_keys := required_fields - set(target.keys()): errors[target_profile].append( f"Missing required fields: {', '.join(sorted(missing_keys))} in target {target_name}" ) - if unsupported_keys := set(target.keys()) - supported_keys: + if ( + unsupported_keys := set(target.keys()) + - required_fields + - supported_fields + ): errors[target_profile].append( f"Unsupported fields found: {', '.join(sorted(unsupported_keys))} in target {target_name}" ) diff --git a/tests/dbt/test_dbt_commands.py b/tests/dbt/test_dbt_commands.py index d0c81bacd4..468d9f62d1 100644 --- a/tests/dbt/test_dbt_commands.py +++ b/tests/dbt/test_dbt_commands.py @@ -71,6 +71,7 @@ def dbt_project_path(self, tmp_path_factory): "database": "testdb", "role": "test_role", "schema": "test_schema", + "threads": 2, "type": "snowflake", "user": "test_user", "warehouse": "test_warehouse", diff --git a/tests/dbt/test_manager.py b/tests/dbt/test_manager.py index 84c479faee..7669b0e7a5 100644 --- a/tests/dbt/test_manager.py +++ b/tests/dbt/test_manager.py @@ -18,6 +18,7 @@ def profile(self): "database": "testdb", "role": "test_role", "schema": "test_schema", + "threads": 4, "type": "snowflake", "user": "test_user", "warehouse": "test_warehouse", diff --git a/tests_integration/test_data/projects/dbt_project/profiles.yml b/tests_integration/test_data/projects/dbt_project/profiles.yml index 77a42e03e4..1452b36448 100644 --- a/tests_integration/test_data/projects/dbt_project/profiles.yml +++ b/tests_integration/test_data/projects/dbt_project/profiles.yml @@ -3,3 +3,4 @@ dbt_integration_project: outputs: dev: type: snowflake + threads: 2 From 9df0c65d3ea8b5e3f72925acbf11a31842348820 Mon Sep 17 00:00:00 2001 From: Jakub Wilkowski Date: Fri, 16 May 2025 16:50:46 +0200 Subject: [PATCH 54/54] feat: [SNOW-2103792] support altering existing dbt objects in deploy command (#2294) * feat: [SNOW-2103792] support altering existing dbt objects in deploy command * refactor: [SNOW-2103792] tiny reformat --- src/snowflake/cli/_plugins/dbt/manager.py | 15 +++-- tests/dbt/test_dbt_commands.py | 53 +++++++++-------- tests_integration/__snapshots__/test_dbt.ambr | 2 +- tests_integration/test_dbt.py | 58 ++++++++++++++++++- 4 files changed, 92 insertions(+), 36 deletions(-) diff --git a/src/snowflake/cli/_plugins/dbt/manager.py b/src/snowflake/cli/_plugins/dbt/manager.py index ccd3c9e1c2..cf2909ae43 100644 --- a/src/snowflake/cli/_plugins/dbt/manager.py +++ b/src/snowflake/cli/_plugins/dbt/manager.py @@ -61,11 +61,6 @@ def deploy( self._validate_profiles(profiles_path, profile) - if self.exists(name=name) and force is not True: - raise CliError( - f"DBT project {name} already exists. Use --force flag to overwrite" - ) - with cli_console.phase("Creating temporary stage"): stage_manager = StageManager() stage_fqn = FQN.from_string(f"dbt_{name}_stage").using_context() @@ -82,9 +77,13 @@ def deploy( cli_console.step(f"Copied {result_count} files") with cli_console.phase("Creating DBT project"): - query = f"""{'CREATE OR REPLACE' if force is True else 'CREATE'} DBT PROJECT {name} -FROM {stage_name}""" - + if force is True: + query = f"CREATE OR REPLACE DBT PROJECT {name}" + elif self.exists(name=name): + query = f"ALTER DBT PROJECT {name} ADD VERSION" + else: + query = f"CREATE DBT PROJECT {name}" + query += f"\nFROM {stage_name}" return self.execute_query(query) @staticmethod diff --git a/tests/dbt/test_dbt_commands.py b/tests/dbt/test_dbt_commands.py index 468d9f62d1..3a6f5492d6 100644 --- a/tests/dbt/test_dbt_commands.py +++ b/tests/dbt/test_dbt_commands.py @@ -128,20 +128,16 @@ def test_deploys_project_from_source( dbt_project_path, "@MockDatabase.MockSchema.dbt_TEST_PIPELINE_stage" ) - @pytest.mark.parametrize("exists", (True, False)) @mock.patch("snowflake.cli._plugins.dbt.manager.StageManager.put_recursive") @mock.patch("snowflake.cli._plugins.dbt.manager.StageManager.create") def test_force_flag_uses_create_or_replace( self, _mock_create, _mock_put_recursive, - exists, mock_connect, runner, dbt_project_path, - mock_exists, ): - mock_exists.return_value = exists result = runner.invoke( [ @@ -158,6 +154,34 @@ def test_force_flag_uses_create_or_replace( "CREATE OR REPLACE DBT PROJECT" ) + @mock.patch("snowflake.cli._plugins.dbt.manager.StageManager.put_recursive") + @mock.patch("snowflake.cli._plugins.dbt.manager.StageManager.create") + def test_alters_existing_object( + self, + _mock_create, + _mock_put_recursive, + mock_connect, + runner, + dbt_project_path, + mock_exists, + ): + mock_exists.return_value = True + + result = runner.invoke( + [ + "dbt", + "deploy", + "TEST_PIPELINE", + f"--source={dbt_project_path}", + ] + ) + + assert result.exit_code == 0, result.output + assert mock_connect.mocked_ctx.get_query().startswith( + """ALTER DBT PROJECT TEST_PIPELINE ADD VERSION +FROM @MockDatabase.MockSchema.dbt_TEST_PIPELINE_stage""" + ) + @mock.patch("snowflake.cli._plugins.dbt.manager.StageManager.put_recursive") @mock.patch("snowflake.cli._plugins.dbt.manager.StageManager.put") @mock.patch("snowflake.cli._plugins.dbt.manager.StageManager.create") @@ -267,27 +291,6 @@ def test_raises_when_profiles_yml_does_not_contain_selected_profile( assert "profile dev is not defined in profiles.yml" in result.output assert mock_connect.mocked_ctx.get_query() == "" - def test_raises_when_dbt_object_exists_and_is_not_force( - self, dbt_project_path, mock_connect, runner, mock_exists - ): - mock_exists.return_value = True - - result = runner.invoke( - [ - "dbt", - "deploy", - "TEST_PIPELINE", - f"--source={dbt_project_path}", - ], - ) - - assert result.exit_code == 1, result.output - assert ( - "DBT project TEST_PIPELINE already exists. Use --force flag to overwrite" - in result.output - ) - assert mock_connect.mocked_ctx.get_query() == "" - class TestDBTExecute: @pytest.mark.parametrize( diff --git a/tests_integration/__snapshots__/test_dbt.ambr b/tests_integration/__snapshots__/test_dbt.ambr index 24f56f7e38..531b4ec430 100644 --- a/tests_integration/__snapshots__/test_dbt.ambr +++ b/tests_integration/__snapshots__/test_dbt.ambr @@ -1,5 +1,5 @@ # serializer version: 1 -# name: test_dbt +# name: test_deploy_and_execute ''' ╭─ Error ──────────────────────────────────────────────────────────────────────╮ │ Found following errors in profiles.yml. Please fix them before proceeding: │ diff --git a/tests_integration/test_dbt.py b/tests_integration/test_dbt.py index 1d220f252f..970da78ae0 100644 --- a/tests_integration/test_dbt.py +++ b/tests_integration/test_dbt.py @@ -20,7 +20,7 @@ @pytest.mark.integration @pytest.mark.qa_only -def test_dbt( +def test_deploy_and_execute( runner, snowflake_session, test_database, @@ -54,7 +54,6 @@ def test_dbt( "dbt", "deploy", name, - "--force", "--profiles-dir", str(new_profiles_directory.resolve()), ] @@ -95,6 +94,61 @@ def test_dbt( assert result.json[0]["COUNT"] == 1, result.json[0] +@pytest.mark.integration +@pytest.mark.qa_only +def test_dbt_deploy_options( + runner, + snowflake_session, + test_database, + project_directory, +): + with project_directory("dbt_project") as root_dir: + # Given a local dbt project + ts = int(datetime.datetime.now().timestamp()) + name = f"dbt_project_{ts}" + + # deploy for the first time - create new dbt object + _setup_dbt_profile(root_dir, snowflake_session, include_password=False) + result = runner.invoke_with_connection_json(["dbt", "deploy", name]) + assert result.exit_code == 0, result.output + + timestamp_after_create = _fetch_creation_date(name, runner) + + # deploy for the second time - alter existing object + result = runner.invoke_with_connection_json(["dbt", "deploy", name]) + assert result.exit_code == 0, result.output + + timestamp_after_alter = _fetch_creation_date(name, runner) + assert ( + timestamp_after_alter == timestamp_after_create + ), f"Timestamps differ: {timestamp_after_alter} vs {timestamp_after_create}" + + # deploy for the third time - this time with --force flag to replace dbt object + result = runner.invoke_with_connection_json(["dbt", "deploy", name, "--force"]) + assert result.exit_code == 0, result.output + + timestamp_after_replace = _fetch_creation_date(name, runner) + assert ( + timestamp_after_replace > timestamp_after_create + ), f"Timestamps are the same: {timestamp_after_replace} vs {timestamp_after_create}" + + +def _fetch_creation_date(name, runner) -> datetime.datetime: + result = runner.invoke_with_connection_json( + [ + "dbt", + "list", + "--like", + name, + ] + ) + assert result.exit_code == 0, result.output + assert len(result.json) == 1 + dbt_object = result.json[0] + assert dbt_object["name"].lower() == name.lower() + return datetime.datetime.fromisoformat(dbt_object["created_on"]) + + def _setup_dbt_profile(root_dir: Path, snowflake_session, include_password: bool): with open((root_dir / "profiles.yml"), "r") as f: profiles = yaml.safe_load(f)