From 7b956af09dce55873183164d8174d59c4f5ed0da Mon Sep 17 00:00:00 2001 From: Dan Flippo Date: Mon, 24 Feb 2025 13:20:36 -0500 Subject: [PATCH 01/17] Added support for the connection_name parameter. This allows using a connections.toml file to supply additional authentication parameters. --- .../src/dbt/adapters/snowflake/connections.py | 45 +++++++++++-------- 1 file changed, 26 insertions(+), 19 deletions(-) diff --git a/dbt-snowflake/src/dbt/adapters/snowflake/connections.py b/dbt-snowflake/src/dbt/adapters/snowflake/connections.py index af286cb77..f036a95da 100644 --- a/dbt-snowflake/src/dbt/adapters/snowflake/connections.py +++ b/dbt-snowflake/src/dbt/adapters/snowflake/connections.py @@ -112,32 +112,36 @@ class SnowflakeCredentials(Credentials): insecure_mode: Optional[bool] = False # this needs to default to `None` so that we can tell if the user set it; see `__post_init__()` reuse_connections: Optional[bool] = None + connection_name: Optional[str] = None def __post_init__(self): - if self.authenticator != "oauth" and (self.oauth_client_secret or self.oauth_client_id): - # the user probably forgot to set 'authenticator' like I keep doing - warn_or_error( - AdapterEventWarning( - base_msg="Authenticator is not set to oauth, but an oauth-only parameter is set! Did you mean to set authenticator: oauth?" - ) - ) - - if self.authenticator not in ["oauth", "jwt"]: - if self.token: + # skip authentication parameter checking if the connection_name is specified because additional + # parameters can be provided in a connections.toml file + if not self.connection_name: + if self.authenticator != "oauth" and (self.oauth_client_secret or self.oauth_client_id): + # the user probably forgot to set 'authenticator' like I keep doing warn_or_error( AdapterEventWarning( - base_msg=( - "The token parameter was set, but the authenticator was " - "not set to 'oauth' or 'jwt'." - ) + base_msg="Authenticator is not set to oauth, but an oauth-only parameter is set! Did you mean to set authenticator: oauth?" ) ) - if not self.user: - # The user attribute is only optional if 'authenticator' is 'jwt' or 'oauth' - warn_or_error( - AdapterEventError(base_msg="Invalid profile: 'user' is a required property.") - ) + if self.authenticator not in ["oauth", "jwt"]: + if self.token: + warn_or_error( + AdapterEventWarning( + base_msg=( + "The token parameter was set, but the authenticator was " + "not set to 'oauth' or 'jwt'." + ) + ) + ) + + if not self.user: + # The user attribute is only optional if 'authenticator' is 'jwt' or 'oauth' + warn_or_error( + AdapterEventError(base_msg="Invalid profile: 'user' is a required property.") + ) self.account = self.account.replace("_", "-") @@ -179,12 +183,15 @@ def _connection_keys(self): "retry_all", "insecure_mode", "reuse_connections", + "connection_name", ) def auth_args(self): # Pull all of the optional authentication args for the connector, # let connector handle the actual arg validation result = {} + if self.connection_name: + result["connection_name"] = self.connection_name if self.user: result["user"] = self.user if self.password: From d33ccc34da332515d974ac93846de0a736fcc154 Mon Sep 17 00:00:00 2001 From: Dan Flippo Date: Mon, 24 Feb 2025 14:08:25 -0500 Subject: [PATCH 02/17] Added unit test for the connection_name parameter. --- .../auth_tests/test_connection_name.py | 51 +++++++++++++++++++ 1 file changed, 51 insertions(+) create mode 100644 dbt-snowflake/tests/functional/auth_tests/test_connection_name.py diff --git a/dbt-snowflake/tests/functional/auth_tests/test_connection_name.py b/dbt-snowflake/tests/functional/auth_tests/test_connection_name.py new file mode 100644 index 000000000..aaced7192 --- /dev/null +++ b/dbt-snowflake/tests/functional/auth_tests/test_connection_name.py @@ -0,0 +1,51 @@ +""" +Create a connections.toml file at ~/.snowflake/connections.toml +or you can specify a different folder using env variable SNOWFLAKE_HOME. + +The file should have an entry similar to the following +with your credentials. Any type of authentication can be used. + +[default] +user = "test_user" +warehouse = "test_warehouse" +database = "test_database" +schema = "test_schema" +role = "test_role" +password = "test_password" +authenticator = "snowflake" + +You can name you connection something other than "default" by also setting +the SNOWFLAKE_DEFAULT_CONNECTION_NAME environment variable. + +On Linux and Mac OS you will need to set the following +permissions on your connections.toml or you will receive an error. + +chown $USER ~/.snowflake/connections.toml +chmod 0600 ~/.snowflake/connections.toml + +""" + +import os + +from dbt.tests.util import run_dbt +import pytest + + +class TestConnectionName: + @pytest.fixture(scope="class", autouse=True) + def dbt_profile_target(self): + return { + "type": "snowflake", + "threads": 4, + "account": os.getenv("SNOWFLAKE_TEST_ACCOUNT"), + "database": os.getenv("SNOWFLAKE_TEST_DATABASE"), + "warehouse": os.getenv("SNOWFLAKE_TEST_WAREHOUSE"), + "connection_name": os.getenv("SNOWFLAKE_DEFAULT_CONNECTION_NAME", "default"), + } + + @pytest.fixture(scope="class") + def models(self): + return {"my_model.sql": "select 1 as id"} + + def test_connection(self, project): + run_dbt() From 65f594d9a05e8c7f17089dcc0fdd1e7787e0f702 Mon Sep 17 00:00:00 2001 From: Dan Flippo Date: Mon, 24 Feb 2025 16:04:44 -0500 Subject: [PATCH 03/17] Added changie documentation --- .../.changes/unreleased/Features-20250224-160353.yaml | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 dbt-snowflake/.changes/unreleased/Features-20250224-160353.yaml diff --git a/dbt-snowflake/.changes/unreleased/Features-20250224-160353.yaml b/dbt-snowflake/.changes/unreleased/Features-20250224-160353.yaml new file mode 100644 index 000000000..10f224d86 --- /dev/null +++ b/dbt-snowflake/.changes/unreleased/Features-20250224-160353.yaml @@ -0,0 +1,6 @@ +kind: Features +body: Added support for the connection_name parameter and associated connections.toml file when connecting to Snowflake. +time: 2025-02-24T16:03:53.872855-05:00 +custom: + Author: sfc-gh-dflippo + Issue: "684" From 86c88309e327ae66fc7bd99276871b451ac8d531 Mon Sep 17 00:00:00 2001 From: Dan Flippo Date: Mon, 24 Feb 2025 16:48:55 -0500 Subject: [PATCH 04/17] Improved test_connection_name unit test to not rely on external files existing --- .../auth_tests/test_connection_name.py | 60 ++++++++++++------- 1 file changed, 40 insertions(+), 20 deletions(-) diff --git a/dbt-snowflake/tests/functional/auth_tests/test_connection_name.py b/dbt-snowflake/tests/functional/auth_tests/test_connection_name.py index aaced7192..f2cbc26fa 100644 --- a/dbt-snowflake/tests/functional/auth_tests/test_connection_name.py +++ b/dbt-snowflake/tests/functional/auth_tests/test_connection_name.py @@ -1,46 +1,43 @@ """ -Create a connections.toml file at ~/.snowflake/connections.toml -or you can specify a different folder using env variable SNOWFLAKE_HOME. +This class creates a connections.toml file at ~/.snowflake/connections.toml +or a different folder using env variable SNOWFLAKE_HOME. -The file should have an entry similar to the following -with your credentials. Any type of authentication can be used. +The script will populate the toml file based on the testing environment variables +but will look something like: [default] -user = "test_user" -warehouse = "test_warehouse" -database = "test_database" -schema = "test_schema" -role = "test_role" -password = "test_password" +account = "SNOWFLAKE_TEST_ACCOUNT" authenticator = "snowflake" +database = "SNOWFLAKE_TEST_DATABASE" +password = "SNOWFLAKE_TEST_PASSWORD" +role = "DBT_TEST_USER_1" +user = "SNOWFLAKE_TEST_USER" +warehouse = "SNOWFLAKE_TEST_WAREHOUSE" -You can name you connection something other than "default" by also setting -the SNOWFLAKE_DEFAULT_CONNECTION_NAME environment variable. - -On Linux and Mac OS you will need to set the following -permissions on your connections.toml or you will receive an error. - -chown $USER ~/.snowflake/connections.toml -chmod 0600 ~/.snowflake/connections.toml +By putting the password in the connections.toml file and the connection_name in the +profiles.yml, we can test that we can connect based on credentials in the connections.toml """ import os +import pytest +import tempfile +import toml from dbt.tests.util import run_dbt -import pytest class TestConnectionName: @pytest.fixture(scope="class", autouse=True) def dbt_profile_target(self): + # We are returning a profile that does not contain the password return { "type": "snowflake", "threads": 4, "account": os.getenv("SNOWFLAKE_TEST_ACCOUNT"), "database": os.getenv("SNOWFLAKE_TEST_DATABASE"), "warehouse": os.getenv("SNOWFLAKE_TEST_WAREHOUSE"), - "connection_name": os.getenv("SNOWFLAKE_DEFAULT_CONNECTION_NAME", "default"), + "connection_name": "default", } @pytest.fixture(scope="class") @@ -48,4 +45,27 @@ def models(self): return {"my_model.sql": "select 1 as id"} def test_connection(self, project): + + # We are creating a toml file that contains the password + connections_for_toml = { + "default": { + "account": os.getenv("SNOWFLAKE_TEST_ACCOUNT"), + "authenticator": "snowflake", + "database": os.getenv("SNOWFLAKE_TEST_DATABASE"), + "password": os.getenv("SNOWFLAKE_TEST_PASSWORD"), + "role": os.getenv("DBT_TEST_USER_1"), + "user": os.getenv("SNOWFLAKE_TEST_USER"), + "warehouse": os.getenv("SNOWFLAKE_TEST_WAREHOUSE"), + } + } + temp_dir = tempfile.gettempdir() + connections_toml = os.path.join(temp_dir, "connections.toml") + os.environ["SNOWFLAKE_HOME"] = temp_dir + + with open(connections_toml, "w") as f: + toml.dump(connections_for_toml, f) + os.chmod(connections_toml, 0o600) + run_dbt() + + os.unlink(connections_toml) From a011929cdfeb30d0fe83492266b33522df8a4898 Mon Sep 17 00:00:00 2001 From: Dan Flippo Date: Mon, 24 Feb 2025 17:00:09 -0500 Subject: [PATCH 05/17] Executed pre-commit to reformat connections.py --- .../src/dbt/adapters/snowflake/connections.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/dbt-snowflake/src/dbt/adapters/snowflake/connections.py b/dbt-snowflake/src/dbt/adapters/snowflake/connections.py index f036a95da..4bd9913bd 100644 --- a/dbt-snowflake/src/dbt/adapters/snowflake/connections.py +++ b/dbt-snowflake/src/dbt/adapters/snowflake/connections.py @@ -115,10 +115,12 @@ class SnowflakeCredentials(Credentials): connection_name: Optional[str] = None def __post_init__(self): - # skip authentication parameter checking if the connection_name is specified because additional + # skip authentication parameter checking if the connection_name is specified because additional # parameters can be provided in a connections.toml file if not self.connection_name: - if self.authenticator != "oauth" and (self.oauth_client_secret or self.oauth_client_id): + if self.authenticator != "oauth" and ( + self.oauth_client_secret or self.oauth_client_id + ): # the user probably forgot to set 'authenticator' like I keep doing warn_or_error( AdapterEventWarning( @@ -140,7 +142,9 @@ def __post_init__(self): if not self.user: # The user attribute is only optional if 'authenticator' is 'jwt' or 'oauth' warn_or_error( - AdapterEventError(base_msg="Invalid profile: 'user' is a required property.") + AdapterEventError( + base_msg="Invalid profile: 'user' is a required property." + ) ) self.account = self.account.replace("_", "-") From edf630ec067f670135132754894d2ff2d787df74 Mon Sep 17 00:00:00 2001 From: Dan Flippo Date: Mon, 24 Feb 2025 17:41:35 -0500 Subject: [PATCH 06/17] Removed dependency on toml library in unit test --- .../auth_tests/test_connection_name.py | 24 +++++++++---------- 1 file changed, 11 insertions(+), 13 deletions(-) diff --git a/dbt-snowflake/tests/functional/auth_tests/test_connection_name.py b/dbt-snowflake/tests/functional/auth_tests/test_connection_name.py index f2cbc26fa..c476cf933 100644 --- a/dbt-snowflake/tests/functional/auth_tests/test_connection_name.py +++ b/dbt-snowflake/tests/functional/auth_tests/test_connection_name.py @@ -22,7 +22,6 @@ import os import pytest import tempfile -import toml from dbt.tests.util import run_dbt @@ -47,23 +46,22 @@ def models(self): def test_connection(self, project): # We are creating a toml file that contains the password - connections_for_toml = { - "default": { - "account": os.getenv("SNOWFLAKE_TEST_ACCOUNT"), - "authenticator": "snowflake", - "database": os.getenv("SNOWFLAKE_TEST_DATABASE"), - "password": os.getenv("SNOWFLAKE_TEST_PASSWORD"), - "role": os.getenv("DBT_TEST_USER_1"), - "user": os.getenv("SNOWFLAKE_TEST_USER"), - "warehouse": os.getenv("SNOWFLAKE_TEST_WAREHOUSE"), - } - } + connections_for_toml = f""" +[default] +account = "{ os.getenv("SNOWFLAKE_TEST_ACCOUNT") }" +authenticator = "snowflake" +database = "{ os.getenv("SNOWFLAKE_TEST_DATABASE") }" +password = "{ os.getenv("SNOWFLAKE_TEST_PASSWORD") }" +role = "{ os.getenv("DBT_TEST_USER_1") }" +user = "{ os.getenv("SNOWFLAKE_TEST_USER") }" +warehouse = "{ os.getenv("SNOWFLAKE_TEST_WAREHOUSE") }" +""" temp_dir = tempfile.gettempdir() connections_toml = os.path.join(temp_dir, "connections.toml") os.environ["SNOWFLAKE_HOME"] = temp_dir with open(connections_toml, "w") as f: - toml.dump(connections_for_toml, f) + f.write(connections_for_toml) os.chmod(connections_toml, 0o600) run_dbt() From c1cbf1277c5a89495df4294283fc8760220211c1 Mon Sep 17 00:00:00 2001 From: Dan Flippo Date: Tue, 25 Feb 2025 09:57:21 -0500 Subject: [PATCH 07/17] Updated to use default location for connections.toml, ~/.snowflake/connections.toml --- .../auth_tests/test_connection_name.py | 59 ++++++++++++------- 1 file changed, 37 insertions(+), 22 deletions(-) diff --git a/dbt-snowflake/tests/functional/auth_tests/test_connection_name.py b/dbt-snowflake/tests/functional/auth_tests/test_connection_name.py index c476cf933..5b7166077 100644 --- a/dbt-snowflake/tests/functional/auth_tests/test_connection_name.py +++ b/dbt-snowflake/tests/functional/auth_tests/test_connection_name.py @@ -29,6 +29,43 @@ class TestConnectionName: @pytest.fixture(scope="class", autouse=True) def dbt_profile_target(self): + + # We are creating a toml file that contains the password + home_dir = os.getenv("HOME") + snowflake_home_dir = os.getenv("SNOWFLAKE_HOME") + # Use the snowflake home if available + if snowflake_home_dir != None: + config_toml = os.path.join(snowflake_home_dir, "config.toml") + connections_toml = os.path.join(snowflake_home_dir, "connections.toml") + else: + config_toml = os.path.join(home_dir, ".snowflake", "config.toml") + connections_toml = os.path.join(home_dir, ".snowflake", "connections.toml") + snowflake_home_dir = os.path.join(home_dir, ".snowflake") + os.environ["SNOWFLAKE_HOME"] = snowflake_home_dir + + if not os.path.exists(snowflake_home_dir): + os.makedirs(snowflake_home_dir) + os.chmod(snowflake_home_dir, 0o700) + + with open(config_toml, "w") as f: + f.write('default_connection_name = "default"') + os.chmod(config_toml, 0o600) + + with open(connections_toml, "w") as f: + f.write( + f""" +[default] +account = "{ os.getenv("SNOWFLAKE_TEST_ACCOUNT") }" +authenticator = "snowflake" +database = "{ os.getenv("SNOWFLAKE_TEST_DATABASE") }" +password = "{ os.getenv("SNOWFLAKE_TEST_PASSWORD") }" +role = "{ os.getenv("DBT_TEST_USER_1") }" +user = "{ os.getenv("SNOWFLAKE_TEST_USER") }" +warehouse = "{ os.getenv("SNOWFLAKE_TEST_WAREHOUSE") }" +""" + ) + os.chmod(connections_toml, 0o600) + # We are returning a profile that does not contain the password return { "type": "snowflake", @@ -44,26 +81,4 @@ def models(self): return {"my_model.sql": "select 1 as id"} def test_connection(self, project): - - # We are creating a toml file that contains the password - connections_for_toml = f""" -[default] -account = "{ os.getenv("SNOWFLAKE_TEST_ACCOUNT") }" -authenticator = "snowflake" -database = "{ os.getenv("SNOWFLAKE_TEST_DATABASE") }" -password = "{ os.getenv("SNOWFLAKE_TEST_PASSWORD") }" -role = "{ os.getenv("DBT_TEST_USER_1") }" -user = "{ os.getenv("SNOWFLAKE_TEST_USER") }" -warehouse = "{ os.getenv("SNOWFLAKE_TEST_WAREHOUSE") }" -""" - temp_dir = tempfile.gettempdir() - connections_toml = os.path.join(temp_dir, "connections.toml") - os.environ["SNOWFLAKE_HOME"] = temp_dir - - with open(connections_toml, "w") as f: - f.write(connections_for_toml) - os.chmod(connections_toml, 0o600) - run_dbt() - - os.unlink(connections_toml) From 94c11af358b4a8815b115ac785c8cd9845e32397 Mon Sep 17 00:00:00 2001 From: Dan Flippo Date: Tue, 25 Feb 2025 14:01:00 -0500 Subject: [PATCH 08/17] Moved initialization logic to new fixture --- .../auth_tests/test_connection_name.py | 58 +++++++++++-------- 1 file changed, 33 insertions(+), 25 deletions(-) diff --git a/dbt-snowflake/tests/functional/auth_tests/test_connection_name.py b/dbt-snowflake/tests/functional/auth_tests/test_connection_name.py index 5b7166077..3454f8c67 100644 --- a/dbt-snowflake/tests/functional/auth_tests/test_connection_name.py +++ b/dbt-snowflake/tests/functional/auth_tests/test_connection_name.py @@ -19,6 +19,16 @@ """ +connections_toml_template = f"""[default] +account = "{account}" +authenticator = "snowflake" +database = "{database}" +password = "{password}" +role = "{role}" +user = "{user}" +warehouse = "{warehouse}" +""" + import os import pytest import tempfile @@ -27,25 +37,17 @@ class TestConnectionName: - @pytest.fixture(scope="class", autouse=True) - def dbt_profile_target(self): + @pytest.fixture(scope="class", autouse=True) + def connection_name(self): # We are creating a toml file that contains the password - home_dir = os.getenv("HOME") - snowflake_home_dir = os.getenv("SNOWFLAKE_HOME") - # Use the snowflake home if available - if snowflake_home_dir != None: - config_toml = os.path.join(snowflake_home_dir, "config.toml") - connections_toml = os.path.join(snowflake_home_dir, "connections.toml") - else: - config_toml = os.path.join(home_dir, ".snowflake", "config.toml") - connections_toml = os.path.join(home_dir, ".snowflake", "connections.toml") - snowflake_home_dir = os.path.join(home_dir, ".snowflake") - os.environ["SNOWFLAKE_HOME"] = snowflake_home_dir + snowflake_home_dir = os.path.expanduser("~/.snowflake") + config_toml = os.path.join(snowflake_home_dir, "config.toml") + connections_toml = os.path.join(snowflake_home_dir, "connections.toml") + os.environ["SNOWFLAKE_HOME"] = snowflake_home_dir if not os.path.exists(snowflake_home_dir): os.makedirs(snowflake_home_dir) - os.chmod(snowflake_home_dir, 0o700) with open(config_toml, "w") as f: f.write('default_connection_name = "default"') @@ -53,19 +55,25 @@ def dbt_profile_target(self): with open(connections_toml, "w") as f: f.write( - f""" -[default] -account = "{ os.getenv("SNOWFLAKE_TEST_ACCOUNT") }" -authenticator = "snowflake" -database = "{ os.getenv("SNOWFLAKE_TEST_DATABASE") }" -password = "{ os.getenv("SNOWFLAKE_TEST_PASSWORD") }" -role = "{ os.getenv("DBT_TEST_USER_1") }" -user = "{ os.getenv("SNOWFLAKE_TEST_USER") }" -warehouse = "{ os.getenv("SNOWFLAKE_TEST_WAREHOUSE") }" -""" + connections_toml_template.format( + account=os.getenv("SNOWFLAKE_TEST_ACCOUNT"), + database=os.getenv("SNOWFLAKE_TEST_DATABASE"), + password=os.getenv("SNOWFLAKE_TEST_PASSWORD"), + role=os.getenv("SNOWFLAKE_TEST_ROLE"), + user=os.getenv("SNOWFLAKE_TEST_USER"), + warehouse=os.getenv("SNOWFLAKE_TEST_WAREHOUSE"), + ).strip() ) os.chmod(connections_toml, 0o600) + yield "default" + + del os.environ["SNOWFLAKE_HOME"] + os.unlink(config_toml) + os.unlink(connections_toml) + + @pytest.fixture(scope="class", autouse=True) + def dbt_profile_target(self, connection_name): # We are returning a profile that does not contain the password return { "type": "snowflake", @@ -73,7 +81,7 @@ def dbt_profile_target(self): "account": os.getenv("SNOWFLAKE_TEST_ACCOUNT"), "database": os.getenv("SNOWFLAKE_TEST_DATABASE"), "warehouse": os.getenv("SNOWFLAKE_TEST_WAREHOUSE"), - "connection_name": "default", + "connection_name": connection_name, } @pytest.fixture(scope="class") From 87c1ad8d573228102800900044826a95cce51adc Mon Sep 17 00:00:00 2001 From: Dan Flippo Date: Tue, 25 Feb 2025 14:47:44 -0500 Subject: [PATCH 09/17] switch env var setting to monkeypatch and file creation to tmp_path fixtures --- .../auth_tests/test_connection_name.py | 73 +++++++++---------- 1 file changed, 34 insertions(+), 39 deletions(-) diff --git a/dbt-snowflake/tests/functional/auth_tests/test_connection_name.py b/dbt-snowflake/tests/functional/auth_tests/test_connection_name.py index 3454f8c67..749689204 100644 --- a/dbt-snowflake/tests/functional/auth_tests/test_connection_name.py +++ b/dbt-snowflake/tests/functional/auth_tests/test_connection_name.py @@ -10,7 +10,7 @@ authenticator = "snowflake" database = "SNOWFLAKE_TEST_DATABASE" password = "SNOWFLAKE_TEST_PASSWORD" -role = "DBT_TEST_USER_1" +role = "SNOWFLAKE_TEST_ROLE" user = "SNOWFLAKE_TEST_USER" warehouse = "SNOWFLAKE_TEST_WAREHOUSE" @@ -19,7 +19,12 @@ """ -connections_toml_template = f"""[default] +from dbt.tests.util import run_dbt +import tempfile +import pytest +import os + +connections_toml_template = f"""[{connection_name}] account = "{account}" authenticator = "snowflake" database = "{database}" @@ -29,48 +34,38 @@ warehouse = "{warehouse}" """ -import os -import pytest -import tempfile - -from dbt.tests.util import run_dbt - class TestConnectionName: @pytest.fixture(scope="class", autouse=True) - def connection_name(self): + def connection_name(self, tmp_path, monkeypatch): # We are creating a toml file that contains the password - snowflake_home_dir = os.path.expanduser("~/.snowflake") - config_toml = os.path.join(snowflake_home_dir, "config.toml") - connections_toml = os.path.join(snowflake_home_dir, "connections.toml") - os.environ["SNOWFLAKE_HOME"] = snowflake_home_dir - - if not os.path.exists(snowflake_home_dir): - os.makedirs(snowflake_home_dir) - - with open(config_toml, "w") as f: - f.write('default_connection_name = "default"') - os.chmod(config_toml, 0o600) - - with open(connections_toml, "w") as f: - f.write( - connections_toml_template.format( - account=os.getenv("SNOWFLAKE_TEST_ACCOUNT"), - database=os.getenv("SNOWFLAKE_TEST_DATABASE"), - password=os.getenv("SNOWFLAKE_TEST_PASSWORD"), - role=os.getenv("SNOWFLAKE_TEST_ROLE"), - user=os.getenv("SNOWFLAKE_TEST_USER"), - warehouse=os.getenv("SNOWFLAKE_TEST_WAREHOUSE"), - ).strip() - ) - os.chmod(connections_toml, 0o600) - - yield "default" - - del os.environ["SNOWFLAKE_HOME"] - os.unlink(config_toml) - os.unlink(connections_toml) + connection_name = "default" + config_toml = tmp_path / "config.toml" + connections_toml = tmp_path / "connections.toml" + monkeypatch.setenv("SNOWFLAKE_HOME", str(tmp_path.absolute())) + + config_toml.write_text('default_connection_name = "default"\n') + config_toml.chmod(0o600) + + connections_toml.write_text( + connections_toml_template.format( + connection_name=connection_name, + account=os.getenv("SNOWFLAKE_TEST_ACCOUNT"), + database=os.getenv("SNOWFLAKE_TEST_DATABASE"), + password=os.getenv("SNOWFLAKE_TEST_PASSWORD"), + role=os.getenv("SNOWFLAKE_TEST_ROLE"), + user=os.getenv("SNOWFLAKE_TEST_USER"), + warehouse=os.getenv("SNOWFLAKE_TEST_WAREHOUSE"), + ).strip() + ) + connections_toml.chmod(0o600) + + yield connection_name + + monkeypatch.delenv("SNOWFLAKE_HOME") + config_toml.unlink() + connections_toml.unlink() @pytest.fixture(scope="class", autouse=True) def dbt_profile_target(self, connection_name): From fca803ba7ef1185f86d2f685ca74769bd358563a Mon Sep 17 00:00:00 2001 From: Dan Flippo Date: Tue, 25 Feb 2025 15:25:27 -0500 Subject: [PATCH 10/17] hard coding connection_name to `default` to avoid name collision. --- .../tests/functional/auth_tests/test_connection_name.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/dbt-snowflake/tests/functional/auth_tests/test_connection_name.py b/dbt-snowflake/tests/functional/auth_tests/test_connection_name.py index 749689204..6933c6d8f 100644 --- a/dbt-snowflake/tests/functional/auth_tests/test_connection_name.py +++ b/dbt-snowflake/tests/functional/auth_tests/test_connection_name.py @@ -24,7 +24,7 @@ import pytest import os -connections_toml_template = f"""[{connection_name}] +connections_toml_template = f"""[default] account = "{account}" authenticator = "snowflake" database = "{database}" @@ -40,7 +40,6 @@ class TestConnectionName: @pytest.fixture(scope="class", autouse=True) def connection_name(self, tmp_path, monkeypatch): # We are creating a toml file that contains the password - connection_name = "default" config_toml = tmp_path / "config.toml" connections_toml = tmp_path / "connections.toml" monkeypatch.setenv("SNOWFLAKE_HOME", str(tmp_path.absolute())) @@ -50,7 +49,6 @@ def connection_name(self, tmp_path, monkeypatch): connections_toml.write_text( connections_toml_template.format( - connection_name=connection_name, account=os.getenv("SNOWFLAKE_TEST_ACCOUNT"), database=os.getenv("SNOWFLAKE_TEST_DATABASE"), password=os.getenv("SNOWFLAKE_TEST_PASSWORD"), @@ -61,7 +59,7 @@ def connection_name(self, tmp_path, monkeypatch): ) connections_toml.chmod(0o600) - yield connection_name + yield "default" monkeypatch.delenv("SNOWFLAKE_HOME") config_toml.unlink() From 8c9858c33fd5a07747b9093f06b7045d1b27dd30 Mon Sep 17 00:00:00 2001 From: Dan Flippo Date: Tue, 25 Feb 2025 15:35:16 -0500 Subject: [PATCH 11/17] Figured out that it wasn't a name collision --- .../tests/functional/auth_tests/test_connection_name.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/dbt-snowflake/tests/functional/auth_tests/test_connection_name.py b/dbt-snowflake/tests/functional/auth_tests/test_connection_name.py index 6933c6d8f..359a7a932 100644 --- a/dbt-snowflake/tests/functional/auth_tests/test_connection_name.py +++ b/dbt-snowflake/tests/functional/auth_tests/test_connection_name.py @@ -24,7 +24,7 @@ import pytest import os -connections_toml_template = f"""[default] +connections_toml_template = """[{name}] account = "{account}" authenticator = "snowflake" database = "{database}" @@ -40,6 +40,7 @@ class TestConnectionName: @pytest.fixture(scope="class", autouse=True) def connection_name(self, tmp_path, monkeypatch): # We are creating a toml file that contains the password + connection_name = "default" config_toml = tmp_path / "config.toml" connections_toml = tmp_path / "connections.toml" monkeypatch.setenv("SNOWFLAKE_HOME", str(tmp_path.absolute())) @@ -49,6 +50,7 @@ def connection_name(self, tmp_path, monkeypatch): connections_toml.write_text( connections_toml_template.format( + name=connection_name, account=os.getenv("SNOWFLAKE_TEST_ACCOUNT"), database=os.getenv("SNOWFLAKE_TEST_DATABASE"), password=os.getenv("SNOWFLAKE_TEST_PASSWORD"), @@ -59,7 +61,7 @@ def connection_name(self, tmp_path, monkeypatch): ) connections_toml.chmod(0o600) - yield "default" + yield connection_name monkeypatch.delenv("SNOWFLAKE_HOME") config_toml.unlink() From 663abb952a4a9bb73b10f654f1ec985156b6d341 Mon Sep 17 00:00:00 2001 From: Dan Flippo Date: Tue, 25 Feb 2025 17:36:40 -0500 Subject: [PATCH 12/17] changed scope to function for connection_name fixture. --- .../tests/functional/auth_tests/test_connection_name.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dbt-snowflake/tests/functional/auth_tests/test_connection_name.py b/dbt-snowflake/tests/functional/auth_tests/test_connection_name.py index 359a7a932..8b037cad5 100644 --- a/dbt-snowflake/tests/functional/auth_tests/test_connection_name.py +++ b/dbt-snowflake/tests/functional/auth_tests/test_connection_name.py @@ -37,7 +37,7 @@ class TestConnectionName: - @pytest.fixture(scope="class", autouse=True) + @pytest.fixture(autouse=True) def connection_name(self, tmp_path, monkeypatch): # We are creating a toml file that contains the password connection_name = "default" From 378fadc2d266b8abf2ba3c0170b7563256844727 Mon Sep 17 00:00:00 2001 From: Dan Flippo Date: Tue, 25 Feb 2025 17:40:31 -0500 Subject: [PATCH 13/17] removed connection_name from dbt_profile_target fixture to avoid scope error. --- .../auth_tests/test_connection_name.py | 32 +++++++++---------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/dbt-snowflake/tests/functional/auth_tests/test_connection_name.py b/dbt-snowflake/tests/functional/auth_tests/test_connection_name.py index 8b037cad5..e2c1183cc 100644 --- a/dbt-snowflake/tests/functional/auth_tests/test_connection_name.py +++ b/dbt-snowflake/tests/functional/auth_tests/test_connection_name.py @@ -37,6 +37,22 @@ class TestConnectionName: + @pytest.fixture(scope="class", autouse=True) + def dbt_profile_target(self): + # We are returning a profile that does not contain the password + return { + "type": "snowflake", + "threads": 4, + "account": os.getenv("SNOWFLAKE_TEST_ACCOUNT"), + "database": os.getenv("SNOWFLAKE_TEST_DATABASE"), + "warehouse": os.getenv("SNOWFLAKE_TEST_WAREHOUSE"), + "connection_name": "default", + } + + @pytest.fixture(scope="class") + def models(self): + return {"my_model.sql": "select 1 as id"} + @pytest.fixture(autouse=True) def connection_name(self, tmp_path, monkeypatch): # We are creating a toml file that contains the password @@ -67,21 +83,5 @@ def connection_name(self, tmp_path, monkeypatch): config_toml.unlink() connections_toml.unlink() - @pytest.fixture(scope="class", autouse=True) - def dbt_profile_target(self, connection_name): - # We are returning a profile that does not contain the password - return { - "type": "snowflake", - "threads": 4, - "account": os.getenv("SNOWFLAKE_TEST_ACCOUNT"), - "database": os.getenv("SNOWFLAKE_TEST_DATABASE"), - "warehouse": os.getenv("SNOWFLAKE_TEST_WAREHOUSE"), - "connection_name": connection_name, - } - - @pytest.fixture(scope="class") - def models(self): - return {"my_model.sql": "select 1 as id"} - def test_connection(self, project): run_dbt() From 58019b683b841d84fe7d2074908be1e00e4fa2d9 Mon Sep 17 00:00:00 2001 From: Dan Flippo Date: Wed, 26 Feb 2025 09:20:53 -0500 Subject: [PATCH 14/17] It isn't creating toml files so trying moving logic from fixture to the test_connection --- .../functional/auth_tests/test_connection_name.py | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/dbt-snowflake/tests/functional/auth_tests/test_connection_name.py b/dbt-snowflake/tests/functional/auth_tests/test_connection_name.py index e2c1183cc..fb89d6b7e 100644 --- a/dbt-snowflake/tests/functional/auth_tests/test_connection_name.py +++ b/dbt-snowflake/tests/functional/auth_tests/test_connection_name.py @@ -53,13 +53,13 @@ def dbt_profile_target(self): def models(self): return {"my_model.sql": "select 1 as id"} - @pytest.fixture(autouse=True) - def connection_name(self, tmp_path, monkeypatch): + # Test that we can write a connections.toml and use it to connect + def test_connection(self, project, tmp_path, monkeypatch): + # We are creating a toml file that contains the password connection_name = "default" config_toml = tmp_path / "config.toml" connections_toml = tmp_path / "connections.toml" - monkeypatch.setenv("SNOWFLAKE_HOME", str(tmp_path.absolute())) config_toml.write_text('default_connection_name = "default"\n') config_toml.chmod(0o600) @@ -77,11 +77,9 @@ def connection_name(self, tmp_path, monkeypatch): ) connections_toml.chmod(0o600) - yield connection_name + monkeypatch.setattr(os, "environ", {"SNOWFLAKE_HOME": str(tmp_path.absolute())}) + + run_dbt() - monkeypatch.delenv("SNOWFLAKE_HOME") config_toml.unlink() connections_toml.unlink() - - def test_connection(self, project): - run_dbt() From 3e801fe66a26cfc506167113e607513a247d0e13 Mon Sep 17 00:00:00 2001 From: Dan Flippo Date: Wed, 26 Feb 2025 13:16:16 -0500 Subject: [PATCH 15/17] Added connections_file_path parameter to be able to pytest connection_name - Snowflake driver's own tests use the same --- dbt-snowflake/src/dbt/adapters/snowflake/connections.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/dbt-snowflake/src/dbt/adapters/snowflake/connections.py b/dbt-snowflake/src/dbt/adapters/snowflake/connections.py index 4bd9913bd..b96fc52a8 100644 --- a/dbt-snowflake/src/dbt/adapters/snowflake/connections.py +++ b/dbt-snowflake/src/dbt/adapters/snowflake/connections.py @@ -113,6 +113,7 @@ class SnowflakeCredentials(Credentials): # this needs to default to `None` so that we can tell if the user set it; see `__post_init__()` reuse_connections: Optional[bool] = None connection_name: Optional[str] = None + connections_file_path: Optional[str] = None def __post_init__(self): # skip authentication parameter checking if the connection_name is specified because additional @@ -188,6 +189,7 @@ def _connection_keys(self): "insecure_mode", "reuse_connections", "connection_name", + "connections_file_path", ) def auth_args(self): @@ -196,6 +198,8 @@ def auth_args(self): result = {} if self.connection_name: result["connection_name"] = self.connection_name + if self.connections_file_path: + result["connections_file_path"] = self.connections_file_path if self.user: result["user"] = self.user if self.password: From 78e7159f1bec65aa0a53902e2228d94179b52f7a Mon Sep 17 00:00:00 2001 From: Dan Flippo Date: Wed, 26 Feb 2025 13:17:37 -0500 Subject: [PATCH 16/17] Added connections_file_path parameter to be able to pytest connection_name - Snowflake driver's own tests use the same --- .../auth_tests/test_connection_name.py | 29 +++++++------------ 1 file changed, 11 insertions(+), 18 deletions(-) diff --git a/dbt-snowflake/tests/functional/auth_tests/test_connection_name.py b/dbt-snowflake/tests/functional/auth_tests/test_connection_name.py index fb89d6b7e..5ab3fe6e3 100644 --- a/dbt-snowflake/tests/functional/auth_tests/test_connection_name.py +++ b/dbt-snowflake/tests/functional/auth_tests/test_connection_name.py @@ -24,7 +24,8 @@ import pytest import os -connections_toml_template = """[{name}] +connections_toml_template = """ +[{name}] account = "{account}" authenticator = "snowflake" database = "{database}" @@ -38,7 +39,7 @@ class TestConnectionName: @pytest.fixture(scope="class", autouse=True) - def dbt_profile_target(self): + def dbt_profile_target(self, tmp_path): # We are returning a profile that does not contain the password return { "type": "snowflake", @@ -47,6 +48,7 @@ def dbt_profile_target(self): "database": os.getenv("SNOWFLAKE_TEST_DATABASE"), "warehouse": os.getenv("SNOWFLAKE_TEST_WAREHOUSE"), "connection_name": "default", + "connections_file_path": tmp_path / "connections.toml", } @pytest.fixture(scope="class") @@ -54,32 +56,23 @@ def models(self): return {"my_model.sql": "select 1 as id"} # Test that we can write a connections.toml and use it to connect - def test_connection(self, project, tmp_path, monkeypatch): + def test_connection(self, project, dbt_profile_target): + connections_toml = dbt_profile_target.connections_file_path # We are creating a toml file that contains the password - connection_name = "default" - config_toml = tmp_path / "config.toml" - connections_toml = tmp_path / "connections.toml" - - config_toml.write_text('default_connection_name = "default"\n') - config_toml.chmod(0o600) - connections_toml.write_text( connections_toml_template.format( - name=connection_name, - account=os.getenv("SNOWFLAKE_TEST_ACCOUNT"), - database=os.getenv("SNOWFLAKE_TEST_DATABASE"), + name=dbt_profile_target.connection_name, + account=dbt_profile_target.account, + database=dbt_profile_target.database, password=os.getenv("SNOWFLAKE_TEST_PASSWORD"), role=os.getenv("SNOWFLAKE_TEST_ROLE"), user=os.getenv("SNOWFLAKE_TEST_USER"), - warehouse=os.getenv("SNOWFLAKE_TEST_WAREHOUSE"), - ).strip() + warehouse=dbt_profile_target.warehouse, + ) ) connections_toml.chmod(0o600) - monkeypatch.setattr(os, "environ", {"SNOWFLAKE_HOME": str(tmp_path.absolute())}) - run_dbt() - config_toml.unlink() connections_toml.unlink() From c3ef04b2025ac2858de7ec53a2355eacfdcd9c71 Mon Sep 17 00:00:00 2001 From: Dan Flippo Date: Wed, 26 Feb 2025 17:35:08 -0500 Subject: [PATCH 17/17] Switched to having the profile use ~ and setting HOME to tmp_path in test --- .../auth_tests/test_connection_name.py | 52 +++++++++++-------- 1 file changed, 29 insertions(+), 23 deletions(-) diff --git a/dbt-snowflake/tests/functional/auth_tests/test_connection_name.py b/dbt-snowflake/tests/functional/auth_tests/test_connection_name.py index 5ab3fe6e3..fff446b8b 100644 --- a/dbt-snowflake/tests/functional/auth_tests/test_connection_name.py +++ b/dbt-snowflake/tests/functional/auth_tests/test_connection_name.py @@ -1,6 +1,7 @@ """ -This class creates a connections.toml file at ~/.snowflake/connections.toml -or a different folder using env variable SNOWFLAKE_HOME. +This class sets the profile to use a connection_name and connections_file_path. +The connection_name is "default" and the connections.toml file is set to "~/connections.toml" +During the test we set the HOME to a temporary folder where we write this file. The script will populate the toml file based on the testing environment variables but will look something like: @@ -14,8 +15,9 @@ user = "SNOWFLAKE_TEST_USER" warehouse = "SNOWFLAKE_TEST_WAREHOUSE" -By putting the password in the connections.toml file and the connection_name in the -profiles.yml, we can test that we can connect based on credentials in the connections.toml +By putting the password in the connections.toml file and the connection_name +& connections_file_path in the profiles.yml, we can test that we can connect +based on credentials in the connections.toml """ @@ -23,6 +25,7 @@ import tempfile import pytest import os +from pathlib import Path connections_toml_template = """ [{name}] @@ -39,7 +42,7 @@ class TestConnectionName: @pytest.fixture(scope="class", autouse=True) - def dbt_profile_target(self, tmp_path): + def dbt_profile_target(self): # We are returning a profile that does not contain the password return { "type": "snowflake", @@ -48,7 +51,7 @@ def dbt_profile_target(self, tmp_path): "database": os.getenv("SNOWFLAKE_TEST_DATABASE"), "warehouse": os.getenv("SNOWFLAKE_TEST_WAREHOUSE"), "connection_name": "default", - "connections_file_path": tmp_path / "connections.toml", + "connections_file_path": "~/connections.toml", } @pytest.fixture(scope="class") @@ -56,23 +59,26 @@ def models(self): return {"my_model.sql": "select 1 as id"} # Test that we can write a connections.toml and use it to connect - def test_connection(self, project, dbt_profile_target): - connections_toml = dbt_profile_target.connections_file_path + def test_connection(self, project, tmp_path, monkeypatch): - # We are creating a toml file that contains the password - connections_toml.write_text( - connections_toml_template.format( - name=dbt_profile_target.connection_name, - account=dbt_profile_target.account, - database=dbt_profile_target.database, - password=os.getenv("SNOWFLAKE_TEST_PASSWORD"), - role=os.getenv("SNOWFLAKE_TEST_ROLE"), - user=os.getenv("SNOWFLAKE_TEST_USER"), - warehouse=dbt_profile_target.warehouse, - ) - ) - connections_toml.chmod(0o600) + with monkeypatch.context() as m: + # Set HOME to our temporary folder for later tilde expansion in the driver + m.setenv("HOME", str(tmp_path.absolute())) + + connections_toml = tmp_path / "connections.toml" - run_dbt() + # We are creating a toml file that contains the password + connections_toml.write_text( + connections_toml_template.format( + name="default", + account=os.getenv("SNOWFLAKE_TEST_ACCOUNT"), + database=os.getenv("SNOWFLAKE_TEST_DATABASE"), + password=os.getenv("SNOWFLAKE_TEST_PASSWORD"), + role=os.getenv("SNOWFLAKE_TEST_ROLE"), + user=os.getenv("SNOWFLAKE_TEST_USER"), + warehouse=os.getenv("SNOWFLAKE_TEST_WAREHOUSE"), + ) + ) + connections_toml.chmod(0o600) - connections_toml.unlink() + run_dbt()