Merge branch 'master' into dependabot/github_actions/docker/login-action-3.3.0

cody-scott · web-flow · commit 557ddec35ded · 2024-09-05T16:27:49.000-04:00
diff --git a/.github/workflows/release-version.yml b/.github/workflows/release-version.yml
@@ -13,7 +13,7 @@ jobs:
     steps:
       - uses: actions/checkout@v4
 
-      - uses: actions/setup-python@v4
+      - uses: actions/setup-python@v5
         with:
           python-version: '3.9'
 
diff --git a/README.md b/README.md
@@ -17,9 +17,9 @@ Join us on the [dbt Slack](https://getdbt.slack.com/archives/CMRMDDQ9W) to ask q
 ## Installation
 
 This adapter requires the Microsoft ODBC driver to be installed:
-[Windows](https://docs.microsoft.com/nl-be/sql/connect/odbc/download-odbc-driver-for-sql-server?view=sql-server-ver16#download-for-windows) |
-[macOS](https://docs.microsoft.com/nl-be/sql/connect/odbc/linux-mac/install-microsoft-odbc-driver-sql-server-macos?view=sql-server-ver16) |
-[Linux](https://docs.microsoft.com/nl-be/sql/connect/odbc/linux-mac/installing-the-microsoft-odbc-driver-for-sql-server?view=sql-server-ver16)
+[Windows](https://docs.microsoft.com/en-us/sql/connect/odbc/download-odbc-driver-for-sql-server?view=sql-server-ver16#download-for-windows) |
+[macOS](https://docs.microsoft.com/en-us/sql/connect/odbc/linux-mac/install-microsoft-odbc-driver-sql-server-macos?view=sql-server-ver16) |
+[Linux](https://docs.microsoft.com/en-us/sql/connect/odbc/linux-mac/installing-the-microsoft-odbc-driver-for-sql-server?view=sql-server-ver16)
 
 <details><summary>Debian/Ubuntu</summary>
 <p>
diff --git a/dbt/adapters/sqlserver/__version__.py b/dbt/adapters/sqlserver/__version__.py
@@ -1 +1 @@
-version = "1.8.0rc1"
+version = "1.8.0rc2"
diff --git a/dbt/adapters/sqlserver/sqlserver_connections.py b/dbt/adapters/sqlserver/sqlserver_connections.py
@@ -130,7 +130,7 @@ def open(cls, connection: Connection) -> Connection:
 
         plugin_version = __version__.version
         application_name = f"dbt-{credentials.type}/{plugin_version}"
-        con_str.append(f"Application Name={application_name}")
+        con_str.append(f"APP={application_name}")
 
         con_str_concat = ";".join(con_str)
 
diff --git a/dbt/include/sqlserver/macros/materializations/models/incremental/incremental.sql b/dbt/include/sqlserver/macros/materializations/models/incremental/incremental.sql
@@ -59,6 +59,7 @@
     {% set strategy_arg_dict = ({'target_relation': target_relation, 'temp_relation': temp_relation, 'unique_key': unique_key, 'dest_columns': dest_columns, 'incremental_predicates': incremental_predicates }) %}
     {% set build_sql = strategy_sql_macro_func(strategy_arg_dict) %}
 
+    {% do to_drop.append(temp_relation) %}
   {% endif %}
 
   {% call statement("main") %}
diff --git a/dbt/include/sqlserver/macros/materializations/snapshot/snapshot.sql b/dbt/include/sqlserver/macros/materializations/snapshot/snapshot.sql
@@ -35,6 +35,7 @@
 
   {% set temp_snapshot_relation_sql = model['compiled_code'].replace("'", "''") %}
   {% call statement('create temp_snapshot_relation') %}
+    USE [{{ model.database}}];
     EXEC('DROP VIEW IF EXISTS {{ temp_snapshot_relation.include(database=False) }};');
     EXEC('create view {{ temp_snapshot_relation.include(database=False) }} as {{ temp_snapshot_relation_sql }};');
   {% endcall %}
diff --git a/dbt/include/sqlserver/macros/relations/seeds/helpers.sql b/dbt/include/sqlserver/macros/relations/seeds/helpers.sql
@@ -0,0 +1,57 @@
+{% macro sqlserver__get_binding_char() %}
+  {{ return('?') }}
+{% endmacro %}
+
+{% macro sqlserver__get_batch_size() %}
+  {{ return(400) }}
+{% endmacro %}
+
+{% macro calc_batch_size(num_columns) %}
+    {#
+        SQL Server allows for a max of 2098 parameters in a single statement.
+        Check if the max_batch_size fits with the number of columns, otherwise
+        reduce the batch size so it fits.
+    #}
+    {% set max_batch_size = get_batch_size() %}
+    {% set calculated_batch = (2098 / num_columns)|int %}
+    {% set batch_size = [max_batch_size, calculated_batch] | min %}
+
+    {{ return(batch_size) }}
+{%  endmacro %}
+
+{% macro sqlserver__load_csv_rows(model, agate_table) %}
+  {% set cols_sql = get_seed_column_quoted_csv(model, agate_table.column_names) %}
+  {% set batch_size = calc_batch_size(agate_table.column_names|length) %}
+  {% set bindings = [] %}
+  {% set statements = [] %}
+
+  {{ log("Inserting batches of " ~ batch_size ~ " records") }}
+
+  {% for chunk in agate_table.rows | batch(batch_size) %}
+      {% set bindings = [] %}
+
+      {% for row in chunk %}
+          {% do bindings.extend(row) %}
+      {% endfor %}
+
+      {% set sql %}
+          insert into {{ this.render() }} ({{ cols_sql }}) values
+          {% for row in chunk -%}
+              ({%- for column in agate_table.column_names -%}
+                  {{ get_binding_char() }}
+                  {%- if not loop.last%},{%- endif %}
+              {%- endfor -%})
+              {%- if not loop.last%},{%- endif %}
+          {%- endfor %}
+      {% endset %}
+
+      {% do adapter.add_query(sql, bindings=bindings, abridge_sql_log=True) %}
+
+      {% if loop.index0 == 0 %}
+          {% do statements.append(sql) %}
+      {% endif %}
+  {% endfor %}
+
+  {# Return SQL so we can render it out into the compiled files #}
+  {{ return(statements[0]) }}
+{% endmacro %}
diff --git a/dbt/include/sqlserver/macros/relations/table/create.sql b/dbt/include/sqlserver/macros/relations/table/create.sql
@@ -3,6 +3,7 @@
     {%- set tmp_relation = relation.incorporate(path={"identifier": relation.identifier ~ '__dbt_tmp_vw'}, type='view') -%}
 
     {%- do adapter.drop_relation(tmp_relation) -%}
+    USE [{{ relation.database }}];
     {{ get_create_view_as_sql(tmp_relation, sql) }}
 
     {%- set table_name -%}
diff --git a/dbt/include/sqlserver/macros/relations/views/create.sql b/dbt/include/sqlserver/macros/relations/views/create.sql
@@ -13,7 +13,7 @@
     {% set tst %}
     SELECT '1' as col
     {% endset %}
-
+    USE [{{ relation.database }}];
     EXEC('{{- escape_single_quotes(query) -}}')
 
 {% endmacro %}
diff --git a/dev_requirements.txt b/dev_requirements.txt
@@ -2,7 +2,7 @@
 dbt-tests-adapter>=1.8.0, <1.9.0
 
 ruff
-black==24.2.0
+black==24.8.0
 bumpversion
 flake8
 flaky
diff --git a/tests/functional/adapter/mssql/test_cross_db.py b/tests/functional/adapter/mssql/test_cross_db.py
@@ -0,0 +1,170 @@
+import pytest
+from dbt.tests.util import get_connection, run_dbt
+
+snapshot_sql = """
+{% snapshot claims_snapshot %}
+
+{{
+   config(
+       target_database='secondary_db',
+       target_schema='dbo',
+       unique_key='id',
+
+       strategy='timestamp',
+       updated_at='updated_at',
+   )
+}}
+
+select * from {{source('mysource', 'claims')}}
+
+{% endsnapshot %}
+"""
+
+source_csv = """id,updated_date
+1,2024-01-01
+2,2024-01-01
+3,2024-01-01
+"""
+
+sources_yml = """
+version: 2
+sources:
+  - name: mysource
+    database: TestDB
+    tables:
+      - name: claims
+"""
+
+
+class TestCrossDB:
+    def create_secondary_db(self, project):
+        create_sql = """
+        DECLARE @col NVARCHAR(256)
+        SET @col = (SELECT CONVERT (varchar(256), SERVERPROPERTY('collation')));
+
+        IF NOT EXISTS (SELECT * FROM sys.databases WHERE name='secondary_db')
+        BEGIN
+            EXEC ('CREATE DATABASE secondary_db COLLATE ' + @col)
+        END
+        """
+
+        with get_connection(project.adapter):
+            project.adapter.execute(
+                create_sql.format(database=project.database),
+                fetch=True,
+            )
+
+    def cleanup_secondary_database(self, project):
+        drop_sql = "DROP DATABASE IF EXISTS secondary_db"
+        with get_connection(project.adapter):
+            project.adapter.execute(
+                drop_sql.format(database=project.database),
+                fetch=True,
+            )
+
+    def cleanup_primary_table(self, project):
+        drop_sql = "DROP TABLE IF EXISTS {database}.mysource.claims"
+        with get_connection(project.adapter):
+            project.adapter.execute(
+                drop_sql.format(database=project.database),
+                fetch=True,
+            )
+
+    def cleanup_snapshot_table(self, project):
+        drop_sql = "DROP TABLE IF EXISTS TestDB_Secondary.dbo.claims_snapshot"
+        with get_connection(project.adapter):
+            project.adapter.execute(
+                drop_sql,
+                fetch=True,
+            )
+
+    def create_source_schema(self, project):
+        create_sql = """
+        IF NOT EXISTS (SELECT * FROM sys.schemas WHERE name = 'mysource')
+        BEGIN
+            EXEC('CREATE SCHEMA mysource')
+        END
+        """
+        with get_connection(project.adapter):
+            project.adapter.execute(
+                create_sql,
+                fetch=True,
+            )
+
+    def create_primary_table(self, project):
+        src_query = """
+        SELECT *
+        INTO
+            {database}.mysource.claims
+        FROM
+            (
+                SELECT
+                    1 as id,
+                    CAST('2024-01-01' as DATETIME2(6)) updated_at
+
+                UNION ALL
+
+                SELECT
+                    2 as id,
+                    CAST('2024-01-01' as DATETIME2(6)) updated_at
+
+                UNION ALL
+
+                SELECT
+                    3 as id,
+                    CAST('2024-01-01' as DATETIME2(6)) updated_at
+            ) as src_data
+        """
+        with get_connection(project.adapter):
+            project.adapter.execute(
+                src_query.format(database=project.database, schema=project.test_schema),
+                fetch=True,
+            )
+
+    def create_secondary_schema(self, project):
+        src_query = """
+        USE [secondary_db]
+        EXEC ('CREATE SCHEMA {schema}')
+        """
+        with get_connection(project.adapter):
+            project.adapter.execute(
+                src_query.format(database=project.database, schema=project.test_schema),
+                fetch=True,
+            )
+
+    def update_primary_table(self, project):
+        sql = """
+        UPDATE [{database}].[mysource].[claims]
+        SET
+        updated_at = CAST('2024-02-01' as datetime2(6))
+        WHERE
+        id = 3
+        """
+        with get_connection(project.adapter):
+            project.adapter.execute(
+                sql.format(database=project.database),
+                fetch=True,
+            )
+
+    @pytest.fixture(scope="class")
+    def models(self):
+        return {"sources.yml": sources_yml}
+
+    @pytest.fixture(scope="class")
+    def snapshots(self):
+        return {"claims_snapshot.sql": snapshot_sql}
+
+    def test_cross_db_snapshot(self, project):
+        self.create_secondary_db(project)
+
+        self.cleanup_primary_table(project)
+        self.cleanup_snapshot_table(project)
+
+        self.create_source_schema(project)
+        self.create_primary_table(project)
+        run_dbt(["snapshot"])
+        self.update_primary_table(project)
+        run_dbt(["snapshot"])
+
+        self.cleanup_snapshot_table(project)
+        self.cleanup_secondary_database(project)
diff --git a/tests/functional/adapter/mssql/test_mssql_seed.py b/tests/functional/adapter/mssql/test_mssql_seed.py
@@ -0,0 +1,41 @@
+import pytest
+from dbt.tests.util import run_dbt
+
+seed_schema_yml = """
+version: 2
+seeds:
+  - name: raw_data
+"""
+
+
+class TestLargeSeed:
+    def build_large_seed_file(self):
+        row_count = 3000
+        column_count = 10
+
+        headers = ",".join(["id"] + [f"column_{_}" for _ in range(1, column_count)])
+        seed_data = [headers]
+        for row in range(1, row_count):
+            row_data = [str(row)]
+            for column in range(1, column_count):
+                row_data += [str(column)]
+
+            row_data = ",".join(row_data)
+            seed_data += [row_data]
+
+        large_seed_file = "\n".join(seed_data)
+        return large_seed_file
+
+    @pytest.fixture(scope="class")
+    def project_config_update(self):
+        return {"name": "generic_tests"}
+
+    @pytest.fixture(scope="class")
+    def seeds(self):
+        return {
+            "raw_data.csv": self.build_large_seed_file(),
+            "schema.yml": seed_schema_yml,
+        }
+
+    def test_large_seed(self, project):
+        run_dbt(["seed"])
diff --git a/tests/functional/adapter/mssql/test_temp_relation_cleanup.py b/tests/functional/adapter/mssql/test_temp_relation_cleanup.py

Original file line number	Diff line number	Diff line change
`@@ -1 +1 @@`
`1`		`-version = "1.8.0rc1"`
	`1`	`+version = "1.8.0rc2"`