Skip to content

Commit f3ea354

Browse files
committed
added incremental fix and validation test
1 parent 09b577b commit f3ea354

File tree

2 files changed

+86
-12
lines changed

2 files changed

+86
-12
lines changed

dbt/include/sqlserver/macros/materializations/models/incremental/incremental.sql

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -59,6 +59,7 @@
5959
{% set strategy_arg_dict = ({'target_relation': target_relation, 'temp_relation': temp_relation, 'unique_key': unique_key, 'dest_columns': dest_columns, 'incremental_predicates': incremental_predicates }) %}
6060
{% set build_sql = strategy_sql_macro_func(strategy_arg_dict) %}
6161

62+
{% do to_drop.append(temp_relation) %}
6263
{% endif %}
6364

6465
{% call statement("main") %}

tests/functional/adapter/mssql/test_temp_relation_cleanup.py

Lines changed: 85 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -25,34 +25,107 @@
2525
WHERE
2626
TABLE_SCHEMA = '{schema}'
2727
AND
28-
TABLE_NAME LIKE '%__dbt_tmp_vw'
28+
TABLE_NAME LIKE '%__dbt_tmp%'
29+
"""
30+
31+
seed_schema_yml = """
32+
version: 2
33+
seeds:
34+
- name: raw_data
35+
config:
36+
column_types:
37+
id: integer
38+
value_col: nvarchar(20)
39+
date_col: datetime2(6)
40+
"""
41+
42+
seed_csv = """id,data,date_col
43+
1,1,2024-01-01
44+
2,1,2024-01-01
45+
3,1,2024-01-01"""
46+
47+
incremental_sql = """
48+
{{
49+
config(
50+
materialized='incremental',
51+
unique_key='id',
52+
on_schema_change='sync_all_columns'
53+
)
54+
}}
55+
56+
WITH source_data AS (SELECT * FROM {{ ref('raw_data') }} )
57+
58+
{% if is_incremental() %}
59+
60+
SELECT id,
61+
data,
62+
date_col
63+
FROM source_data WHERE id NOT IN (SELECT id from {{ this }} )
64+
65+
{% else %}
66+
67+
SELECT id,
68+
data,
69+
date_col
70+
FROM source_data where id <= 1
71+
72+
{% endif %}
2973
"""
3074

3175

32-
class TestTempRelationCleanup:
76+
class BaseTempRelationCleanup:
77+
view_name = "__dbt_tmp_vw"
78+
79+
def validate_temp_objects(self, project):
80+
with get_connection(project.adapter):
81+
result, table = project.adapter.execute(
82+
validation_sql.format(
83+
database=project.database, schema=project.created_schemas[0]
84+
),
85+
fetch=True,
86+
)
87+
assert len(table.rows) == 0
88+
89+
90+
class TestTempRelationCleanup(BaseTempRelationCleanup):
3391
"""
3492
This tests to validate that the temporary relations,
3593
created by the `create_table` statement is cleaned up after a set of runs.
3694
"""
3795

38-
view_name = "__dbt_tmp_vw"
39-
4096
@pytest.fixture(scope="class")
4197
def models(self):
4298
return {
4399
"table_model.sql": table_model,
44100
"schema.yml": model_yml,
45101
}
46102

103+
def test_drops_temp_view_object(self, project):
104+
run_dbt(["run"])
105+
106+
self.validate_temp_objects(project)
107+
108+
109+
class TestIncrementalTempCleanup(BaseTempRelationCleanup):
110+
"""Tests if the `dbt_tmp` views are properly cleaned up in an incremental model"""
111+
112+
@pytest.fixture(scope="class")
113+
def seeds(self):
114+
return {
115+
"raw_data.csv": seed_csv,
116+
"schema.yml": seed_schema_yml,
117+
}
118+
119+
@pytest.fixture(scope="class")
120+
def models(self):
121+
return {
122+
"table_model.sql": incremental_sql,
123+
"schema.yml": model_yml,
124+
}
125+
47126
def test_drops_temp_view_object(self, project):
48127
run_dbt(["seed"])
49128
run_dbt(["run"])
129+
run_dbt(["run"])
50130

51-
with get_connection(project.adapter):
52-
result, table = project.adapter.execute(
53-
validation_sql.format(
54-
database=project.database, schema=project.created_schemas[0]
55-
),
56-
fetch=True,
57-
)
58-
assert len(table.rows) == 0
131+
self.validate_temp_objects(project)

0 commit comments

Comments
 (0)