Skip to content

Commit 6510b76

Browse files
sdebruynsdebruyn
authored
Merge pull request #376 from dbt-msft/incremental-tests
[incremental models] add tests, various bugfixes and support for incremental predicates
2 parents c6a9748 + c28fe8d commit 6510b76

File tree

5 files changed

+188
-43
lines changed

5 files changed

+188
-43
lines changed

‎dbt/adapters/sqlserver/sql_server_column.py

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
from typing import ClassVar, Dict
1+
from typing import Any, ClassVar, Dict
22

33
from dbt.adapters.base import Column
44

@@ -11,3 +11,10 @@ class SQLServerColumn(Column):
1111
"INTEGER": "INT",
1212
"BOOLEAN": "BIT",
1313
}
14+
15+
@classmethod
16+
def string_type(cls, size: int) -> str:
17+
return f"varchar({size if size > 0 else 'MAX'})"
18+
19+
def literal(self, value: Any) -> str:
20+
return "cast('{}' as {})".format(value, self.data_type)

‎dbt/include/sqlserver/macros/adapters/columns.sql

Lines changed: 24 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -46,12 +46,31 @@
4646
{%- set tmp_column = column_name + "__dbt_alter" -%}
4747

4848
{% call statement('alter_column_type') -%}
49-
50-
alter {{ relation.type }} {{ relation }} add {{ tmp_column }} {{ new_column_type }};
51-
update {{ relation }} set {{ tmp_column }} = {{ column_name }};
52-
alter {{ relation.type }} {{ relation }} drop column {{ column_name }};
49+
alter {{ relation.type }} {{ relation }} add "{{ tmp_column }}" {{ new_column_type }};
50+
{%- endcall -%}
51+
{% call statement('alter_column_type') -%}
52+
update {{ relation }} set "{{ tmp_column }}" = "{{ column_name }}";
53+
{%- endcall -%}
54+
{% call statement('alter_column_type') -%}
55+
alter {{ relation.type }} {{ relation }} drop column "{{ column_name }}";
56+
{%- endcall -%}
57+
{% call statement('alter_column_type') -%}
5358
exec sp_rename '{{ relation | replace('"', '') }}.{{ tmp_column }}', '{{ column_name }}', 'column'
54-
5559
{%- endcall -%}
5660
5761
{% endmacro %}
62+
63+
64+
{% macro sqlserver__alter_relation_add_remove_columns(relation, add_columns, remove_columns) %}
65+
{% call statement('add_drop_columns') -%}
66+
{% if add_columns %}
67+
alter {{ relation.type }} {{ relation }}
68+
add {% for column in add_columns %}"{{ column.name }}" {{ column.data_type }}{{ ', ' if not loop.last }}{% endfor %};
69+
{% endif %}
70+
71+
{% if remove_columns %}
72+
alter {{ relation.type }} {{ relation }}
73+
drop column {% for column in remove_columns %}"{{ column.name }}"{{ ',' if not loop.last }}{% endfor %};
74+
{% endif %}
75+
{%- endcall -%}
76+
{% endmacro %}

‎dbt/include/sqlserver/macros/materializations/models/incremental/merge.sql

Lines changed: 40 additions & 36 deletions
Original file line numberDiff line numberDiff line change
@@ -5,49 +5,53 @@
55
https://getdbt.slack.com/archives/C50NEBJGG/p1636045535056600
66
#}
77

8-
{% macro sqlserver__get_merge_sql(target, source, unique_key, dest_columns, predicates) %}
9-
{{ default__get_merge_sql(target, source, unique_key, dest_columns, predicates) }};
8+
{% macro sqlserver__get_merge_sql(target, source, unique_key, dest_columns, incremental_predicates=none) %}
9+
{{ default__get_merge_sql(target, source, unique_key, dest_columns, incremental_predicates) }};
1010
{% endmacro %}
1111

12-
{% macro sqlserver__get_delete_insert_merge_sql(target, source, unique_key, dest_columns, incremental_predicates) %}
13-
{% if incremental_predicates %}
14-
{{ exceptions.raise_not_implemented('incremental_predicates are not implemented in dbt-sqlserver') }}
15-
{% endif %}
12+
{% macro sqlserver__get_insert_overwrite_merge_sql(target, source, dest_columns, predicates, include_sql_header) %}
13+
{{ default__get_insert_overwrite_merge_sql(target, source, dest_columns, predicates, include_sql_header) }};
14+
{% endmacro %}
1615

17-
{%- set dest_cols_csv = get_quoted_csv(dest_columns | map(attribute="name")) -%}
16+
{% macro sqlserver__get_delete_insert_merge_sql(target, source, unique_key, dest_columns, incremental_predicates=none) %}
17+
{%- set dest_cols_csv = get_quoted_csv(dest_columns | map(attribute="name")) -%}
1818

19-
{% if unique_key %}
20-
{% if unique_key is sequence and unique_key is not string %}
21-
delete from {{ target }}
22-
where exists (
23-
SELECT NULL
24-
FROM
25-
{{ source }}
26-
WHERE
19+
{% if unique_key %}
20+
{% if unique_key is sequence and unique_key is not string %}
21+
delete from {{ target }}
22+
where exists (
23+
select null
24+
from {{ source }}
25+
where
2726
{% for key in unique_key %}
2827
{{ source }}.{{ key }} = {{ target }}.{{ key }}
2928
{{ "and " if not loop.last }}
3029
{% endfor %}
31-
);
32-
{% else %}
33-
delete from {{ target }}
34-
where (
35-
{{ unique_key }}) in (
36-
select ({{ unique_key }})
37-
from {{ source }}
38-
);
3930

40-
{% endif %}
41-
{% endif %}
42-
43-
insert into {{ target }} ({{ dest_cols_csv }})
44-
(
45-
select {{ dest_cols_csv }}
46-
from {{ source }}
47-
)
48-
49-
{% endmacro %}
50-
51-
{% macro sqlserver__get_insert_overwrite_merge_sql(target, source, dest_columns, predicates, include_sql_header) %}
52-
{{ default__get_insert_overwrite_merge_sql(target, source, dest_columns, predicates, include_sql_header) }};
31+
)
32+
{% if incremental_predicates %}
33+
{% for predicate in incremental_predicates %}
34+
and {{ predicate }}
35+
{% endfor %}
36+
{% endif %};
37+
{% else %}
38+
delete from {{ target }}
39+
where (
40+
{{ unique_key }}) in (
41+
select ({{ unique_key }})
42+
from {{ source }}
43+
)
44+
{%- if incremental_predicates %}
45+
{% for predicate in incremental_predicates %}
46+
and {{ predicate }}
47+
{% endfor %}
48+
{%- endif -%};
49+
{% endif %}
50+
{% endif %}
51+
52+
insert into {{ target }} ({{ dest_cols_csv }})
53+
(
54+
select {{ dest_cols_csv }}
55+
from {{ source }}
56+
)
5357
{% endmacro %}

‎tests/functional/adapter/test_incremental.py

Lines changed: 115 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,120 @@
1+
import pytest
2+
from dbt.tests.adapter.incremental.fixtures import (
3+
_MODELS__A,
4+
_MODELS__INCREMENTAL_APPEND_NEW_COLUMNS,
5+
_MODELS__INCREMENTAL_APPEND_NEW_COLUMNS_REMOVE_ONE,
6+
_MODELS__INCREMENTAL_APPEND_NEW_COLUMNS_REMOVE_ONE_TARGET,
7+
_MODELS__INCREMENTAL_APPEND_NEW_COLUMNS_TARGET,
8+
_MODELS__INCREMENTAL_FAIL,
9+
_MODELS__INCREMENTAL_IGNORE_TARGET,
10+
_MODELS__INCREMENTAL_SYNC_ALL_COLUMNS,
11+
_MODELS__INCREMENTAL_SYNC_REMOVE_ONLY,
12+
)
13+
from dbt.tests.adapter.incremental.test_incremental_on_schema_change import (
14+
BaseIncrementalOnSchemaChange,
15+
)
16+
from dbt.tests.adapter.incremental.test_incremental_predicates import BaseIncrementalPredicates
117
from dbt.tests.adapter.incremental.test_incremental_unique_id import BaseIncrementalUniqueKey
218

19+
_MODELS__INCREMENTAL_IGNORE = """
20+
{{
21+
config(
22+
materialized='incremental',
23+
unique_key='id',
24+
on_schema_change='ignore'
25+
)
26+
}}
27+
28+
WITH source_data AS (SELECT * FROM {{ ref('model_a') }} )
29+
30+
{% if is_incremental() %}
31+
32+
SELECT
33+
id,
34+
field1,
35+
field2,
36+
field3,
37+
field4
38+
FROM source_data
39+
WHERE id NOT IN (SELECT id from {{ this }} )
40+
41+
{% else %}
42+
43+
SELECT TOP 3 id, field1, field2 FROM source_data
44+
45+
{% endif %}
46+
"""
47+
48+
_MODELS__INCREMENTAL_SYNC_REMOVE_ONLY_TARGET = """
49+
{{
50+
config(materialized='table')
51+
}}
52+
53+
with source_data as (
54+
55+
select * from {{ ref('model_a') }}
56+
57+
)
58+
59+
{% set string_type = dbt.type_string() %}
60+
61+
select id
62+
,cast(field1 as {{string_type}}) as field1
63+
64+
from source_data
65+
"""
66+
67+
_MODELS__INCREMENTAL_SYNC_ALL_COLUMNS_TARGET = """
68+
{{
69+
config(materialized='table')
70+
}}
71+
72+
with source_data as (
73+
74+
select * from {{ ref('model_a') }}
75+
76+
)
77+
78+
{% set string_type = dbt.type_string() %}
79+
80+
select id
81+
,cast(field1 as {{string_type}}) as field1
82+
--,field2
83+
,cast(case when id <= 3 then null else field3 end as {{string_type}}) as field3
84+
,cast(case when id <= 3 then null else field4 end as {{string_type}}) as field4
85+
86+
from source_data
87+
"""
88+
389

490
class TestBaseIncrementalUniqueKeySQLServer(BaseIncrementalUniqueKey):
591
pass
92+
93+
94+
class TestIncrementalOnSchemaChangeSQLServer(BaseIncrementalOnSchemaChange):
95+
@pytest.fixture(scope="class")
96+
def models(self):
97+
return {
98+
"incremental_sync_remove_only.sql": _MODELS__INCREMENTAL_SYNC_REMOVE_ONLY,
99+
"incremental_ignore.sql": _MODELS__INCREMENTAL_IGNORE,
100+
"incremental_sync_remove_only_target.sql": _MODELS__INCREMENTAL_SYNC_REMOVE_ONLY_TARGET, # noqa: E501
101+
"incremental_ignore_target.sql": _MODELS__INCREMENTAL_IGNORE_TARGET,
102+
"incremental_fail.sql": _MODELS__INCREMENTAL_FAIL,
103+
"incremental_sync_all_columns.sql": _MODELS__INCREMENTAL_SYNC_ALL_COLUMNS,
104+
"incremental_append_new_columns_remove_one.sql": _MODELS__INCREMENTAL_APPEND_NEW_COLUMNS_REMOVE_ONE, # noqa: E501
105+
"model_a.sql": _MODELS__A,
106+
"incremental_append_new_columns_target.sql": _MODELS__INCREMENTAL_APPEND_NEW_COLUMNS_TARGET, # noqa: E501
107+
"incremental_append_new_columns.sql": _MODELS__INCREMENTAL_APPEND_NEW_COLUMNS,
108+
"incremental_sync_all_columns_target.sql": _MODELS__INCREMENTAL_SYNC_ALL_COLUMNS_TARGET, # noqa: E501
109+
"incremental_append_new_columns_remove_one_target.sql": _MODELS__INCREMENTAL_APPEND_NEW_COLUMNS_REMOVE_ONE_TARGET, # noqa: E501
110+
}
111+
112+
113+
class TestIncrementalPredicatesDeleteInsertSQLServer(BaseIncrementalPredicates):
114+
pass
115+
116+
117+
class TestPredicatesDeleteInsertSQLServer(BaseIncrementalPredicates):
118+
@pytest.fixture(scope="class")
119+
def project_config_update(self):
120+
return {"models": {"+predicates": ["id != 2"], "+incremental_strategy": "delete+insert"}}

‎tests/functional/adapter/test_seed.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -49,7 +49,7 @@
4949
{% endfor %}
5050
5151
{% set col_type = col_types.get(column_name) %}
52-
{% set col_type = 'text' if col_type and 'character varying' in col_type else col_type %}
52+
{% set col_type = 'text' if col_type and 'varchar' in col_type else col_type %}
5353
5454
{% set validation_message = 'Got a column type of ' ~ col_type ~ ', expected ' ~ type %}
5555

0 commit comments

Comments
 (0)