diff --git a/.github/workflows/integration-tests-sqlserver.yml b/.github/workflows/integration-tests-sqlserver.yml index f821d1b0..5c7694a6 100644 --- a/.github/workflows/integration-tests-sqlserver.yml +++ b/.github/workflows/integration-tests-sqlserver.yml @@ -18,7 +18,7 @@ jobs: name: Regular strategy: matrix: - python_version: ["3.8", "3.9", "3.10", "3.11", "3.12"] + python_version: ["3.9", "3.10", "3.11", "3.12"] msodbc_version: ["17", "18"] sqlserver_version: ["2017", "2019", "2022"] collation: ["SQL_Latin1_General_CP1_CS_AS", "SQL_Latin1_General_CP1_CI_AS"] diff --git a/.github/workflows/publish-docker.yml b/.github/workflows/publish-docker.yml index 4a312ea3..63b1f5c4 100644 --- a/.github/workflows/publish-docker.yml +++ b/.github/workflows/publish-docker.yml @@ -12,7 +12,7 @@ jobs: publish-docker-client: strategy: matrix: - python_version: ["3.7", "3.8", "3.9", "3.10", "3.11", "3.12"] + python_version: ["3.9", "3.10", "3.11", "3.12"] docker_target: ["msodbc17", "msodbc18"] runs-on: ubuntu-latest permissions: diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml index a04ecc1b..5acae556 100644 --- a/.github/workflows/unit-tests.yml +++ b/.github/workflows/unit-tests.yml @@ -18,7 +18,7 @@ jobs: name: Unit tests strategy: matrix: - python_version: ["3.8", "3.9", "3.10", "3.11", "3.12"] + python_version: ["3.9", "3.10", "3.11", "3.12"] runs-on: ubuntu-latest permissions: contents: read diff --git a/dbt/adapters/sqlserver/__version__.py b/dbt/adapters/sqlserver/__version__.py index 9675f250..7aba6409 100644 --- a/dbt/adapters/sqlserver/__version__.py +++ b/dbt/adapters/sqlserver/__version__.py @@ -1 +1 @@ -version = "1.8.7" +version = "1.9.0" diff --git a/dbt/adapters/sqlserver/sqlserver_adapter.py b/dbt/adapters/sqlserver/sqlserver_adapter.py index 646e5e23..6f05c501 100644 --- a/dbt/adapters/sqlserver/sqlserver_adapter.py +++ b/dbt/adapters/sqlserver/sqlserver_adapter.py @@ -59,3 +59,9 @@ def render_model_constraint(cls, constraint) -> Optional[str]: @classmethod def date_function(cls): return "getdate()" + + def valid_incremental_strategies(self): + """The set of standard builtin strategies which this adapter supports out-of-the-box. + Not used to validate custom strategies defined by end users. + """ + return ["append", "delete+insert", "merge", "microbatch"] diff --git a/dbt/adapters/sqlserver/sqlserver_relation.py b/dbt/adapters/sqlserver/sqlserver_relation.py index 279634f3..f95bdd7f 100644 --- a/dbt/adapters/sqlserver/sqlserver_relation.py +++ b/dbt/adapters/sqlserver/sqlserver_relation.py @@ -1,7 +1,7 @@ from dataclasses import dataclass, field from typing import Optional, Type -from dbt.adapters.base.relation import BaseRelation +from dbt.adapters.base.relation import BaseRelation, EventTimeFilter from dbt.adapters.utils import classproperty from dbt_common.exceptions import DbtRuntimeError @@ -49,3 +49,28 @@ def __post_init__(self): def relation_max_name_length(self): return MAX_CHARACTERS_IN_IDENTIFIER + + def _render_event_time_filtered(self, event_time_filter: EventTimeFilter) -> str: + """ + Returns "" if start and end are both None + """ + filter = "" + if event_time_filter.start and event_time_filter.end: + filter = ( + f"{event_time_filter.field_name} >=" + f" cast('{event_time_filter.start}' as datetimeoffset)" + f" and {event_time_filter.field_name} <" + f" cast('{event_time_filter.end}' as datetimeoffset)" + ) + elif event_time_filter.start: + filter = ( + f"{event_time_filter.field_name} >=" + f" cast('{event_time_filter.start}' as datetimeoffset)" + ) + elif event_time_filter.end: + filter = ( + f"{event_time_filter.field_name} <" + f" cast('{event_time_filter.end}' as datetimeoffset)" + ) + + return filter diff --git a/dbt/include/sqlserver/dbt_project.yml b/dbt/include/sqlserver/dbt_project.yml index 511daca7..7b88fd58 100644 --- a/dbt/include/sqlserver/dbt_project.yml +++ b/dbt/include/sqlserver/dbt_project.yml @@ -1,5 +1,5 @@ name: dbt_sqlserver -version: 1.8.0 +version: 1.9.0 config-version: 2 macro-paths: ["macros"] diff --git a/dbt/include/sqlserver/macros/adapter/columns.sql b/dbt/include/sqlserver/macros/adapter/columns.sql index 205ebefb..a98750e7 100644 --- a/dbt/include/sqlserver/macros/adapter/columns.sql +++ b/dbt/include/sqlserver/macros/adapter/columns.sql @@ -1,3 +1,28 @@ +{% macro sqlserver__get_empty_subquery_sql(select_sql, select_sql_header=none) %} + {% if select_sql.strip().lower().startswith('with') %} + {{ select_sql }} + {% else -%} + select * from ( + {{ select_sql }} + ) dbt_sbq_tmp + where 1 = 0 + {%- endif -%} + +{% endmacro %} + +{% macro sqlserver__get_columns_in_query(select_sql) %} + {% set query_label = apply_label() %} + {% call statement('get_columns_in_query', fetch_result=True, auto_begin=False) -%} + select TOP 0 * from ( + {{ select_sql }} + ) as __dbt_sbq + where 0 = 1 + {{ query_label }} + {% endcall %} + + {{ return(load_result('get_columns_in_query').table.columns | map(attribute='name') | list) }} +{% endmacro %} + {% macro sqlserver__alter_column_type(relation, column_name, new_column_type) %} {%- set tmp_column = column_name + "__dbt_alter" -%} diff --git a/dbt/include/sqlserver/macros/materializations/models/incremental/merge.sql b/dbt/include/sqlserver/macros/materializations/models/incremental/merge.sql new file mode 100644 index 00000000..9d8cdc0f --- /dev/null +++ b/dbt/include/sqlserver/macros/materializations/models/incremental/merge.sql @@ -0,0 +1,31 @@ +{% macro sqlserver__get_incremental_microbatch_sql(arg_dict) %} + {%- set target = arg_dict["target_relation"] -%} + {%- set source = arg_dict["temp_relation"] -%} + {%- set dest_columns = arg_dict["dest_columns"] -%} + {%- set incremental_predicates = [] if arg_dict.get('incremental_predicates') is none else arg_dict.get('incremental_predicates') -%} + + {#-- Add additional incremental_predicates to filter for batch --#} + {% if model.config.get("__dbt_internal_microbatch_event_time_start") -%} + {{ log("incremental append event start time > DBT_INTERNAL_TARGET." ~ model.config.event_time ~ " >= cast('" ~ model.config.__dbt_internal_microbatch_event_time_start ~ "' as datetimeoffset)") }} + {% do incremental_predicates.append("DBT_INTERNAL_TARGET." ~ model.config.event_time ~ " >= cast('" ~ model.config.__dbt_internal_microbatch_event_time_start ~ "' as datetimeoffset)") %} + {% endif %} + {% if model.config.__dbt_internal_microbatch_event_time_end -%} + {{ log("incremental append event end time < DBT_INTERNAL_TARGET." ~ model.config.event_time ~ " < cast('" ~ model.config.__dbt_internal_microbatch_event_time_end ~ "' as datetimeoffset)") }} + {% do incremental_predicates.append("DBT_INTERNAL_TARGET." ~ model.config.event_time ~ " < cast('" ~ model.config.__dbt_internal_microbatch_event_time_end ~ "' as datetimeoffset)") %} + {% endif %} + {% do arg_dict.update({'incremental_predicates': incremental_predicates}) %} + + delete DBT_INTERNAL_TARGET from {{ target }} AS DBT_INTERNAL_TARGET + where ( + {% for predicate in incremental_predicates %} + {%- if not loop.first %}and {% endif -%} {{ predicate }} + {% endfor %} + ); + + {%- set dest_cols_csv = get_quoted_csv(dest_columns | map(attribute="name")) -%} + insert into {{ target }} ({{ dest_cols_csv }}) + ( + select {{ dest_cols_csv }} + from {{ source }} + ) +{% endmacro %} diff --git a/dev_requirements.txt b/dev_requirements.txt index 4db3f028..2a3c4c4a 100644 --- a/dev_requirements.txt +++ b/dev_requirements.txt @@ -1,5 +1,5 @@ -dbt-tests-adapter>=1.8.0, <1.9.0 +dbt-tests-adapter>=1.9.0,<2.0 ruff black==24.8.0 diff --git a/setup.py b/setup.py index 4e491bee..0a63ce6d 100644 --- a/setup.py +++ b/setup.py @@ -8,7 +8,7 @@ package_name = "dbt-sqlserver" authors_list = ["Mikael Ene", "Anders Swanson", "Sam Debruyn", "Cor Zuurmond", "Cody Scott"] -dbt_version = "1.8" +dbt_version = "1.9" description = """A Microsoft SQL Server adapter plugin for dbt""" this_directory = os.path.abspath(os.path.dirname(__file__)) @@ -66,10 +66,10 @@ def run(self): packages=find_namespace_packages(include=["dbt", "dbt.*"]), include_package_data=True, install_requires=[ - "dbt-fabric>=1.8.0,<1.9.0", - "dbt-core>=1.8.0,<1.9.0", + "dbt-fabric==1.9.3", + "dbt-core>=1.9.0,<2.0", "dbt-common>=1.0,<2.0", - "dbt-adapters>=1.1.1,<2.0", + "dbt-adapters>=1.11.0,<2.0", ], cmdclass={ "verify": VerifyVersionCommand, diff --git a/tests/functional/adapter/dbt/test_empty.py b/tests/functional/adapter/dbt/test_empty.py index 52ea88de..391b5b3b 100644 --- a/tests/functional/adapter/dbt/test_empty.py +++ b/tests/functional/adapter/dbt/test_empty.py @@ -1,12 +1,11 @@ import pytest +from dbt.tests.adapter.empty._models import model_input_sql, schema_sources_yml # switch for 1.9 # from dbt.tests.adapter.empty import _models from dbt.tests.adapter.empty.test_empty import ( # MetadataWithEmptyFlag BaseTestEmpty, BaseTestEmptyInlineSourceRef, - model_input_sql, - schema_sources_yml, ) from dbt.tests.util import run_dbt diff --git a/tests/functional/adapter/dbt/test_incremental.py b/tests/functional/adapter/dbt/test_incremental.py index 959a81d0..b3a5c9cb 100644 --- a/tests/functional/adapter/dbt/test_incremental.py +++ b/tests/functional/adapter/dbt/test_incremental.py @@ -1,8 +1,5 @@ import pytest from dbt.tests.adapter.incremental import fixtures -from dbt.tests.adapter.incremental.test_incremental_merge_exclude_columns import ( - BaseMergeExcludeColumns, -) from dbt.tests.adapter.incremental.test_incremental_on_schema_change import ( BaseIncrementalOnSchemaChange, ) @@ -76,10 +73,6 @@ """ -class TestIncrementalMergeExcludeColumns(BaseMergeExcludeColumns): - pass - - class TestIncrementalOnSchemaChange(BaseIncrementalOnSchemaChange): @pytest.fixture(scope="class") def models(self): diff --git a/tests/functional/adapter/dbt/test_incremental_microbatch_datetime.py b/tests/functional/adapter/dbt/test_incremental_microbatch_datetime.py new file mode 100644 index 00000000..88581a6a --- /dev/null +++ b/tests/functional/adapter/dbt/test_incremental_microbatch_datetime.py @@ -0,0 +1,47 @@ +import pytest +from dbt.tests.adapter.incremental.test_incremental_microbatch import BaseMicrobatch + +_microbatch_model_no_unique_id_sql_datetime = """ +{{ config(materialized='incremental', incremental_strategy='microbatch', +event_time='event_time', batch_size='day', begin='2020-01-01 00:00:00') }} +select * from {{ ref('input_model') }} +""" + +_input_model_sql_datetime = """ +{{ config(materialized='table', event_time='event_time') }} +select 1 as id, '2020-01-01 00:00:00' as event_time +union all +select 2 as id, '2020-01-02 00:00:00' as event_time +union all +select 3 as id, '2020-01-03 00:00:00' as event_time +""" + + +class TestSQLServerMicrobatchDateTime(BaseMicrobatch): + """ + Setup a version of the microbatch testing that uses a datetime column as the event_time + This is to test that the microbatch strategy can handle datetime columns when passing in + event times as UTC strings + """ + + @pytest.fixture(scope="class") + def microbatch_model_sql(self) -> str: + return _microbatch_model_no_unique_id_sql_datetime + + @pytest.fixture(scope="class") + def input_model_sql(self) -> str: + """ + This is the SQL that defines the input model to the microbatch model, + including any {{ config(..) }}. event_time is a required configuration of this input + """ + return _input_model_sql_datetime + + @pytest.fixture(scope="class") + def insert_two_rows_sql(self, project) -> str: + test_schema_relation = project.adapter.Relation.create( + database=project.database, schema=project.test_schema + ) + return ( + f"insert into {test_schema_relation}.input_model (id, event_time) " + f"values (4, '2020-01-04 00:00:00'), (5, '2020-01-05 00:00:00')" + )