diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 47da7d4d..a706161e 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -4,7 +4,8 @@ "features": { "ghcr.io/devcontainers/features/docker-in-docker:2": {} }, - "postStartCommand": "sudo bash .devcontainer/setup_odbc.sh && bash .devcontainer/install_pyenv.sh && bash .devcontainer/setup_env.sh", + "forwardPorts": [1433], + "postStartCommand": "/bin/bash ./.devcontainer/setup_odbc.sh & /bin/bash ./.devcontainer/setup_env.sh", "containerEnv": { "SQLSERVER_TEST_DRIVER": "ODBC Driver 18 for SQL Server", "SQLSERVER_TEST_HOST": "127.0.0.1", diff --git a/.devcontainer/setup_env.sh b/.devcontainer/setup_env.sh index 2b67fe7d..f15a901c 100644 --- a/.devcontainer/setup_env.sh +++ b/.devcontainer/setup_env.sh @@ -1,8 +1,6 @@ cp test.env.sample test.env -pyenv install 3.10.7 -pyenv virtualenv 3.10.7 dbt-sqlserver -pyenv activate dbt-sqlserver +docker compose build +docker compose up -d -make dev -make server +pip install -r dev_requirements.txt diff --git a/.gitignore b/.gitignore index 3b122e2f..1e5f776c 100644 --- a/.gitignore +++ b/.gitignore @@ -96,3 +96,5 @@ ENV/ env.bak/ venv.bak/ .mise.toml + +**devcontainer-lock.json** diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index a916f3ea..3eaee28d 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -2,7 +2,7 @@ default_language_version: python: python3.10 repos: - repo: 'https://github.com/pre-commit/pre-commit-hooks' - rev: v4.4.0 + rev: v4.6.0 hooks: - id: check-yaml args: @@ -21,7 +21,7 @@ repos: - id: mixed-line-ending - id: check-docstring-first - repo: 'https://github.com/adrienverge/yamllint' - rev: v1.32.0 + rev: v1.35.1 hooks: - id: yamllint args: @@ -32,13 +32,13 @@ repos: hooks: - id: absolufy-imports - repo: 'https://github.com/hadialqattan/pycln' - rev: v2.1.3 + rev: v2.4.0 hooks: - id: pycln args: - '--all' - repo: 'https://github.com/pycqa/isort' - rev: 5.12.0 + rev: 5.13.2 hooks: - id: isort args: @@ -50,7 +50,7 @@ repos: - '--python-version' - '39' - repo: 'https://github.com/psf/black' - rev: 23.3.0 + rev: 24.8.0 hooks: - id: black args: @@ -66,7 +66,7 @@ repos: - '--check' - '--diff' - repo: 'https://github.com/pycqa/flake8' - rev: 6.0.0 + rev: 7.1.1 hooks: - id: flake8 args: @@ -78,7 +78,7 @@ repos: stages: - manual - repo: 'https://github.com/pre-commit/mirrors-mypy' - rev: v1.3.0 + rev: v1.11.1 hooks: - id: mypy args: diff --git a/CHANGELOG.md b/CHANGELOG.md index 1b37cbec..62f5e6a2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,26 @@ # Changelog +### v1.8.0 + +Updates dbt-sqlserver to support dbt 1.8. + +Notable changes + +- Adopts `dbt-common` and `dbt-adapters` as the upstream, in line with dbt projects. +- Implements the majority of the tests from the `dbt-test-adapters` project to provide better coverage. +- Implements better testing for `dbt-sqlserver` specific functions, including indexes. +- Realigns to closer to the global project, overriding some fabric specific implementations + +Update also fixes a number of regressions related to the fabric adapter. These include + +- Proper ALTER syntax for column changes (in both ) + - https://github.com/dbt-msft/dbt-sqlserver/pull/504/files +- Restoring cluster columntables post create on `tables` + - https://github.com/dbt-msft/dbt-sqlserver/issues/473 +- Adds proper constraints to tables and columns + - https://github.com/dbt-msft/dbt-sqlserver/pull/500 + + ### v1.7.2 Huge thanks to GitHub users **@cody-scott** and **@prescode** for help with this long-awaited update to enable `dbt-core` 1.7.2 compatibility! diff --git a/dbt/adapters/sqlserver/__init__.py b/dbt/adapters/sqlserver/__init__.py index 6bd51201..879ea74c 100644 --- a/dbt/adapters/sqlserver/__init__.py +++ b/dbt/adapters/sqlserver/__init__.py @@ -1,10 +1,10 @@ from dbt.adapters.base import AdapterPlugin -from dbt.adapters.sqlserver.sql_server_adapter import SQLServerAdapter -from dbt.adapters.sqlserver.sql_server_column import SQLServerColumn -from dbt.adapters.sqlserver.sql_server_configs import SQLServerConfigs -from dbt.adapters.sqlserver.sql_server_connection_manager import SQLServerConnectionManager -from dbt.adapters.sqlserver.sql_server_credentials import SQLServerCredentials +from dbt.adapters.sqlserver.sqlserver_adapter import SQLServerAdapter +from dbt.adapters.sqlserver.sqlserver_column import SQLServerColumn +from dbt.adapters.sqlserver.sqlserver_configs import SQLServerConfigs +from dbt.adapters.sqlserver.sqlserver_connections import SQLServerConnectionManager # noqa +from dbt.adapters.sqlserver.sqlserver_credentials import SQLServerCredentials from dbt.include import sqlserver Plugin = AdapterPlugin( diff --git a/dbt/adapters/sqlserver/__version__.py b/dbt/adapters/sqlserver/__version__.py index 582554e8..6aaa73b8 100644 --- a/dbt/adapters/sqlserver/__version__.py +++ b/dbt/adapters/sqlserver/__version__.py @@ -1 +1 @@ -version = "1.7.4" +version = "1.8.0" diff --git a/dbt/adapters/sqlserver/relation_configs/__init__.py b/dbt/adapters/sqlserver/relation_configs/__init__.py new file mode 100644 index 00000000..b93c52a0 --- /dev/null +++ b/dbt/adapters/sqlserver/relation_configs/__init__.py @@ -0,0 +1,13 @@ +from dbt.adapters.sqlserver.relation_configs.policies import ( + MAX_CHARACTERS_IN_IDENTIFIER, + SQLServerIncludePolicy, + SQLServerQuotePolicy, + SQLServerRelationType, +) + +__all__ = [ + "MAX_CHARACTERS_IN_IDENTIFIER", + "SQLServerIncludePolicy", + "SQLServerQuotePolicy", + "SQLServerRelationType", +] diff --git a/dbt/adapters/sqlserver/relation_configs/policies.py b/dbt/adapters/sqlserver/relation_configs/policies.py new file mode 100644 index 00000000..e53ed9cd --- /dev/null +++ b/dbt/adapters/sqlserver/relation_configs/policies.py @@ -0,0 +1,25 @@ +from dataclasses import dataclass + +from dbt.adapters.contracts.relation import Policy +from dbt_common.dataclass_schema import StrEnum + +MAX_CHARACTERS_IN_IDENTIFIER = 127 + + +class SQLServerRelationType(StrEnum): + Table = "table" + View = "view" + CTE = "cte" + + +class SQLServerIncludePolicy(Policy): + database: bool = True + schema: bool = True + identifier: bool = True + + +@dataclass +class SQLServerQuotePolicy(Policy): + database: bool = True + schema: bool = True + identifier: bool = True diff --git a/dbt/adapters/sqlserver/sql_server_adapter.py b/dbt/adapters/sqlserver/sql_server_adapter.py deleted file mode 100644 index e444b26a..00000000 --- a/dbt/adapters/sqlserver/sql_server_adapter.py +++ /dev/null @@ -1,86 +0,0 @@ -# https://github.com/microsoft/dbt-fabric/blob/main/dbt/adapters/fabric/fabric_adapter.py -from typing import Optional - -import dbt.exceptions -from dbt.adapters.fabric import FabricAdapter -from dbt.contracts.graph.nodes import ConstraintType, ModelLevelConstraint - -from dbt.adapters.sqlserver.sql_server_column import SQLServerColumn -from dbt.adapters.sqlserver.sql_server_configs import SQLServerConfigs -from dbt.adapters.sqlserver.sql_server_connection_manager import SQLServerConnectionManager - -# from dbt.adapters.capability import Capability, CapabilityDict, CapabilitySupport, Support - - -class SQLServerAdapter(FabricAdapter): - ConnectionManager = SQLServerConnectionManager - Column = SQLServerColumn - AdapterSpecificConfigs = SQLServerConfigs - - # _capabilities: CapabilityDict = CapabilityDict( - # { - # Capability.SchemaMetadataByRelations: CapabilitySupport(support=Support.Full), - # Capability.TableLastModifiedMetadata: CapabilitySupport(support=Support.Full), - # } - # ) - - # region - these are implement in fabric but not in sqlserver - # _capabilities: CapabilityDict = CapabilityDict( - # { - # Capability.SchemaMetadataByRelations: CapabilitySupport(support=Support.Full), - # Capability.TableLastModifiedMetadata: CapabilitySupport(support=Support.Full), - # } - # ) - # CONSTRAINT_SUPPORT = { - # ConstraintType.check: ConstraintSupport.NOT_SUPPORTED, - # ConstraintType.not_null: ConstraintSupport.ENFORCED, - # ConstraintType.unique: ConstraintSupport.ENFORCED, - # ConstraintType.primary_key: ConstraintSupport.ENFORCED, - # ConstraintType.foreign_key: ConstraintSupport.ENFORCED, - # } - - # @available.parse(lambda *a, **k: []) - # def get_column_schema_from_query(self, sql: str) -> List[BaseColumn]: - # """Get a list of the Columns with names and data types from the given sql.""" - # _, cursor = self.connections.add_select_query(sql) - - # columns = [ - # self.Column.create( - # column_name, self.connections.data_type_code_to_name(column_type_code) - # ) - # # https://peps.python.org/pep-0249/#description - # for column_name, column_type_code, *_ in cursor.description - # ] - # return columns - # endregion - - @classmethod - def render_model_constraint(cls, constraint: ModelLevelConstraint) -> Optional[str]: - constraint_prefix = "add constraint " - column_list = ", ".join(constraint.columns) - - if constraint.name is None: - raise dbt.exceptions.DbtDatabaseError( - "Constraint name cannot be empty. Provide constraint name - column " - + column_list - + " and run the project again." - ) - - if constraint.type == ConstraintType.unique: - return constraint_prefix + f"{constraint.name} unique nonclustered({column_list})" - elif constraint.type == ConstraintType.primary_key: - return constraint_prefix + f"{constraint.name} primary key nonclustered({column_list})" - elif constraint.type == ConstraintType.foreign_key and constraint.expression: - return ( - constraint_prefix - + f"{constraint.name} foreign key({column_list}) references " - + constraint.expression - ) - elif constraint.type == ConstraintType.custom and constraint.expression: - return f"{constraint_prefix}{constraint.expression}" - else: - return None - - @classmethod - def date_function(cls): - return "getdate()" diff --git a/dbt/adapters/sqlserver/sql_server_column.py b/dbt/adapters/sqlserver/sql_server_column.py deleted file mode 100644 index 2ee5f7b7..00000000 --- a/dbt/adapters/sqlserver/sql_server_column.py +++ /dev/null @@ -1,5 +0,0 @@ -from dbt.adapters.fabric import FabricColumn - - -class SQLServerColumn(FabricColumn): - ... diff --git a/dbt/adapters/sqlserver/sqlserver_adapter.py b/dbt/adapters/sqlserver/sqlserver_adapter.py new file mode 100644 index 00000000..646e5e23 --- /dev/null +++ b/dbt/adapters/sqlserver/sqlserver_adapter.py @@ -0,0 +1,61 @@ +from typing import Optional + +import dbt.exceptions +from dbt.adapters.base.impl import ConstraintSupport +from dbt.adapters.fabric import FabricAdapter +from dbt.contracts.graph.nodes import ConstraintType + +from dbt.adapters.sqlserver.sqlserver_column import SQLServerColumn +from dbt.adapters.sqlserver.sqlserver_connections import SQLServerConnectionManager +from dbt.adapters.sqlserver.sqlserver_relation import SQLServerRelation + + +class SQLServerAdapter(FabricAdapter): + """ + Controls actual implmentation of adapter, and ability to override certain methods. + """ + + ConnectionManager = SQLServerConnectionManager + Column = SQLServerColumn + Relation = SQLServerRelation + + CONSTRAINT_SUPPORT = { + ConstraintType.check: ConstraintSupport.ENFORCED, + ConstraintType.not_null: ConstraintSupport.ENFORCED, + ConstraintType.unique: ConstraintSupport.ENFORCED, + ConstraintType.primary_key: ConstraintSupport.ENFORCED, + ConstraintType.foreign_key: ConstraintSupport.ENFORCED, + } + + @classmethod + def render_model_constraint(cls, constraint) -> Optional[str]: + constraint_prefix = "add constraint " + column_list = ", ".join(constraint.columns) + + if constraint.name is None: + raise dbt.exceptions.DbtDatabaseError( + "Constraint name cannot be empty. Provide constraint name - column " + + column_list + + " and run the project again." + ) + + if constraint.type == ConstraintType.unique: + return constraint_prefix + f"{constraint.name} unique nonclustered({column_list})" + elif constraint.type == ConstraintType.primary_key: + return constraint_prefix + f"{constraint.name} primary key nonclustered({column_list})" + elif constraint.type == ConstraintType.foreign_key and constraint.expression: + return ( + constraint_prefix + + f"{constraint.name} foreign key({column_list}) references " + + constraint.expression + ) + elif constraint.type == ConstraintType.check and constraint.expression: + return f"{constraint_prefix} {constraint.name} check ({constraint.expression})" + elif constraint.type == ConstraintType.custom and constraint.expression: + return f"{constraint_prefix} {constraint.name} {constraint.expression}" + else: + return None + + @classmethod + def date_function(cls): + return "getdate()" diff --git a/dbt/adapters/sqlserver/sqlserver_column.py b/dbt/adapters/sqlserver/sqlserver_column.py new file mode 100644 index 00000000..68ef98e3 --- /dev/null +++ b/dbt/adapters/sqlserver/sqlserver_column.py @@ -0,0 +1,22 @@ +from dbt.adapters.fabric import FabricColumn + + +class SQLServerColumn(FabricColumn): + def is_integer(self) -> bool: + return self.dtype.lower() in [ + # real types + "smallint", + "integer", + "bigint", + "smallserial", + "serial", + "bigserial", + # aliases + "int2", + "int4", + "int8", + "serial2", + "serial4", + "serial8", + "int", + ] diff --git a/dbt/adapters/sqlserver/sql_server_configs.py b/dbt/adapters/sqlserver/sqlserver_configs.py similarity index 93% rename from dbt/adapters/sqlserver/sql_server_configs.py rename to dbt/adapters/sqlserver/sqlserver_configs.py index 2804179b..35ce4262 100644 --- a/dbt/adapters/sqlserver/sql_server_configs.py +++ b/dbt/adapters/sqlserver/sqlserver_configs.py @@ -5,4 +5,4 @@ @dataclass class SQLServerConfigs(FabricConfigs): - ... + pass diff --git a/dbt/adapters/sqlserver/sql_server_connection_manager.py b/dbt/adapters/sqlserver/sqlserver_connections.py similarity index 69% rename from dbt/adapters/sqlserver/sql_server_connection_manager.py rename to dbt/adapters/sqlserver/sqlserver_connections.py index cf42bff4..79852252 100644 --- a/dbt/adapters/sqlserver/sql_server_connection_manager.py +++ b/dbt/adapters/sqlserver/sqlserver_connections.py @@ -1,8 +1,9 @@ -from typing import Callable, Mapping - +import dbt_common.exceptions # noqa import pyodbc from azure.core.credentials import AccessToken from azure.identity import ClientSecretCredential, ManagedIdentityCredential +from dbt.adapters.contracts.connection import Connection, ConnectionState +from dbt.adapters.events.logging import AdapterLogger from dbt.adapters.fabric import FabricConnectionManager from dbt.adapters.fabric.fabric_connection_manager import ( AZURE_AUTH_FUNCTIONS as AZURE_AUTH_FUNCTIONS_FABRIC, @@ -12,15 +13,11 @@ bool_to_connection_string_arg, get_pyodbc_attrs_before, ) -from dbt.contracts.connection import Connection, ConnectionState -from dbt.events import AdapterLogger from dbt.adapters.sqlserver import __version__ -from dbt.adapters.sqlserver.sql_server_credentials import SQLServerCredentials - -AZURE_AUTH_FUNCTION_TYPE = Callable[[SQLServerCredentials], AccessToken] +from dbt.adapters.sqlserver.sqlserver_credentials import SQLServerCredentials -logger = AdapterLogger("SQLServer") +logger = AdapterLogger("sqlserver") def get_msi_access_token(credentials: SQLServerCredentials) -> AccessToken: @@ -63,7 +60,7 @@ def get_sp_access_token(credentials: SQLServerCredentials) -> AccessToken: return token -AZURE_AUTH_FUNCTIONS: Mapping[str, AZURE_AUTH_FUNCTION_TYPE] = { +AZURE_AUTH_FUNCTIONS = { **AZURE_AUTH_FUNCTIONS_FABRIC, "serviceprincipal": get_sp_access_token, "msi": get_msi_access_token, @@ -73,6 +70,27 @@ def get_sp_access_token(credentials: SQLServerCredentials) -> AccessToken: class SQLServerConnectionManager(FabricConnectionManager): TYPE = "sqlserver" + # @contextmanager + # def exception_handler(self, sql: str): + # """ + # Returns a context manager, that will handle exceptions raised + # from queries, catch, log, and raise dbt exceptions it knows how to handle. + # """ + # # ## Example ## + # # try: + # # yield + # # except myadapter_library.DatabaseError as exc: + # # self.release(connection_name) + + # # logger.debug("myadapter error: {}".format(str(e))) + # # raise dbt.exceptions.DatabaseException(str(exc)) + # # except Exception as exc: + # # logger.debug("Error running SQL: {}".format(sql)) + # # logger.debug("Rolling back transaction.") + # # self.release(connection_name) + # # raise dbt.exceptions.RuntimeException(str(exc)) + # pass + @classmethod def open(cls, connection: Connection) -> Connection: if connection.state == ConnectionState.OPEN: @@ -156,3 +174,28 @@ def connect(): retry_limit=credentials.retries, retryable_exceptions=retryable_exceptions, ) + + # @classmethod + # def get_response(cls,cursor): + # """ + # Gets a cursor object and returns adapter-specific information + # about the last executed command generally a AdapterResponse ojbect + # that has items such as code, rows_affected,etc. can also just be a string ex. "OK" + # if your cursor does not offer rich metadata. + # """ + # # ## Example ## + # # return cursor.status_message + # pass + + # def cancel(self, connection): + # """ + # Gets a connection object and attempts to cancel any ongoing queries. + # """ + # # ## Example ## + # # tid = connection.handle.transaction_id() + # # sql = "select cancel_transaction({})".format(tid) + # # logger.debug("Cancelling query "{}" ({})".format(connection_name, pid)) + # # _, cursor = self.add_query(sql, "master") + # # res = cursor.fetchone() + # # logger.debug("Canceled query "{}": {}".format(connection_name, res)) + # pass diff --git a/dbt/adapters/sqlserver/sql_server_credentials.py b/dbt/adapters/sqlserver/sqlserver_credentials.py similarity index 76% rename from dbt/adapters/sqlserver/sql_server_credentials.py rename to dbt/adapters/sqlserver/sqlserver_credentials.py index db9274d3..bf1f5075 100644 --- a/dbt/adapters/sqlserver/sql_server_credentials.py +++ b/dbt/adapters/sqlserver/sqlserver_credentials.py @@ -6,6 +6,11 @@ @dataclass class SQLServerCredentials(FabricCredentials): + """ + Defines database specific credentials that get added to + profiles.yml to connect to new adapter + """ + port: Optional[int] = 1433 authentication: Optional[str] = "sql" diff --git a/dbt/adapters/sqlserver/sqlserver_relation.py b/dbt/adapters/sqlserver/sqlserver_relation.py new file mode 100644 index 00000000..279634f3 --- /dev/null +++ b/dbt/adapters/sqlserver/sqlserver_relation.py @@ -0,0 +1,51 @@ +from dataclasses import dataclass, field +from typing import Optional, Type + +from dbt.adapters.base.relation import BaseRelation +from dbt.adapters.utils import classproperty +from dbt_common.exceptions import DbtRuntimeError + +from dbt.adapters.sqlserver.relation_configs import ( + MAX_CHARACTERS_IN_IDENTIFIER, + SQLServerIncludePolicy, + SQLServerQuotePolicy, + SQLServerRelationType, +) + + +@dataclass(frozen=True, eq=False, repr=False) +class SQLServerRelation(BaseRelation): + type: Optional[SQLServerRelationType] = None # type: ignore + include_policy: SQLServerIncludePolicy = field( + default_factory=lambda: SQLServerIncludePolicy() + ) + quote_policy: SQLServerQuotePolicy = field(default_factory=lambda: SQLServerQuotePolicy()) + + @classproperty + def get_relation_type(cls) -> Type[SQLServerRelationType]: + return SQLServerRelationType + + def render_limited(self) -> str: + rendered = self.render() + if self.limit is None: + return rendered + elif self.limit == 0: + return f"(select * from {rendered} where 1=0) {self._render_limited_alias()}" + else: + return f"(select TOP {self.limit} * from {rendered}) {self._render_limited_alias()}" + + def __post_init__(self): + # Check for length of Redshift table/view names. + # Check self.type to exclude test relation identifiers + if ( + self.identifier is not None + and self.type is not None + and len(self.identifier) > MAX_CHARACTERS_IN_IDENTIFIER + ): + raise DbtRuntimeError( + f"Relation name '{self.identifier}' " + f"is longer than {MAX_CHARACTERS_IN_IDENTIFIER} characters" + ) + + def relation_max_name_length(self): + return MAX_CHARACTERS_IN_IDENTIFIER diff --git a/dbt/include/sqlserver/dbt_project.yml b/dbt/include/sqlserver/dbt_project.yml index 8952ba41..511daca7 100644 --- a/dbt/include/sqlserver/dbt_project.yml +++ b/dbt/include/sqlserver/dbt_project.yml @@ -1,6 +1,5 @@ name: dbt_sqlserver -version: 1.0 - +version: 1.8.0 config-version: 2 macro-paths: ["macros"] diff --git a/dbt/include/sqlserver/macros/adapter/columns.sql b/dbt/include/sqlserver/macros/adapter/columns.sql new file mode 100644 index 00000000..205ebefb --- /dev/null +++ b/dbt/include/sqlserver/macros/adapter/columns.sql @@ -0,0 +1,25 @@ +{% macro sqlserver__alter_column_type(relation, column_name, new_column_type) %} + + {%- set tmp_column = column_name + "__dbt_alter" -%} + {% set alter_column_type %} + alter {{ relation.type }} {{ relation }} add "{{ tmp_column }}" {{ new_column_type }}; + {%- endset %} + + {% set update_column %} + update {{ relation }} set "{{ tmp_column }}" = "{{ column_name }}"; + {%- endset %} + + {% set drop_column %} + alter {{ relation.type }} {{ relation }} drop column "{{ column_name }}"; + {%- endset %} + + {% set rename_column %} + exec sp_rename '{{ relation | replace('"', '') }}.{{ tmp_column }}', '{{ column_name }}', 'column' + {%- endset %} + + {% do run_query(alter_column_type) %} + {% do run_query(update_column) %} + {% do run_query(drop_column) %} + {% do run_query(rename_column) %} + +{% endmacro %} diff --git a/dbt/include/sqlserver/macros/adapter/indexes.sql b/dbt/include/sqlserver/macros/adapter/indexes.sql new file mode 100644 index 00000000..08906012 --- /dev/null +++ b/dbt/include/sqlserver/macros/adapter/indexes.sql @@ -0,0 +1,169 @@ +{% macro sqlserver__create_clustered_columnstore_index(relation) -%} + {%- set cci_name = (relation.schema ~ '_' ~ relation.identifier ~ '_cci') | replace(".", "") | replace(" ", "") -%} + {%- set relation_name = relation.schema ~ '_' ~ relation.identifier -%} + {%- set full_relation = '"' ~ relation.schema ~ '"."' ~ relation.identifier ~ '"' -%} + use [{{ relation.database }}]; + if EXISTS ( + SELECT * + FROM sys.indexes {{ information_schema_hints() }} + WHERE name = '{{cci_name}}' + AND object_id=object_id('{{relation_name}}') + ) + DROP index {{full_relation}}.{{cci_name}} + CREATE CLUSTERED COLUMNSTORE INDEX {{cci_name}} + ON {{full_relation}} +{% endmacro %} + +{% macro drop_xml_indexes() -%} + {{ log("Running drop_xml_indexes() macro...") }} + + declare @drop_xml_indexes nvarchar(max); + select @drop_xml_indexes = ( + select 'IF INDEXPROPERTY(' + CONVERT(VARCHAR(MAX), sys.tables.[object_id]) + ', ''' + sys.indexes.[name] + ''', ''IndexId'') IS NOT NULL DROP INDEX [' + sys.indexes.[name] + '] ON ' + '[' + SCHEMA_NAME(sys.tables.[schema_id]) + '].[' + OBJECT_NAME(sys.tables.[object_id]) + ']; ' + from sys.indexes {{ information_schema_hints() }} + inner join sys.tables {{ information_schema_hints() }} + on sys.indexes.object_id = sys.tables.object_id + where sys.indexes.[name] is not null + and sys.indexes.type_desc = 'XML' + and sys.tables.[name] = '{{ this.table }}' + for xml path('') + ); exec sp_executesql @drop_xml_indexes; +{%- endmacro %} + + +{% macro drop_spatial_indexes() -%} + {# Altered from https://stackoverflow.com/q/1344401/10415173 #} + {# and https://stackoverflow.com/a/33785833/10415173 #} + + {{ log("Running drop_spatial_indexes() macro...") }} + + declare @drop_spatial_indexes nvarchar(max); + select @drop_spatial_indexes = ( + select 'IF INDEXPROPERTY(' + CONVERT(VARCHAR(MAX), sys.tables.[object_id]) + ', ''' + sys.indexes.[name] + ''', ''IndexId'') IS NOT NULL DROP INDEX [' + sys.indexes.[name] + '] ON ' + '[' + SCHEMA_NAME(sys.tables.[schema_id]) + '].[' + OBJECT_NAME(sys.tables.[object_id]) + ']; ' + from sys.indexes {{ information_schema_hints() }} + inner join sys.tables {{ information_schema_hints() }} + on sys.indexes.object_id = sys.tables.object_id + where sys.indexes.[name] is not null + and sys.indexes.type_desc = 'Spatial' + and sys.tables.[name] = '{{ this.table }}' + for xml path('') + ); exec sp_executesql @drop_spatial_indexes; +{%- endmacro %} + + +{% macro drop_fk_constraints() -%} + {# Altered from https://stackoverflow.com/q/1344401/10415173 #} + + {{ log("Running drop_fk_constraints() macro...") }} + + declare @drop_fk_constraints nvarchar(max); + select @drop_fk_constraints = ( + select 'IF OBJECT_ID(''' + SCHEMA_NAME(CONVERT(VARCHAR(MAX), sys.foreign_keys.[schema_id])) + '.' + sys.foreign_keys.[name] + ''', ''F'') IS NOT NULL ALTER TABLE [' + SCHEMA_NAME(sys.foreign_keys.[schema_id]) + '].[' + OBJECT_NAME(sys.foreign_keys.[parent_object_id]) + '] DROP CONSTRAINT [' + sys.foreign_keys.[name]+ '];' + from sys.foreign_keys + inner join sys.tables on sys.foreign_keys.[referenced_object_id] = sys.tables.[object_id] + where sys.tables.[name] = '{{ this.table }}' + for xml path('') + ); exec sp_executesql @drop_fk_constraints; + +{%- endmacro %} + + +{% macro drop_pk_constraints() -%} + {# Altered from https://stackoverflow.com/q/1344401/10415173 #} + {# and https://stackoverflow.com/a/33785833/10415173 #} + + {{ drop_xml_indexes() }} + + {{ drop_spatial_indexes() }} + + {{ drop_fk_constraints() }} + + {{ log("Running drop_pk_constraints() macro...") }} + + declare @drop_pk_constraints nvarchar(max); + select @drop_pk_constraints = ( + select 'IF INDEXPROPERTY(' + CONVERT(VARCHAR(MAX), sys.tables.[object_id]) + ', ''' + sys.indexes.[name] + ''', ''IndexId'') IS NOT NULL ALTER TABLE [' + SCHEMA_NAME(sys.tables.[schema_id]) + '].[' + sys.tables.[name] + '] DROP CONSTRAINT [' + sys.indexes.[name]+ '];' + from sys.indexes + inner join sys.tables on sys.indexes.[object_id] = sys.tables.[object_id] + where sys.indexes.is_primary_key = 1 + and sys.tables.[name] = '{{ this.table }}' + for xml path('') + ); exec sp_executesql @drop_pk_constraints; + +{%- endmacro %} + + +{% macro drop_all_indexes_on_table() -%} + {# Altered from https://stackoverflow.com/q/1344401/10415173 #} + {# and https://stackoverflow.com/a/33785833/10415173 #} + + {{ drop_pk_constraints() }} + + {{ log("Dropping remaining indexes...") }} + + declare @drop_remaining_indexes_last nvarchar(max); + select @drop_remaining_indexes_last = ( + select 'IF INDEXPROPERTY(' + CONVERT(VARCHAR(MAX), sys.tables.[object_id]) + ', ''' + sys.indexes.[name] + ''', ''IndexId'') IS NOT NULL DROP INDEX [' + sys.indexes.[name] + '] ON ' + '[' + SCHEMA_NAME(sys.tables.[schema_id]) + '].[' + OBJECT_NAME(sys.tables.[object_id]) + ']; ' + from sys.indexes {{ information_schema_hints() }} + inner join sys.tables {{ information_schema_hints() }} + on sys.indexes.object_id = sys.tables.object_id + where sys.indexes.[name] is not null + and sys.tables.[name] = '{{ this.table }}' + for xml path('') + ); exec sp_executesql @drop_remaining_indexes_last; + +{%- endmacro %} + + +{% macro create_clustered_index(columns, unique=False) -%} + {{ log("Creating clustered index...") }} + + {% set idx_name = "clustered_" + local_md5(columns | join("_")) %} + + if not exists(select * + from sys.indexes {{ information_schema_hints() }} + where name = '{{ idx_name }}' + and object_id = OBJECT_ID('{{ this }}') + ) + begin + + create + {% if unique -%} + unique + {% endif %} + clustered index + {{ idx_name }} + on {{ this }} ({{ '[' + columns|join("], [") + ']' }}) + end +{%- endmacro %} + + +{% macro create_nonclustered_index(columns, includes=False) %} + + {{ log("Creating nonclustered index...") }} + + {% if includes -%} + {% set idx_name = ( + "nonclustered_" + + local_md5(columns | join("_")) + + "_incl_" + + local_md5(includes | join("_")) + ) %} + {% else -%} + {% set idx_name = "nonclustered_" + local_md5(columns | join("_")) %} + {% endif %} + + if not exists(select * + from sys.indexes {{ information_schema_hints() }} + where name = '{{ idx_name }}' + and object_id = OBJECT_ID('{{ this }}') + ) + begin + create nonclustered index + {{ idx_name }} + on {{ this }} ({{ '[' + columns|join("], [") + ']' }}) + {% if includes -%} + include ({{ '[' + includes|join("], [") + ']' }}) + {% endif %} + end +{% endmacro %} diff --git a/dbt/include/sqlserver/macros/adapter/metadata.sql b/dbt/include/sqlserver/macros/adapter/metadata.sql new file mode 100644 index 00000000..df4e2927 --- /dev/null +++ b/dbt/include/sqlserver/macros/adapter/metadata.sql @@ -0,0 +1,5 @@ +{% macro apply_label() %} + {{ log (config.get('query_tag','dbt-sqlserver'))}} + {%- set query_label = config.get('query_tag','dbt-sqlserver') -%} + OPTION (LABEL = '{{query_label}}'); +{% endmacro %} diff --git a/dbt/include/sqlserver/macros/adapters/relation.sql b/dbt/include/sqlserver/macros/adapter/relation.sql similarity index 100% rename from dbt/include/sqlserver/macros/adapters/relation.sql rename to dbt/include/sqlserver/macros/adapter/relation.sql diff --git a/dbt/include/sqlserver/macros/adapter/schemas.sql b/dbt/include/sqlserver/macros/adapter/schemas.sql new file mode 100644 index 00000000..8317d6cb --- /dev/null +++ b/dbt/include/sqlserver/macros/adapter/schemas.sql @@ -0,0 +1,5 @@ + +{% macro sqlserver__drop_schema_named(schema_name) %} + {% set schema_relation = api.Relation.create(schema=schema_name) %} + {{ adapter.drop_schema(schema_relation) }} +{% endmacro %} diff --git a/dbt/include/sqlserver/macros/adapter/validate_sql.sql b/dbt/include/sqlserver/macros/adapter/validate_sql.sql new file mode 100644 index 00000000..aa73794b --- /dev/null +++ b/dbt/include/sqlserver/macros/adapter/validate_sql.sql @@ -0,0 +1,6 @@ +{% macro sqlserver__validate_sql(sql) -%} + {% call statement('validate_sql') -%} + {{ sql }} + {% endcall %} + {{ return(load_result('validate_sql')) }} +{% endmacro %} diff --git a/dbt/include/sqlserver/macros/adapters/columns.sql b/dbt/include/sqlserver/macros/adapters/columns.sql deleted file mode 100644 index bce8e5ee..00000000 --- a/dbt/include/sqlserver/macros/adapters/columns.sql +++ /dev/null @@ -1,18 +0,0 @@ -{% macro sqlserver__alter_column_type(relation, column_name, new_column_type) %} - - {%- set tmp_column = column_name + "__dbt_alter" -%} - - {% call statement('alter_column_type') -%} - alter {{ relation.type }} {{ relation }} add "{{ tmp_column }}" {{ new_column_type }}; - {%- endcall -%} - {% call statement('alter_column_type') -%} - update {{ relation }} set "{{ tmp_column }}" = "{{ column_name }}"; - {%- endcall -%} - {% call statement('alter_column_type') -%} - alter {{ relation.type }} {{ relation }} drop column "{{ column_name }}"; - {%- endcall -%} - {% call statement('alter_column_type') -%} - exec sp_rename '{{ relation | replace('"', '') }}.{{ tmp_column }}', '{{ column_name }}', 'column' - {%- endcall -%} - -{% endmacro %} diff --git a/dbt/include/sqlserver/macros/adapters/indexes.sql b/dbt/include/sqlserver/macros/adapters/indexes.sql deleted file mode 100644 index bba787c8..00000000 --- a/dbt/include/sqlserver/macros/adapters/indexes.sql +++ /dev/null @@ -1,185 +0,0 @@ -{% macro sqlserver__create_clustered_columnstore_index(relation) -%} - {%- set cci_name = (relation.schema ~ '_' ~ relation.identifier ~ '_cci') | replace(".", "") | replace(" ", "") -%} - {%- set relation_name = relation.schema ~ '_' ~ relation.identifier -%} - {%- set full_relation = '"' ~ relation.schema ~ '"."' ~ relation.identifier ~ '"' -%} - use [{{ relation.database }}]; - if EXISTS ( - SELECT * - FROM sys.indexes {{ information_schema_hints() }} - WHERE name = '{{cci_name}}' - AND object_id=object_id('{{relation_name}}') - ) - DROP index {{full_relation}}.{{cci_name}} - CREATE CLUSTERED COLUMNSTORE INDEX {{cci_name}} - ON {{full_relation}} -{% endmacro %} - - -{# TODO - move to dbt-postgres's index implementation strategy - https://github.com/dbt-msft/dbt-sqlserver/issues/163 - #} - -{# most of this code is from https://github.com/jacobm001/dbt-mssql/blob/master/dbt/include/mssql/macros/indexes.sql #} - - -{% macro drop_xml_indexes() -%} -{# Altered from https://stackoverflow.com/q/1344401/10415173 #} -{# and https://stackoverflow.com/a/33785833/10415173 #} - - -{{ log("Running drop_xml_indexes() macro...") }} - -declare @drop_xml_indexes nvarchar(max); -select @drop_xml_indexes = ( - select 'IF INDEXPROPERTY(' + CONVERT(VARCHAR(MAX), sys.tables.[object_id]) + ', ''' + sys.indexes.[name] + ''', ''IndexId'') IS NOT NULL DROP INDEX [' + sys.indexes.[name] + '] ON ' + '[' + SCHEMA_NAME(sys.tables.[schema_id]) + '].[' + OBJECT_NAME(sys.tables.[object_id]) + ']; ' - from sys.indexes {{ information_schema_hints() }} - inner join sys.tables {{ information_schema_hints() }} - on sys.indexes.object_id = sys.tables.object_id - where sys.indexes.[name] is not null - and sys.indexes.type_desc = 'XML' - and sys.tables.[name] = '{{ this.table }}' - for xml path('') -); exec sp_executesql @drop_xml_indexes; - -{%- endmacro %} - - -{% macro drop_spatial_indexes() -%} -{# Altered from https://stackoverflow.com/q/1344401/10415173 #} -{# and https://stackoverflow.com/a/33785833/10415173 #} - -{{ log("Running drop_spatial_indexes() macro...") }} - -declare @drop_spatial_indexes nvarchar(max); -select @drop_spatial_indexes = ( - select 'IF INDEXPROPERTY(' + CONVERT(VARCHAR(MAX), sys.tables.[object_id]) + ', ''' + sys.indexes.[name] + ''', ''IndexId'') IS NOT NULL DROP INDEX [' + sys.indexes.[name] + '] ON ' + '[' + SCHEMA_NAME(sys.tables.[schema_id]) + '].[' + OBJECT_NAME(sys.tables.[object_id]) + ']; ' - from sys.indexes {{ information_schema_hints() }} - inner join sys.tables {{ information_schema_hints() }} - on sys.indexes.object_id = sys.tables.object_id - where sys.indexes.[name] is not null - and sys.indexes.type_desc = 'Spatial' - and sys.tables.[name] = '{{ this.table }}' - for xml path('') -); exec sp_executesql @drop_spatial_indexes; - -{%- endmacro %} - - -{% macro drop_fk_constraints() -%} -{# Altered from https://stackoverflow.com/q/1344401/10415173 #} - -{{ log("Running drop_fk_constraints() macro...") }} - -declare @drop_fk_constraints nvarchar(max); -select @drop_fk_constraints = ( - select 'IF OBJECT_ID(''' + SCHEMA_NAME(CONVERT(VARCHAR(MAX), sys.foreign_keys.[schema_id])) + '.' + sys.foreign_keys.[name] + ''', ''F'') IS NOT NULL ALTER TABLE [' + SCHEMA_NAME(sys.foreign_keys.[schema_id]) + '].[' + OBJECT_NAME(sys.foreign_keys.[parent_object_id]) + '] DROP CONSTRAINT [' + sys.foreign_keys.[name]+ '];' - from sys.foreign_keys - inner join sys.tables on sys.foreign_keys.[referenced_object_id] = sys.tables.[object_id] - where sys.tables.[name] = '{{ this.table }}' - for xml path('') -); exec sp_executesql @drop_fk_constraints; - -{%- endmacro %} - - -{% macro drop_pk_constraints() -%} -{# Altered from https://stackoverflow.com/q/1344401/10415173 #} -{# and https://stackoverflow.com/a/33785833/10415173 #} - -{{ drop_xml_indexes() }} - -{{ drop_spatial_indexes() }} - -{{ drop_fk_constraints() }} - -{{ log("Running drop_pk_constraints() macro...") }} - -declare @drop_pk_constraints nvarchar(max); -select @drop_pk_constraints = ( - select 'IF INDEXPROPERTY(' + CONVERT(VARCHAR(MAX), sys.tables.[object_id]) + ', ''' + sys.indexes.[name] + ''', ''IndexId'') IS NOT NULL ALTER TABLE [' + SCHEMA_NAME(sys.tables.[schema_id]) + '].[' + sys.tables.[name] + '] DROP CONSTRAINT [' + sys.indexes.[name]+ '];' - from sys.indexes - inner join sys.tables on sys.indexes.[object_id] = sys.tables.[object_id] - where sys.indexes.is_primary_key = 1 - and sys.tables.[name] = '{{ this.table }}' - for xml path('') -); exec sp_executesql @drop_pk_constraints; - -{%- endmacro %} - - -{% macro drop_all_indexes_on_table() -%} -{# Altered from https://stackoverflow.com/q/1344401/10415173 #} -{# and https://stackoverflow.com/a/33785833/10415173 #} - -{{ drop_pk_constraints() }} - -{{ log("Dropping remaining indexes...") }} - -declare @drop_remaining_indexes_last nvarchar(max); -select @drop_remaining_indexes_last = ( - select 'IF INDEXPROPERTY(' + CONVERT(VARCHAR(MAX), sys.tables.[object_id]) + ', ''' + sys.indexes.[name] + ''', ''IndexId'') IS NOT NULL DROP INDEX [' + sys.indexes.[name] + '] ON ' + '[' + SCHEMA_NAME(sys.tables.[schema_id]) + '].[' + OBJECT_NAME(sys.tables.[object_id]) + ']; ' - from sys.indexes {{ information_schema_hints() }} - inner join sys.tables {{ information_schema_hints() }} - on sys.indexes.object_id = sys.tables.object_id - where sys.indexes.[name] is not null - and sys.tables.[name] = '{{ this.table }}' - for xml path('') -); exec sp_executesql @drop_remaining_indexes_last; - -{%- endmacro %} - - -{% macro create_clustered_index(columns, unique=False) -%} - -{{ log("Creating clustered index...") }} - -{% set idx_name = "clustered_" + local_md5(columns | join("_")) %} - -if not exists(select * - from sys.indexes {{ information_schema_hints() }} - where name = '{{ idx_name }}' - and object_id = OBJECT_ID('{{ this }}') -) -begin - -create -{% if unique -%} -unique -{% endif %} -clustered index - {{ idx_name }} - on {{ this }} ({{ '[' + columns|join("], [") + ']' }}) -end -{%- endmacro %} - - -{% macro create_nonclustered_index(columns, includes=False) %} - -{{ log("Creating nonclustered index...") }} - -{% if includes -%} - {% set idx_name = ( - "nonclustered_" - + local_md5(columns | join("_")) - + "_incl_" - + local_md5(includes | join("_")) - ) %} -{% else -%} - {% set idx_name = "nonclustered_" + local_md5(columns | join("_")) %} -{% endif %} - -if not exists(select * - from sys.indexes {{ information_schema_hints() }} - where name = '{{ idx_name }}' - and object_id = OBJECT_ID('{{ this }}') -) -begin -create nonclustered index - {{ idx_name }} - on {{ this }} ({{ '[' + columns|join("], [") + ']' }}) - {% if includes -%} - include ({{ '[' + includes|join("], [") + ']' }}) - {% endif %} -end -{% endmacro %} diff --git a/dbt/include/sqlserver/macros/materializations/models/incremental/incremental.sql b/dbt/include/sqlserver/macros/materializations/models/incremental/incremental.sql new file mode 100644 index 00000000..cb19b344 --- /dev/null +++ b/dbt/include/sqlserver/macros/materializations/models/incremental/incremental.sql @@ -0,0 +1,96 @@ +{% materialization incremental, adapter='sqlserver' -%} + + -- relations + {%- set existing_relation = load_cached_relation(this) -%} + {%- set target_relation = this.incorporate(type='table') -%} + {%- set temp_relation = make_temp_relation(target_relation)-%} + {%- set intermediate_relation = make_intermediate_relation(target_relation)-%} + {%- set backup_relation_type = 'table' if existing_relation is none else existing_relation.type -%} + {%- set backup_relation = make_backup_relation(target_relation, backup_relation_type) -%} + + -- configs + {%- set unique_key = config.get('unique_key') -%} + {%- set full_refresh_mode = (should_full_refresh() or existing_relation.is_view) -%} + {%- set on_schema_change = incremental_validate_on_schema_change(config.get('on_schema_change'), default='ignore') -%} + + -- the temp_ and backup_ relations should not already exist in the database; get_relation + -- will return None in that case. Otherwise, we get a relation that we can drop + -- later, before we try to use this name for the current operation. This has to happen before + -- BEGIN, in a separate transaction + {%- set preexisting_intermediate_relation = load_cached_relation(intermediate_relation)-%} + {%- set preexisting_backup_relation = load_cached_relation(backup_relation) -%} + -- grab current tables grants config for comparision later on + {% set grant_config = config.get('grants') %} + {{ drop_relation_if_exists(preexisting_intermediate_relation) }} + {{ drop_relation_if_exists(preexisting_backup_relation) }} + + {{ run_hooks(pre_hooks, inside_transaction=False) }} + + -- `BEGIN` happens here: + {{ run_hooks(pre_hooks, inside_transaction=True) }} + + {% set to_drop = [] %} + + {% if existing_relation is none %} + {% set build_sql = get_create_table_as_sql(False, target_relation, sql) %} + {% elif full_refresh_mode %} + {% set build_sql = get_create_table_as_sql(False, intermediate_relation, sql) %} + {% set need_swap = true %} + {% else %} + + {% do run_query(get_create_table_as_sql(True, temp_relation, sql)) %} + + {% set contract_config = config.get('contract') %} + {% if not contract_config or not contract_config.enforced %} + {% do adapter.expand_target_column_types( + from_relation=temp_relation, + to_relation=target_relation) %} + {% endif %} + {#-- Process schema changes. Returns dict of changes if successful. Use source columns for upserting/merging --#} + {% set dest_columns = process_schema_changes(on_schema_change, temp_relation, existing_relation) %} + {% if not dest_columns %} + {% set dest_columns = adapter.get_columns_in_relation(existing_relation) %} + {% endif %} + + {#-- Get the incremental_strategy, the macro to use for the strategy, and build the sql --#} + {% set incremental_strategy = config.get('incremental_strategy') or 'default' %} + {% set incremental_predicates = config.get('predicates', none) or config.get('incremental_predicates', none) %} + {% set strategy_sql_macro_func = adapter.get_incremental_strategy_macro(context, incremental_strategy) %} + {% set strategy_arg_dict = ({'target_relation': target_relation, 'temp_relation': temp_relation, 'unique_key': unique_key, 'dest_columns': dest_columns, 'incremental_predicates': incremental_predicates }) %} + {% set build_sql = strategy_sql_macro_func(strategy_arg_dict) %} + + {% endif %} + + {% call statement("main") %} + {{ build_sql }} + {% endcall %} + + {% if need_swap %} + {% do adapter.rename_relation(target_relation, backup_relation) %} + {% do adapter.rename_relation(intermediate_relation, target_relation) %} + {% do to_drop.append(backup_relation) %} + {% endif %} + + {% set should_revoke = should_revoke(existing_relation, full_refresh_mode) %} + {% do apply_grants(target_relation, grant_config, should_revoke=should_revoke) %} + + {% do persist_docs(target_relation, model) %} + + {% if existing_relation is none or existing_relation.is_view or should_full_refresh() %} + {% do create_indexes(target_relation) %} + {% endif %} + + {{ run_hooks(post_hooks, inside_transaction=True) }} + + -- `COMMIT` happens here + {% do adapter.commit() %} + + {% for rel in to_drop %} + {% do adapter.drop_relation(rel) %} + {% endfor %} + + {{ run_hooks(post_hooks, inside_transaction=False) }} + + {{ return({'relations': [target_relation]}) }} + +{%- endmaterialization %} diff --git a/dbt/include/sqlserver/macros/materializations/models/table/create_table_as.sql b/dbt/include/sqlserver/macros/materializations/models/table/create_table_as.sql deleted file mode 100644 index adb420ba..00000000 --- a/dbt/include/sqlserver/macros/materializations/models/table/create_table_as.sql +++ /dev/null @@ -1,37 +0,0 @@ -{# -Fabric uses the 'CREATE TABLE XYZ AS SELECT * FROM ABC' syntax to create tables. -SQL Server doesnt support this, so we use the 'SELECT * INTO XYZ FROM ABC' syntax instead. -#} - -{% macro sqlserver__create_table_as(temporary, relation, sql) -%} - - {% set tmp_relation = relation.incorporate( - path={"identifier": relation.identifier.replace("#", "") ~ '_temp_view'}, - type='view')-%} - {% do run_query(fabric__drop_relation_script(tmp_relation)) %} - - {% set contract_config = config.get('contract') %} - - {{ fabric__create_view_as(tmp_relation, sql) }} - {% if contract_config.enforced %} - - CREATE TABLE [{{relation.database}}].[{{relation.schema}}].[{{relation.identifier}}] - {{ fabric__table_columns_and_constraints(relation) }} - {{ get_assert_columns_equivalent(sql) }} - - {% set listColumns %} - {% for column in model['columns'] %} - {{ "["~column~"]" }}{{ ", " if not loop.last }} - {% endfor %} - {%endset%} - - INSERT INTO [{{relation.database}}].[{{relation.schema}}].[{{relation.identifier}}] - ({{listColumns}}) SELECT {{listColumns}} FROM [{{tmp_relation.database}}].[{{tmp_relation.schema}}].[{{tmp_relation.identifier}}]; - - {%- else %} - EXEC('SELECT * INTO [{{relation.database}}].[{{relation.schema}}].[{{relation.identifier}}] FROM [{{tmp_relation.database}}].[{{tmp_relation.schema}}].[{{tmp_relation.identifier}}];'); - {% endif %} - - {{ fabric__drop_relation_script(tmp_relation) }} - -{% endmacro %} diff --git a/dbt/include/sqlserver/macros/materializations/models/table/table.sql b/dbt/include/sqlserver/macros/materializations/models/table/table.sql new file mode 100644 index 00000000..c347701b --- /dev/null +++ b/dbt/include/sqlserver/macros/materializations/models/table/table.sql @@ -0,0 +1,64 @@ +{% materialization table, adapter='sqlserver' %} + + {%- set existing_relation = load_cached_relation(this) -%} + {%- set target_relation = this.incorporate(type='table') %} + {%- set intermediate_relation = make_intermediate_relation(target_relation) -%} + -- the intermediate_relation should not already exist in the database; get_relation + -- will return None in that case. Otherwise, we get a relation that we can drop + -- later, before we try to use this name for the current operation + {%- set preexisting_intermediate_relation = load_cached_relation(intermediate_relation) -%} + /* + See ../view/view.sql for more information about this relation. + */ + {%- set backup_relation_type = 'table' if existing_relation is none else existing_relation.type -%} + {%- set backup_relation = make_backup_relation(target_relation, backup_relation_type) -%} + -- as above, the backup_relation should not already exist + {%- set preexisting_backup_relation = load_cached_relation(backup_relation) -%} + -- grab current tables grants config for comparision later on + {% set grant_config = config.get('grants') %} + + -- drop the temp relations if they exist already in the database + {{ drop_relation_if_exists(preexisting_intermediate_relation) }} + {{ drop_relation_if_exists(preexisting_backup_relation) }} + + {{ run_hooks(pre_hooks, inside_transaction=False) }} + + -- `BEGIN` happens here: + {{ run_hooks(pre_hooks, inside_transaction=True) }} + + -- build model + {% call statement('main') -%} + {{ get_create_table_as_sql(False, intermediate_relation, sql) }} + {%- endcall %} + + -- cleanup + {% if existing_relation is not none %} + /* Do the equivalent of rename_if_exists. 'existing_relation' could have been dropped + since the variable was first set. */ + {% set existing_relation = load_cached_relation(existing_relation) %} + {% if existing_relation is not none %} + {{ adapter.rename_relation(existing_relation, backup_relation) }} + {% endif %} + {% endif %} + + {{ adapter.rename_relation(intermediate_relation, target_relation) }} + + {% do create_indexes(target_relation) %} + + {{ run_hooks(post_hooks, inside_transaction=True) }} + + {% set should_revoke = should_revoke(existing_relation, full_refresh_mode=True) %} + {% do apply_grants(target_relation, grant_config, should_revoke=should_revoke) %} + + {% do persist_docs(target_relation, model) %} + + -- `COMMIT` happens here + {{ adapter.commit() }} + + -- finally, drop the existing/backup relation after the commit + {{ drop_relation_if_exists(backup_relation) }} + + {{ run_hooks(post_hooks, inside_transaction=False) }} + + {{ return({'relations': [target_relation]}) }} +{% endmaterialization %} diff --git a/dbt/include/sqlserver/macros/materializations/models/view/view.sql b/dbt/include/sqlserver/macros/materializations/models/view/view.sql new file mode 100644 index 00000000..4ae35c9e --- /dev/null +++ b/dbt/include/sqlserver/macros/materializations/models/view/view.sql @@ -0,0 +1,71 @@ +{%- materialization view, adapter='sqlserver' -%} + {%- set existing_relation = load_cached_relation(this) -%} + {%- set target_relation = this.incorporate(type='view') -%} + {%- set intermediate_relation = make_intermediate_relation(target_relation) -%} + + -- the intermediate_relation should not already exist in the database; get_relation + -- will return None in that case. Otherwise, we get a relation that we can drop + -- later, before we try to use this name for the current operation + {%- set preexisting_intermediate_relation = load_cached_relation(intermediate_relation) -%} + /* + This relation (probably) doesn't exist yet. If it does exist, it's a leftover from + a previous run, and we're going to try to drop it immediately. At the end of this + materialization, we're going to rename the "existing_relation" to this identifier, + and then we're going to drop it. In order to make sure we run the correct one of: + - drop view ... + - drop table ... + + We need to set the type of this relation to be the type of the existing_relation, if it exists, + or else "view" as a sane default if it does not. Note that if the existing_relation does not + exist, then there is nothing to move out of the way and subsequentally drop. In that case, + this relation will be effectively unused. + */ + {%- set backup_relation_type = 'view' if existing_relation is none else existing_relation.type -%} + {%- set backup_relation = make_backup_relation(target_relation, backup_relation_type) -%} + -- as above, the backup_relation should not already exist + {%- set preexisting_backup_relation = load_cached_relation(backup_relation) -%} + -- grab current tables grants config for comparision later on + {% set grant_config = config.get('grants') %} + + {{ run_hooks(pre_hooks, inside_transaction=False) }} + + -- drop the temp relations if they exist already in the database + {{ drop_relation_if_exists(preexisting_intermediate_relation) }} + {{ drop_relation_if_exists(preexisting_backup_relation) }} + + -- `BEGIN` happens here: + {{ run_hooks(pre_hooks, inside_transaction=True) }} + + -- build model + {% call statement('main') -%} + {{ get_create_view_as_sql(intermediate_relation, sql) }} + {%- endcall %} + + -- cleanup + -- move the existing view out of the way + {% if existing_relation is not none %} + /* Do the equivalent of rename_if_exists. 'existing_relation' could have been dropped + since the variable was first set. */ + {% set existing_relation = load_cached_relation(existing_relation) %} + {% if existing_relation is not none %} + {{ adapter.rename_relation(existing_relation, backup_relation) }} + {% endif %} + {% endif %} + {{ adapter.rename_relation(intermediate_relation, target_relation) }} + + {% set should_revoke = should_revoke(existing_relation, full_refresh_mode=True) %} + {% do apply_grants(target_relation, grant_config, should_revoke=should_revoke) %} + + {% do persist_docs(target_relation, model) %} + + {{ run_hooks(post_hooks, inside_transaction=True) }} + + {{ adapter.commit() }} + + {{ drop_relation_if_exists(backup_relation) }} + + {{ run_hooks(post_hooks, inside_transaction=False) }} + + {{ return({'relations': [target_relation]}) }} + +{%- endmaterialization -%} diff --git a/dbt/include/sqlserver/macros/materializations/seeds/helpers.sql b/dbt/include/sqlserver/macros/materializations/seeds/helpers.sql deleted file mode 100644 index 34c8e726..00000000 --- a/dbt/include/sqlserver/macros/materializations/seeds/helpers.sql +++ /dev/null @@ -1,57 +0,0 @@ -{% macro sqlserver__get_binding_char() %} - {{ return('?') }} -{% endmacro %} - -{% macro sqlserver__get_batch_size() %} - {{ return(400) }} -{% endmacro %} - -{% macro calc_batch_size(num_columns) %} - {# - SQL Server allows for a max of 2098 parameters in a single statement. - Check if the max_batch_size fits with the number of columns, otherwise - reduce the batch size so it fits. - #} - {% set max_batch_size = get_batch_size() %} - {% set calculated_batch = (2098 / num_columns)|int %} - {% set batch_size = [max_batch_size, calculated_batch] | min %} - - {{ return(batch_size) }} -{% endmacro %} - -{% macro sqlserver__load_csv_rows(model, agate_table) %} - {% set cols_sql = get_seed_column_quoted_csv(model, agate_table.column_names) %} - {% set batch_size = calc_batch_size(agate_table.column_names|length) %} - {% set bindings = [] %} - {% set statements = [] %} - - {{ log("Inserting batches of " ~ batch_size ~ " records") }} - - {% for chunk in agate_table.rows | batch(batch_size) %} - {% set bindings = [] %} - - {% for row in chunk %} - {% do bindings.extend(row) %} - {% endfor %} - - {% set sql %} - insert into {{ this.render() }} ({{ cols_sql }}) values - {% for row in chunk -%} - ({%- for column in agate_table.column_names -%} - {{ get_binding_char() }} - {%- if not loop.last%},{%- endif %} - {%- endfor -%}) - {%- if not loop.last%},{%- endif %} - {%- endfor %} - {% endset %} - - {% do adapter.add_query(sql, bindings=bindings, abridge_sql_log=True) %} - - {% if loop.index0 == 0 %} - {% do statements.append(sql) %} - {% endif %} - {% endfor %} - - {# Return SQL so we can render it out into the compiled files #} - {{ return(statements[0]) }} -{% endmacro %} diff --git a/dbt/include/sqlserver/macros/materializations/snapshot/helpers.sql b/dbt/include/sqlserver/macros/materializations/snapshot/helpers.sql new file mode 100644 index 00000000..3317a9f3 --- /dev/null +++ b/dbt/include/sqlserver/macros/materializations/snapshot/helpers.sql @@ -0,0 +1,35 @@ +{% macro sqlserver__create_columns(relation, columns) %} + {% set column_list %} + {% for column_entry in columns %} + {{column_entry.name}} {{column_entry.data_type}}{{ ", " if not loop.last }} + {% endfor %} + {% endset %} + + {% set alter_sql %} + ALTER TABLE {{ relation }} + ADD {{ column_list }} + {% endset %} + + {% set results = run_query(alter_sql) %} + +{% endmacro %} + +{% macro build_snapshot_staging_table(strategy, temp_snapshot_relation, target_relation) %} + {% set temp_relation = make_temp_relation(target_relation) %} + {{ adapter.drop_relation(temp_relation) }} + + {% set select = snapshot_staging_table(strategy, temp_snapshot_relation, target_relation) %} + + {% set tmp_tble_vw_relation = temp_relation.incorporate(path={"identifier": temp_relation.identifier ~ '__dbt_tmp_vw'}, type='view')-%} + -- Dropping temp view relation if it exists + {{ adapter.drop_relation(tmp_tble_vw_relation) }} + + {% call statement('build_snapshot_staging_relation') %} + {{ get_create_table_as_sql(True, temp_relation, select) }} + {% endcall %} + + -- Dropping temp view relation if it exists + {{ adapter.drop_relation(tmp_tble_vw_relation) }} + + {% do return(temp_relation) %} +{% endmacro %} diff --git a/dbt/include/sqlserver/macros/materializations/snapshot/snapshot.sql b/dbt/include/sqlserver/macros/materializations/snapshot/snapshot.sql new file mode 100644 index 00000000..aca7d397 --- /dev/null +++ b/dbt/include/sqlserver/macros/materializations/snapshot/snapshot.sql @@ -0,0 +1,116 @@ +{% materialization snapshot, adapter='sqlserver' %} + {%- set config = model['config'] -%} + + {%- set target_table = model.get('alias', model.get('name')) -%} + + {%- set strategy_name = config.get('strategy') -%} + {%- set unique_key = config.get('unique_key') %} + -- grab current tables grants config for comparision later on + {%- set grant_config = config.get('grants') -%} + + {% set target_relation_exists, target_relation = get_or_create_relation( + database=model.database, + schema=model.schema, + identifier=target_table, + type='table') -%} + + {%- if not target_relation.is_table -%} + {% do exceptions.relation_wrong_type(target_relation, 'table') %} + {%- endif -%} + + + {{ run_hooks(pre_hooks, inside_transaction=False) }} + + {{ run_hooks(pre_hooks, inside_transaction=True) }} + + {% set strategy_macro = strategy_dispatch(strategy_name) %} + {% set strategy = strategy_macro(model, "snapshotted_data", "source_data", config, target_relation_exists) %} + + {% set temp_snapshot_relation_exists, temp_snapshot_relation = get_or_create_relation( + database=model.database, + schema=model.schema, + identifier=target_table+"_snapshot_staging_temp_view", + type='view') + -%} + + {% set temp_snapshot_relation_sql = model['compiled_code'].replace("'", "''") %} + {% call statement('create temp_snapshot_relation') %} + EXEC('DROP VIEW IF EXISTS {{ temp_snapshot_relation.include(database=False) }};'); + EXEC('create view {{ temp_snapshot_relation.include(database=False) }} as {{ temp_snapshot_relation_sql }};'); + {% endcall %} + + {% if not target_relation_exists %} + + {% set build_sql = build_snapshot_table(strategy, temp_snapshot_relation) %} + {% set final_sql = create_table_as(False, target_relation, build_sql) %} + + {% else %} + + {{ adapter.valid_snapshot_target(target_relation) }} + + {% set staging_table = build_snapshot_staging_table(strategy, temp_snapshot_relation, target_relation) %} + + -- this may no-op if the database does not require column expansion + {% do adapter.expand_target_column_types(from_relation=staging_table, + to_relation=target_relation) %} + + {% set missing_columns = adapter.get_missing_columns(staging_table, target_relation) + | rejectattr('name', 'equalto', 'dbt_change_type') + | rejectattr('name', 'equalto', 'DBT_CHANGE_TYPE') + | rejectattr('name', 'equalto', 'dbt_unique_key') + | rejectattr('name', 'equalto', 'DBT_UNIQUE_KEY') + | list %} + {% if missing_columns|length > 0 %} + {{log("Missing columns length is: "~ missing_columns|length)}} + {% do create_columns(target_relation, missing_columns) %} + {% endif %} + + {% set source_columns = adapter.get_columns_in_relation(staging_table) + | rejectattr('name', 'equalto', 'dbt_change_type') + | rejectattr('name', 'equalto', 'DBT_CHANGE_TYPE') + | rejectattr('name', 'equalto', 'dbt_unique_key') + | rejectattr('name', 'equalto', 'DBT_UNIQUE_KEY') + | list %} + + {% set quoted_source_columns = [] %} + {% for column in source_columns %} + {% do quoted_source_columns.append(adapter.quote(column.name)) %} + {% endfor %} + + {% set final_sql = snapshot_merge_sql( + target = target_relation, + source = staging_table, + insert_cols = quoted_source_columns + ) + %} + + {% endif %} + + {% call statement('main') %} + {{ final_sql }} + {% endcall %} + + {{ adapter.drop_relation(temp_snapshot_relation) }} + + {% set should_revoke = should_revoke(target_relation_exists, full_refresh_mode=False) %} + {% do apply_grants(target_relation, grant_config, should_revoke=should_revoke) %} + + {% do persist_docs(target_relation, model) %} + + {% if not target_relation_exists %} + {% do create_indexes(target_relation) %} + {% endif %} + + {{ run_hooks(post_hooks, inside_transaction=True) }} + + {{ adapter.commit() }} + + {% if staging_table is defined %} + {% do post_snapshot(staging_table) %} + {% endif %} + + {{ run_hooks(post_hooks, inside_transaction=False) }} + + {{ return({'relations': [target_relation]}) }} + +{% endmaterialization %} diff --git a/dbt/include/sqlserver/macros/materializations/snapshot/snapshot_merge.sql b/dbt/include/sqlserver/macros/materializations/snapshot/snapshot_merge.sql new file mode 100644 index 00000000..caae7740 --- /dev/null +++ b/dbt/include/sqlserver/macros/materializations/snapshot/snapshot_merge.sql @@ -0,0 +1,19 @@ +{% macro sqlserver__snapshot_merge_sql(target, source, insert_cols) -%} + {%- set insert_cols_csv = insert_cols | join(', ') -%} + + merge into {{ target.render() }} as DBT_INTERNAL_DEST + using {{ source }} as DBT_INTERNAL_SOURCE + on DBT_INTERNAL_SOURCE.dbt_scd_id = DBT_INTERNAL_DEST.dbt_scd_id + + when matched + and DBT_INTERNAL_DEST.dbt_valid_to is null + and DBT_INTERNAL_SOURCE.dbt_change_type in ('update', 'delete') + then update + set dbt_valid_to = DBT_INTERNAL_SOURCE.dbt_valid_to + + when not matched + and DBT_INTERNAL_SOURCE.dbt_change_type = 'insert' + then insert ({{ insert_cols_csv }}) + values ({{ insert_cols_csv }}) + ; +{% endmacro %} diff --git a/dbt/include/sqlserver/macros/materializations/snapshots/snapshot.sql b/dbt/include/sqlserver/macros/materializations/snapshots/snapshot.sql deleted file mode 100644 index 8d646c60..00000000 --- a/dbt/include/sqlserver/macros/materializations/snapshots/snapshot.sql +++ /dev/null @@ -1,52 +0,0 @@ -{# -Fabric uses the 'CREATE TABLE XYZ AS SELECT * FROM ABC' syntax to create tables. -SQL Server doesnt support this, so we use the 'SELECT * INTO XYZ FROM ABC' syntax instead. -#} - -{% macro sqlserver__create_columns(relation, columns) %} - {# default__ macro uses "add column" - TSQL preferes just "add" - #} - - {% set columns %} - {% for column in columns %} - , CAST(NULL AS {{column.data_type}}) AS {{ column.quoted }} - {% endfor %} - {% endset %} - - {% set tempTableName %} - [{{relation.database}}].[{{ relation.schema }}].[{{ relation.identifier }}_{{ range(1300, 19000) | random }}] - {% endset %} - - {% set tempTable %} - SELECT * {{columns}} INTO {{tempTableName}} FROM [{{relation.database}}].[{{ relation.schema }}].[{{ relation.identifier }}] {{ information_schema_hints() }} - {% endset %} - - {% call statement('create_temp_table') -%} - {{ tempTable }} - {%- endcall %} - - {% set dropTable %} - DROP TABLE [{{relation.database}}].[{{ relation.schema }}].[{{ relation.identifier }}] - {% endset %} - - {% call statement('drop_table') -%} - {{ dropTable }} - {%- endcall %} - - {% set createTable %} - SELECT * INTO {{ relation }} FROM {{tempTableName}} {{ information_schema_hints() }} - {% endset %} - - {% call statement('create_Table') -%} - {{ createTable }} - {%- endcall %} - - {% set dropTempTable %} - DROP TABLE {{tempTableName}} - {% endset %} - - {% call statement('drop_temp_table') -%} - {{ dropTempTable }} - {%- endcall %} -{% endmacro %} diff --git a/dbt/include/sqlserver/macros/readme.md b/dbt/include/sqlserver/macros/readme.md new file mode 100644 index 00000000..6ba572ab --- /dev/null +++ b/dbt/include/sqlserver/macros/readme.md @@ -0,0 +1,50 @@ +# Alterations from Fabric + +## `materialization incremental` + +This is reset to the original logic from the global project. + +## `materialization view` + +This is reset to the original logic from the global project + +## `materialization table` + +This is resets to the original logic from the global project + +## `sqlserver__create_columns` + +SQLServer supports ALTER; this updates the logic to apply alter instead of the drop/recreate + +## `sqlserver__alter_column_type` + +SQLServer supports ALTER; this updates the logic to apply alter instead of the drop/recreate + + +## `sqlserver__can_clone_table` + +SQLServer cannot clone, so this just returns False + +## `sqlserver__create_table_as` + +Logic is slightly re-written from original. +There is an underlying issue with the structure in that its embedding in EXEC calls. + +This creates an issue where temporary tables cannot be used, as they dont exist within the context of the EXEC call. + +One work around might be to issue the create table from a `{{ run_query }}` statement in order to have it accessible outside the exec context. + +Additionally the expected {% do adapter.drop_relation(tmp_relation) %} does not fire. Possible cache issue? +Resolved by calling `DROP VIEW IF EXISTS` on the relation + +## `sqlserver__create_view_as` + +Updated to remove `create_view_as_exec` call. + +## `listagg` + +DBT expects a limit function, but the sqlserver syntax does not support it. Fabric also does not implement this properly + +## `sqlserver__snapshot_merge_sql` + +Restores logic to the merge statement logic like the dbt core. Merge will probably be slower then the existing logic diff --git a/dbt/include/sqlserver/macros/materializations/models/table/clone.sql b/dbt/include/sqlserver/macros/relations/table/clone.sql similarity index 100% rename from dbt/include/sqlserver/macros/materializations/models/table/clone.sql rename to dbt/include/sqlserver/macros/relations/table/clone.sql diff --git a/dbt/include/sqlserver/macros/relations/table/create.sql b/dbt/include/sqlserver/macros/relations/table/create.sql new file mode 100644 index 00000000..49416ba4 --- /dev/null +++ b/dbt/include/sqlserver/macros/relations/table/create.sql @@ -0,0 +1,48 @@ +{% macro sqlserver__create_table_as(temporary, relation, sql) -%} + {%- set query_label = apply_label() -%} + {%- set tmp_relation = relation.incorporate(path={"identifier": relation.identifier ~ '__dbt_tmp_vw'}, type='view') -%} + + {%- do adapter.drop_relation(tmp_relation) -%} + {{ get_create_view_as_sql(tmp_relation, sql) }} + + {%- set table_name -%} + {{ relation.database}}.{{ relation.schema }}.{{ relation.identifier }} + {%- endset -%} + + {%- set contract_config = config.get('contract') -%} + {%- set query -%} + {% if contract_config.enforced and (not temporary) %} + CREATE TABLE {{table_name}} + {{ get_assert_columns_equivalent(sql) }} + {{ build_columns_constraints(relation) }} + {% set listColumns %} + {% for column in model['columns'] %} + {{ "["~column~"]" }}{{ ", " if not loop.last }} + {% endfor %} + {%endset%} + INSERT INTO {{relation}} ({{listColumns}}) + SELECT {{listColumns}} FROM {{tmp_relation}} {{ query_label }} + + {% else %} + SELECT * INTO {{ table_name }} FROM {{ tmp_relation }} {{ query_label }} + {% endif %} + {%- endset -%} + + EXEC('{{- escape_single_quotes(query) -}}') + + {# For some reason drop_relation is not firing. This solves the issue for now. #} + EXEC('DROP VIEW IF EXISTS {{tmp_relation.schema}}.{{tmp_relation.identifier}}') + + + + {% set as_columnstore = config.get('as_columnstore', default=true) %} + {% if not temporary and as_columnstore -%} + {#- + add columnstore index + this creates with dbt_temp as its coming from a temporary relation before renaming + could alter relation to drop the dbt_temp portion if needed + -#} + {{ sqlserver__create_clustered_columnstore_index(relation) }} + {% endif %} + +{% endmacro %} diff --git a/dbt/include/sqlserver/macros/adapters/.gitkeep b/dbt/include/sqlserver/macros/relations/views/.gitkeep similarity index 100% rename from dbt/include/sqlserver/macros/adapters/.gitkeep rename to dbt/include/sqlserver/macros/relations/views/.gitkeep diff --git a/dbt/include/sqlserver/macros/relations/views/create.sql b/dbt/include/sqlserver/macros/relations/views/create.sql new file mode 100644 index 00000000..52c4e332 --- /dev/null +++ b/dbt/include/sqlserver/macros/relations/views/create.sql @@ -0,0 +1,19 @@ +{% macro sqlserver__create_view_as(relation, sql) -%} + + {{ get_use_database_sql(relation.database) }} + {% set contract_config = config.get('contract') %} + {% if contract_config.enforced %} + {{ get_assert_columns_equivalent(sql) }} + {%- endif %} + + {% set query %} + create view {{ relation.include(database=False) }} as {{ sql }}; + {% endset %} + + {% set tst %} + SELECT '1' as col + {% endset %} + + EXEC('{{- escape_single_quotes(query) -}}') + +{% endmacro %} diff --git a/dbt/include/sqlserver/macros/utils/.gitkeep b/dbt/include/sqlserver/macros/utils/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/dbt/include/sqlserver/macros/utils/split_part.sql b/dbt/include/sqlserver/macros/utils/split_part.sql index 2e94e1fa..5d7434e3 100644 --- a/dbt/include/sqlserver/macros/utils/split_part.sql +++ b/dbt/include/sqlserver/macros/utils/split_part.sql @@ -1,5 +1,5 @@ {# - For more information on how this XML trick works with splitting strings, see https://www.mssqltips.com/sqlservertip/1771/splitting-delimited-strings-using-xml-in-sql-server/ + For more information on how this XML trick works with splitting strings, see https://www.sqlservertips.com/sqlservertip/1771/splitting-delimited-strings-using-xml-in-sql-server/ On Azure SQL and SQL Server 2019, we can use the string_split function instead of the XML trick. But since we don't know which version of SQL Server the user is using, we'll stick with the XML trick in this adapter. However, since the XML data type is not supported in Synapse, it has to be overriden in that adapter. diff --git a/dbt/include/sqlserver/macros/utils/timestamps.sql b/dbt/include/sqlserver/macros/utils/timestamps.sql deleted file mode 100644 index f10bd734..00000000 --- a/dbt/include/sqlserver/macros/utils/timestamps.sql +++ /dev/null @@ -1,8 +0,0 @@ -{% macro sqlserver__current_timestamp() -%} - SYSDATETIME() -{%- endmacro %} - -{% macro sqlserver__snapshot_string_as_time(timestamp) -%} - {%- set result = "CONVERT(DATETIME2, '" ~ timestamp ~ "')" -%} - {{ return(result) }} -{%- endmacro %} diff --git a/dbt/include/sqlserver/profile_template.yml b/dbt/include/sqlserver/profile_template.yml new file mode 100644 index 00000000..32c80711 --- /dev/null +++ b/dbt/include/sqlserver/profile_template.yml @@ -0,0 +1,19 @@ +fixed: + type: sqlserver +prompts: + host: + hint: "your host name" + port: + default: 5432 + type: "int" + user: + hint: "dev username" + password: + hint: "dev password" + hide_input: true + database: + hint: "default database" + threads: + hint: "1 or more" + type: "int" + default: 1 diff --git a/dev_requirements.txt b/dev_requirements.txt index 6bb328e1..1c77177f 100644 --- a/dev_requirements.txt +++ b/dev_requirements.txt @@ -1,10 +1,23 @@ -pytest==7.4.2 -twine==4.0.2 -wheel==0.42 -pre-commit==3.5 -pytest-dotenv==0.5.2 -dbt-tests-adapter==1.7.2 -dbt-fabric==1.7.2 -flaky==3.7.0 -pytest-xdist==3.5.0 + +dbt-tests-adapter>=1.8.0, <1.9.0 + +ruff +black==24.2.0 +bumpversion +flake8 +flaky +freezegun==1.4.0 +ipdb +mypy==1.8.0 +pip-tools +pre-commit +pytest +pytest-dotenv +pytest-logbook +pytest-csv +pytest-xdist +pytz +tox>=3.13 +twine +wheel -e . diff --git a/devops/scripts/init_db.sh b/devops/scripts/init_db.sh index 87c75dbf..32bf126f 100755 --- a/devops/scripts/init_db.sh +++ b/devops/scripts/init_db.sh @@ -1,8 +1,12 @@ #!/usr/bin/env bash +if [ -d "/opt/mssql-tools18" ]; then + cp -r /opt/mssql-tools18 /opt/mssql-tools +fi + for i in {1..50}; do - /opt/mssql-tools/bin/sqlcmd -S localhost -U sa -P "${SA_PASSWORD}" -d master -I -Q "CREATE DATABASE TestDB COLLATE ${COLLATION}" + /opt/mssql-tools/bin/sqlcmd -C -S localhost -U sa -P "${SA_PASSWORD}" -d master -I -Q "CREATE DATABASE TestDB COLLATE ${COLLATION}" if [ $? -eq 0 ] then echo "database creation completed" @@ -15,7 +19,7 @@ done for i in {1..50}; do - /opt/mssql-tools/bin/sqlcmd -S localhost -U sa -P "${SA_PASSWORD}" -d TestDB -I -i init.sql + /opt/mssql-tools/bin/sqlcmd -C -S localhost -U sa -P "${SA_PASSWORD}" -d TestDB -I -i init.sql if [ $? -eq 0 ] then echo "user creation completed" diff --git a/devops/server.Dockerfile b/devops/server.Dockerfile index 8f7c3ece..d5743136 100644 --- a/devops/server.Dockerfile +++ b/devops/server.Dockerfile @@ -1,10 +1,14 @@ -ARG MSSQL_VERSION="2022" -FROM mcr.microsoft.com/mssql/server:${MSSQL_VERSION}-latest +ARG SQLServer_VERSION="2022" +FROM mcr.microsoft.com/mssql/server:${SQLServer_VERSION}-latest ENV COLLATION="SQL_Latin1_General_CP1_CI_AS" + USER root + RUN mkdir -p /opt/init_scripts WORKDIR /opt/init_scripts COPY scripts/* /opt/init_scripts/ +RUN chmod +x /opt/init_scripts/*.sh + ENTRYPOINT /bin/bash ./entrypoint.sh diff --git a/setup.py b/setup.py index 749553ed..2872ec7c 100644 --- a/setup.py +++ b/setup.py @@ -7,8 +7,8 @@ from setuptools.command.install import install package_name = "dbt-sqlserver" -authors_list = ["Mikael Ene", "Anders Swanson", "Sam Debruyn", "Cor Zuurmond"] -dbt_version = "1.7" +authors_list = ["Mikael Ene", "Anders Swanson", "Sam Debruyn", "Cor Zuurmond", "Cody Scott"] +dbt_version = "1.8" description = """A Microsoft SQL Server adapter plugin for dbt""" this_directory = os.path.abspath(os.path.dirname(__file__)) @@ -66,10 +66,10 @@ def run(self): packages=find_namespace_packages(include=["dbt", "dbt.*"]), include_package_data=True, install_requires=[ - "dbt-core~=1.7.2", - "dbt-fabric~=1.7.2", - "pyodbc>=4.0.35,<5.1.0", - "azure-identity>=1.12.0", + "dbt-fabric>=1.8.0,<1.9.0", + "dbt-core>=1.8.0,<1.9.0", + "dbt-common>=1.0,<2.0", + "dbt-adapters>=1.1.1,<2.0", ], cmdclass={ "verify": VerifyVersionCommand, diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 00000000..b29bb009 --- /dev/null +++ b/tests/__init__.py @@ -0,0 +1,41 @@ +import pytest +from azure.identity import AzureCliCredential + +from dbt.adapters.sqlserver.sqlserver_connections import ( # byte_array_to_datetime, + bool_to_connection_string_arg, + get_pyodbc_attrs_before, +) +from dbt.adapters.sqlserver.sqlserver_credentials import SQLServerCredentials + +# See +# https://github.com/Azure/azure-sdk-for-python/blob/azure-identity_1.5.0/sdk/identity/azure-identity/tests/test_cli_credential.py +CHECK_OUTPUT = AzureCliCredential.__module__ + ".subprocess.check_output" + + +@pytest.fixture +def credentials() -> SQLServerCredentials: + credentials = SQLServerCredentials( + driver="ODBC Driver 18 for SQL Server", + host="fake.sql.sqlserver.net", + database="dbt", + schema="sqlserver", + ) + return credentials + + +def test_get_pyodbc_attrs_before_empty_dict_when_service_principal( + credentials: SQLServerCredentials, +) -> None: + """ + When the authentication is set to sql we expect an empty attrs before. + """ + attrs_before = get_pyodbc_attrs_before(credentials) + assert attrs_before == {} + + +@pytest.mark.parametrize( + "key, value, expected", + [("somekey", False, "somekey=No"), ("somekey", True, "somekey=Yes")], +) +def test_bool_to_connection_string_arg(key: str, value: bool, expected: str) -> None: + assert bool_to_connection_string_arg(key, value) == expected diff --git a/tests/functional/adapter/dbt/test_aliases.py b/tests/functional/adapter/dbt/test_aliases.py new file mode 100644 index 00000000..0ab58689 --- /dev/null +++ b/tests/functional/adapter/dbt/test_aliases.py @@ -0,0 +1,58 @@ +import pytest +from dbt.tests.adapter.aliases import fixtures +from dbt.tests.adapter.aliases.test_aliases import ( + BaseAliasErrors, + BaseAliases, + BaseSameAliasDifferentDatabases, + BaseSameAliasDifferentSchemas, +) + +# we override the default as the SQLServer adapter uses CAST instead of :: for type casting +MACROS__CAST_SQL_SQLServer = """ + + +{% macro string_literal(s) -%} + {{ adapter.dispatch('string_literal', macro_namespace='test')(s) }} +{%- endmacro %} + +{% macro default__string_literal(s) %} + CAST('{{ s }}' AS VARCHAR(8000)) +{% endmacro %} + +""" + + +class TestAliases(BaseAliases): + @pytest.fixture(scope="class") + def macros(self): + return { + "cast.sql": MACROS__CAST_SQL_SQLServer, + "expect_value.sql": fixtures.MACROS__EXPECT_VALUE_SQL, + } + + +class TestAliasesError(BaseAliasErrors): + @pytest.fixture(scope="class") + def macros(self): + return { + "cast.sql": MACROS__CAST_SQL_SQLServer, + "expect_value.sql": fixtures.MACROS__EXPECT_VALUE_SQL, + } + + +class TestSameAliasDifferentSchemas(BaseSameAliasDifferentSchemas): + @pytest.fixture(scope="class") + def macros(self): + return { + "cast.sql": MACROS__CAST_SQL_SQLServer, + "expect_value.sql": fixtures.MACROS__EXPECT_VALUE_SQL, + } + + +class TestSameAliasDifferentDatabases(BaseSameAliasDifferentDatabases): + @pytest.fixture(scope="class") + def macros(self): + return { + "cast.sql": MACROS__CAST_SQL_SQLServer, + "expect_value.sql": fixtures.MACROS__EXPECT_VALUE_SQL, + } diff --git a/tests/functional/adapter/dbt/test_basic.py b/tests/functional/adapter/dbt/test_basic.py new file mode 100644 index 00000000..bdbe38f8 --- /dev/null +++ b/tests/functional/adapter/dbt/test_basic.py @@ -0,0 +1,53 @@ +import pytest +from dbt.tests.adapter.basic.test_adapter_methods import BaseAdapterMethod +from dbt.tests.adapter.basic.test_base import BaseSimpleMaterializations +from dbt.tests.adapter.basic.test_empty import BaseEmpty +from dbt.tests.adapter.basic.test_ephemeral import BaseEphemeral +from dbt.tests.adapter.basic.test_generic_tests import BaseGenericTests +from dbt.tests.adapter.basic.test_incremental import BaseIncremental +from dbt.tests.adapter.basic.test_singular_tests import BaseSingularTests +from dbt.tests.adapter.basic.test_singular_tests_ephemeral import BaseSingularTestsEphemeral +from dbt.tests.adapter.basic.test_snapshot_check_cols import BaseSnapshotCheckCols +from dbt.tests.adapter.basic.test_snapshot_timestamp import BaseSnapshotTimestamp + + +class TestSimpleMaterializations(BaseSimpleMaterializations): + pass + + +class TestSingularTests(BaseSingularTests): + pass + + +@pytest.mark.skip(reason="SQLServer doesn't support nested CTE") +class TestSingularTestsEphemeral(BaseSingularTestsEphemeral): + pass + + +class TestEmpty(BaseEmpty): + pass + + +@pytest.mark.skip(reason="SQLServer doesn't support nested CTE") +class TestEphemeral(BaseEphemeral): + pass + + +class TestIncremental(BaseIncremental): + pass + + +class TestGenericTests(BaseGenericTests): + pass + + +class TestSnapshotCheckCols(BaseSnapshotCheckCols): + pass + + +class TestSnapshotTimestamp(BaseSnapshotTimestamp): + pass + + +class TestBaseAdapterMethod(BaseAdapterMethod): + pass diff --git a/tests/functional/adapter/dbt/test_caching.py b/tests/functional/adapter/dbt/test_caching.py new file mode 100644 index 00000000..1374ed60 --- /dev/null +++ b/tests/functional/adapter/dbt/test_caching.py @@ -0,0 +1,29 @@ +import pytest +from dbt.tests.adapter.caching.test_caching import ( + BaseCachingLowercaseModel, + BaseCachingSelectedSchemaOnly, + BaseCachingUppercaseModel, + BaseNoPopulateCache, +) + + +class TestCachingLowercaseModel(BaseCachingLowercaseModel): + pass + + +@pytest.mark.skip( + reason=""" + Fails because of case sensitivity. + MODEL is coereced to model which fails the test as it sees conflicting naming + """ +) +class TestCachingUppercaseModel(BaseCachingUppercaseModel): + pass + + +class TestCachingSelectedSchemaOnly(BaseCachingSelectedSchemaOnly): + pass + + +class TestNoPopulateCache(BaseNoPopulateCache): + pass diff --git a/tests/functional/adapter/dbt/test_catalog.py b/tests/functional/adapter/dbt/test_catalog.py new file mode 100644 index 00000000..cd3a6228 --- /dev/null +++ b/tests/functional/adapter/dbt/test_catalog.py @@ -0,0 +1,52 @@ +import pytest +from dbt.artifacts.schemas.catalog import CatalogArtifact +from dbt.tests.adapter.catalog import files +from dbt.tests.adapter.catalog.relation_types import CatalogRelationTypes +from dbt.tests.util import run_dbt + + +class TestRelationTypes(CatalogRelationTypes): + """ + This is subclassed to remove the references to the materialized views, + as SQLServer does not support them. + + Likely does not need to be subclassed since we implement everything, + but prefer keeping it here for clarity. + """ + + @pytest.fixture(scope="class", autouse=True) + def seeds(self): + return {"my_seed.csv": files.MY_SEED} + + @pytest.fixture(scope="class", autouse=True) + def models(self): + yield { + "my_table.sql": files.MY_TABLE, + "my_view.sql": files.MY_VIEW, + # "my_materialized_view.sql": files.MY_MATERIALIZED_VIEW, + } + + @pytest.fixture(scope="class", autouse=True) + def docs(self, project): + run_dbt(["seed"]) + run_dbt(["run"]) + yield run_dbt(["docs", "generate"]) + + @pytest.mark.parametrize( + "node_name,relation_type", + [ + ("seed.test.my_seed", "BASE TABLE"), + ("model.test.my_table", "BASE TABLE"), + ("model.test.my_view", "VIEW"), + # ("model.test.my_materialized_view", "MATERIALIZED VIEW"), + ], + ) + def test_relation_types_populate_correctly( + self, docs: CatalogArtifact, node_name: str, relation_type: str + ): + """ + This test addresses: https://github.com/dbt-labs/dbt-core/issues/8864 + """ + assert node_name in docs.nodes + node = docs.nodes[node_name] + assert node.metadata.type == relation_type diff --git a/tests/functional/adapter/dbt/test_column_types.py b/tests/functional/adapter/dbt/test_column_types.py new file mode 100644 index 00000000..331e69f1 --- /dev/null +++ b/tests/functional/adapter/dbt/test_column_types.py @@ -0,0 +1,122 @@ +import pytest +from dbt.tests.adapter.column_types.test_column_types import BaseColumnTypes + +# flake8: noqa: E501 + +macro_test_is_type_sql_2 = """ +{% macro simple_type_check_column(column, check) %} + {% if check == 'string' %} + {{ return(column.is_string()) }} + {% elif check == 'float' %} + {{ return(column.is_float()) }} + {% elif check == 'number' %} + {{ return(column.is_number()) }} + {% elif check == 'numeric' %} + {{ return(column.is_numeric()) }} + {% elif check == 'integer' %} + {{ return(column.is_integer()) }} + {% else %} + {% do exceptions.raise_compiler_error('invalid type check value: ' ~ check) %} + {% endif %} +{% endmacro %} + +{% macro type_check_column(column, type_checks) %} + {% set failures = [] %} + {% for type_check in type_checks %} + {% if type_check.startswith('not ') %} + {% if simple_type_check_column(column, type_check[4:]) %} + {% do log('simple_type_check_column got ', True) %} + {% do failures.append(type_check) %} + {% endif %} + {% else %} + {% if not simple_type_check_column(column, type_check) %} + {% do failures.append(type_check) %} + {% endif %} + {% endif %} + {% endfor %} + {% if (failures | length) > 0 %} + {% do log('column ' ~ column.name ~ ' had failures: ' ~ failures, info=True) %} + {% endif %} + {% do return((failures | length) == 0) %} +{% endmacro %} + +{% test is_type(model, column_map) %} + {% if not execute %} + {{ return(None) }} + {% endif %} + {% if not column_map %} + {% do exceptions.raise_compiler_error('test_is_type must have a column name') %} + {% endif %} + {% set columns = adapter.get_columns_in_relation(model) %} + {% if (column_map | length) != (columns | length) %} + {% set column_map_keys = (column_map | list | string) %} + {% set column_names = (columns | map(attribute='name') | list | string) %} + {% do exceptions.raise_compiler_error('did not get all the columns/all columns not specified:\n' ~ column_map_keys ~ '\nvs\n' ~ column_names) %} + {% endif %} + {% set bad_columns = [] %} + {% for column in columns %} + {% set column_key = (column.name | lower) %} + {% if column_key in column_map %} + {% set type_checks = column_map[column_key] %} + {% if not type_checks %} + {% do exceptions.raise_compiler_error('no type checks?') %} + {% endif %} + {% if not type_check_column(column, type_checks) %} + {% do bad_columns.append(column.name) %} + {% endif %} + {% else %} + {% do exceptions.raise_compiler_error( + 'column key ' ~ column_key ~ ' not found in ' ~ (column_map | list | string)) %} + {% endif %} + {% endfor %} + {% do log('bad columns: ' ~ bad_columns, info=True) %} + {% for bad_column in bad_columns %} + select '{{ bad_column }}' as bad_column + {{ 'union all'}} + --{{ 'union all' if not loop.last }} + {% endfor %} + select * from (select 1 as c where 1 = 0) as nothing +{% endtest %} +""" + +model_sql = """ +select + CAST(1 AS smallint) as smallint_col, + CAST(2 AS integer) as int_col, + CAST(3 AS bigint) as bigint_col, + CAST(4.0 AS real) as real_col, + CAST(5.0 AS float) as double_col, + CAST(6.0 AS numeric) as numeric_col, + CAST(7 AS varchar(20)) as text_col, + CAST(8 AS varchar(20)) as varchar_col +""" + +schema_yml = """ +version: 2 +models: + - name: model + data_tests: + - is_type: + column_map: + smallint_col: ['integer', 'number'] + int_col: ['integer', 'number'] + bigint_col: ['integer', 'number'] + real_col: ['float', 'number'] + double_col: ['float', 'number'] + numeric_col: ['numeric', 'number'] + text_col: ['string', 'not number'] + varchar_col: ['string', 'not number'] +""" # noqa + + +class TestColumnTypes(BaseColumnTypes): + @pytest.fixture(scope="class") + def macros(self): + return {"test_is_type.sql": macro_test_is_type_sql_2} + + @pytest.fixture(scope="class") + def models(self): + return {"model.sql": model_sql, "schema.yml": schema_yml} + + def test_run_and_test(self, project): + self.run_and_test() diff --git a/tests/functional/adapter/dbt/test_concurrency.py b/tests/functional/adapter/dbt/test_concurrency.py new file mode 100644 index 00000000..f7e4b0e4 --- /dev/null +++ b/tests/functional/adapter/dbt/test_concurrency.py @@ -0,0 +1,5 @@ +from dbt.tests.adapter.concurrency.test_concurrency import BaseConcurrency + + +class TestConcurrency(BaseConcurrency): + pass diff --git a/tests/functional/adapter/dbt/test_constraints.py b/tests/functional/adapter/dbt/test_constraints.py new file mode 100644 index 00000000..d2ec1080 --- /dev/null +++ b/tests/functional/adapter/dbt/test_constraints.py @@ -0,0 +1,751 @@ +import re + +import pytest +from dbt.tests.adapter.constraints.fixtures import ( + model_data_type_schema_yml, + my_incremental_model_sql, + my_model_data_type_sql, + my_model_incremental_with_nulls_sql, + my_model_incremental_wrong_name_sql, + my_model_incremental_wrong_order_sql, + my_model_sql, + my_model_view_wrong_name_sql, + my_model_view_wrong_order_sql, + my_model_with_nulls_sql, + my_model_wrong_name_sql, + my_model_wrong_order_sql, +) +from dbt.tests.util import ( + get_manifest, + read_file, + relation_from_name, + run_dbt, + run_dbt_and_capture, + write_file, +) + +# flake8: noqa: E501 + + +model_schema_yml = """ +version: 2 +models: + - name: my_model + config: + contract: + enforced: true + constraints: + - type: primary_key + columns: [id] + name: pk_my_model_pk + columns: + - name: id + data_type: int + description: hello + constraints: + - type: not_null + - type: unique + - type: check + expression: (id > 0) + - type: check + expression: id >= 1 + tests: + - unique + - name: color + data_type: varchar(100) + - name: date_day + data_type: varchar(100) + - name: my_model_error + config: + contract: + enforced: true + columns: + - name: id + data_type: int + description: hello + constraints: + - type: not_null + - type: primary_key + - type: check + expression: (id > 0) + tests: + - unique + - name: color + data_type: varchar(100) + - name: date_day + data_type: varchar(100) + - name: my_model_wrong_order + config: + contract: + enforced: true + constraints: + - type: primary_key + columns: [id] + name: pk_my_model_pk + - type: unique + columns: [id] + name: uk_my_model_pk + columns: + - name: id + data_type: int + description: hello + constraints: + - type: not_null + - type: check + expression: (id > 0) + tests: + - unique + - name: color + data_type: varchar(100) + - name: date_day + data_type: varchar(100) + - name: my_model_wrong_name + config: + contract: + enforced: true + columns: + - name: id + data_type: int + description: hello + constraints: + - type: not_null + - type: primary_key + - type: check + expression: (id > 0) + tests: + - unique + - name: color + data_type: varchar(100) + - name: date_day + data_type: varchar(100) +""" + +model_fk_constraint_schema_yml = """ +version: 2 +models: + - name: my_model + config: + contract: + enforced: true + constraints: + - type: primary_key + columns: [id] + name: pk_my_model_pk + - type: foreign_key + expression: {schema}.foreign_key_model (id) + name: fk_my_model_id + columns: [id] + - type: unique + name: uk_my_model_id + columns: [id] + columns: + - name: id + data_type: int + description: hello + constraints: + - type: not_null + - type: check + expression: (id > 0) + - type: check + expression: id >= 1 + tests: + - unique + - name: color + data_type: varchar(100) + - name: date_day + data_type: varchar(100) + + - name: foreign_key_model + config: + contract: + enforced: true + constraints: + - type: primary_key + columns: [id] + name: pk_my_ref_model_id + - type: unique + name: uk_my_ref_model_id + columns: [id] + columns: + - name: id + data_type: int + constraints: + - type: not_null +""" + +constrained_model_schema_yml = """ +version: 2 +models: + - name: my_model + config: + contract: + enforced: true + constraints: + - type: check + expression: (id > 0) + - type: check + expression: id >= 1 + - type: primary_key + columns: [ id ] + name: strange_pk_requirement_my_model + - type: unique + columns: [ color, date_day ] + name: strange_uniqueness_requirement_my_model + - type: foreign_key + columns: [ id ] + expression: {schema}.foreign_key_model (id) + name: strange_pk_fk_requirement_my_model + columns: + - name: id + data_type: int + description: hello + constraints: + - type: not_null + tests: + - unique + - name: color + data_type: varchar(100) + - name: date_day + data_type: varchar(100) + - name: foreign_key_model + config: + contract: + enforced: true + constraints: + - type: primary_key + columns: [ id ] + name: strange_pk_requirement_fk_my_model + - type: unique + columns: [ id ] + name: fk_id_uniqueness_requirement + columns: + - name: id + data_type: int + constraints: + - type: not_null +""" + + +def _normalize_whitespace(input: str) -> str: + subbed = re.sub(r"\s+", " ", input) + return re.sub(r"\s?([\(\),])\s?", r"\1", subbed).lower().strip() + + +def _find_and_replace(sql, find, replace): + sql_tokens = sql.split() + for idx in [n for n, x in enumerate(sql_tokens) if find in x]: + sql_tokens[idx] = replace + return " ".join(sql_tokens) + + +class BaseConstraintsColumnsEqual: + """ + dbt should catch these mismatches during its "preflight" checks. + """ + + @pytest.fixture + def string_type(self): + return "varchar" + + @pytest.fixture + def int_type(self): + return "int" + + @pytest.fixture + def schema_string_type(self, string_type): + return string_type + + @pytest.fixture + def schema_int_type(self, int_type): + return int_type + + @pytest.fixture + def data_types(self, schema_int_type, int_type, string_type): + # sql_column_value, schema_data_type, error_data_type + return [ + ["1", schema_int_type, int_type], + ["'1'", string_type, string_type], + ["cast('2019-01-01' as date)", "date", "date"], + ["cast(1 as bit)", "bit", "bit"], + ["cast('2013-11-03 00:00:00.000000' as datetime2(6))", "datetime2(6)", "datetime2(6)"], + ["cast(1 as decimal(5,2))", "decimal", "decimal"], + ] + + def test__constraints_wrong_column_order(self, project): + # This no longer causes an error, since we enforce yaml column order + run_dbt(["run", "-s", "my_model_wrong_order"], expect_pass=True) + manifest = get_manifest(project.project_root) + model_id = "model.test.my_model_wrong_order" + my_model_config = manifest.nodes[model_id].config + contract_actual_config = my_model_config.contract + + assert contract_actual_config.enforced is True + + def test__constraints_wrong_column_names(self, project, string_type, int_type): + _, log_output = run_dbt_and_capture( + ["run", "-s", "my_model_wrong_name"], expect_pass=False + ) + manifest = get_manifest(project.project_root) + model_id = "model.test.my_model_wrong_name" + my_model_config = manifest.nodes[model_id].config + contract_actual_config = my_model_config.contract + + assert contract_actual_config.enforced is True + + expected = ["id", "error", "missing in definition", "missing in contract"] + assert all([(exp in log_output or exp.upper() in log_output) for exp in expected]) + + def test__constraints_wrong_column_data_types( + self, project, string_type, int_type, schema_string_type, schema_int_type, data_types + ): + for sql_column_value, schema_data_type, error_data_type in data_types: + # Write parametrized data_type to sql file + write_file( + my_model_data_type_sql.format(sql_value=sql_column_value), + "models", + "my_model_data_type.sql", + ) + + # Write wrong data_type to corresponding schema file + # Write integer type for all schema yaml values except when testing integer type itself + wrong_schema_data_type = ( + schema_int_type + if schema_data_type.upper() != schema_int_type.upper() + else schema_string_type + ) + wrong_schema_error_data_type = ( + int_type if schema_data_type.upper() != schema_int_type.upper() else string_type + ) + write_file( + model_data_type_schema_yml.format(data_type=wrong_schema_data_type), + "models", + "constraints_schema.yml", + ) + + results, log_output = run_dbt_and_capture( + ["run", "-s", "my_model_data_type"], expect_pass=False + ) + manifest = get_manifest(project.project_root) + model_id = "model.test.my_model_data_type" + my_model_config = manifest.nodes[model_id].config + contract_actual_config = my_model_config.contract + + assert contract_actual_config.enforced is True + expected = [ + "wrong_data_type_column_name", + error_data_type, + wrong_schema_error_data_type, + "data type mismatch", + ] + assert all([(exp in log_output or exp.upper() in log_output) for exp in expected]) + + def test__constraints_correct_column_data_types(self, project, data_types): + for sql_column_value, schema_data_type, _ in data_types: + # Write parametrized data_type to sql file + write_file( + my_model_data_type_sql.format(sql_value=sql_column_value), + "models", + "my_model_data_type.sql", + ) + # Write correct data_type to corresponding schema file + write_file( + model_data_type_schema_yml.format(data_type=schema_data_type), + "models", + "constraints_schema.yml", + ) + + run_dbt(["run", "-s", "my_model_data_type"]) + + manifest = get_manifest(project.project_root) + model_id = "model.test.my_model_data_type" + my_model_config = manifest.nodes[model_id].config + contract_actual_config = my_model_config.contract + + assert contract_actual_config.enforced is True + + +class BaseTableConstraintsColumnsEqual(BaseConstraintsColumnsEqual): + @pytest.fixture(scope="class") + def models(self): + return { + "my_model_wrong_order.sql": my_model_wrong_order_sql, + "my_model_wrong_name.sql": my_model_wrong_name_sql, + "constraints_schema.yml": model_schema_yml, + } + + +class BaseViewConstraintsColumnsEqual(BaseConstraintsColumnsEqual): + @pytest.fixture(scope="class") + def models(self): + return { + "my_model_wrong_order.sql": my_model_view_wrong_order_sql, + "my_model_wrong_name.sql": my_model_view_wrong_name_sql, + "constraints_schema.yml": model_schema_yml, + } + + +class BaseIncrementalConstraintsColumnsEqual(BaseConstraintsColumnsEqual): + @pytest.fixture(scope="class") + def models(self): + return { + "my_model_wrong_order.sql": my_model_incremental_wrong_order_sql, + "my_model_wrong_name.sql": my_model_incremental_wrong_name_sql, + "constraints_schema.yml": model_schema_yml, + } + + +class TestTableConstraintsColumnsEqual(BaseTableConstraintsColumnsEqual): + pass + + +class TestViewConstraintsColumnsEqual(BaseViewConstraintsColumnsEqual): + pass + + +class TestIncrementalConstraintsColumnsEqual(BaseIncrementalConstraintsColumnsEqual): + pass + + +class BaseConstraintsRuntimeDdlEnforcement: + """ + These constraints pass muster for dbt's preflight checks. Make sure they're + passed into the DDL statement. If they don't match up with the underlying data, + the data platform should raise an error at runtime. + """ + + my_model_sql = """ + {{ + config( + { + 'materialized': "table", + 'as_columnstore': False + } + ) + }} + + select + 1 as id, + 'blue' as color, + '2019-01-01' as date_day + """ + + foreign_key_model_sql = """ + {{ + config({ + 'materialized': "table", + 'as_columnstore': False + } + ) + }} + + select + 1 as id + """ + + my_model_wrong_order_depends_on_fk_sql = """ + {{ + config( + { + 'materialized': "table", + 'as_columnstore': False + } + ) + }} + + -- depends_on: {{ ref('foreign_key_model') }} + + select + 'blue' as color, + 1 as id, + '2019-01-01' as date_day + """ + + @pytest.fixture(scope="class") + def models(self): + return { + "my_model.sql": self.my_model_wrong_order_depends_on_fk_sql, + "foreign_key_model.sql": self.foreign_key_model_sql, + "constraints_schema.yml": model_fk_constraint_schema_yml, + } + + @pytest.fixture(scope="class") + def expected_sql(self): + return """ + EXEC(' create view as -- depends_on: select ''blue'' as color, 1 as id, ''2019-01-01'' as date_day; ') EXEC(' CREATE TABLE ( id int not null , color varchar(100), date_day varchar(100) ) INSERT INTO ( [id], [color], [date_day] ) SELECT [id], [color], [date_day] FROM ') EXEC('DROP VIEW IF EXISTS + """ + + # EXEC('DROP view IF EXISTS + + def test__constraints_ddl(self, project, expected_sql): + unformatted_constraint_schema_yml = read_file("models", "constraints_schema.yml") + write_file( + unformatted_constraint_schema_yml.format(schema=project.test_schema), + "models", + "constraints_schema.yml", + ) + + results = run_dbt(["run", "-s", "+my_model"]) + assert len(results) >= 1 + + # TODO: consider refactoring this to introspect logs instead + generated_sql = read_file("target", "run", "test", "models", "my_model.sql") + generated_sql_generic = _find_and_replace(generated_sql, "my_model", "") + generated_sql_generic = _find_and_replace( + generated_sql_generic, "foreign_key_model", "" + ) + generated_sql_wodb = generated_sql_generic.replace("USE [" + project.database + "];", "") + generated_sql_option_clause = generated_sql_wodb.replace( + "OPTION (LABEL = ''dbt-sqlserver'');", "" + ) + generated_sql_option_clause = generated_sql_option_clause.replace( + " -- add columnstore index ", "" + ) + generated_sql_option_clause = generated_sql_option_clause.replace( + "CREATE CLUSTERED COLUMNSTORE INDEX ", "" + ) + assert _normalize_whitespace(expected_sql.strip()) == _normalize_whitespace( + generated_sql_option_clause.strip() + ) + + +class TestTableConstraintsRuntimeDdlEnforcement(BaseConstraintsRuntimeDdlEnforcement): + pass + + +class BaseIncrementalConstraintsRuntimeDdlEnforcement(BaseConstraintsRuntimeDdlEnforcement): + my_model_incremental_wrong_order_depends_on_fk_sql = """ + {{ + config( + materialized = "incremental", + on_schema_change='append_new_columns', + as_columnstore = False + ) + }} + + -- depends_on: {{ ref('foreign_key_model') }} + + select + 'blue' as color, + 1 as id, + '2019-01-01' as date_day + """ + + @pytest.fixture(scope="class") + def models(self): + return { + "my_model.sql": self.my_model_incremental_wrong_order_depends_on_fk_sql, + "foreign_key_model.sql": self.foreign_key_model_sql, + "constraints_schema.yml": model_fk_constraint_schema_yml, + } + + +class TestIncrementalConstraintsRuntimeDdlEnforcement( + BaseIncrementalConstraintsRuntimeDdlEnforcement +): + pass + + +class BaseModelConstraintsRuntimeEnforcement: + """ + These model-level constraints pass muster for dbt's preflight checks. Make sure they're + passed into the DDL statement. If they don't match up with the underlying data, + the data platform should raise an error at runtime. + """ + + foreign_key_model_sql = """ + {{ + config( + materialized = "table", + as_columnstore = False + ) + }} + + select + 1 as id + """ + + my_model_wrong_order_depends_on_fk_sql = """ + {{ + config( + materialized = "table", + as_columnstore = False + ) + }} + + -- depends_on: {{ ref('foreign_key_model') }} + + select + 'blue' as color, + 1 as id, + '2019-01-01' as date_day + """ + + @pytest.fixture(scope="class") + def models(self): + return { + "my_model.sql": self.my_model_wrong_order_depends_on_fk_sql, + "foreign_key_model.sql": self.foreign_key_model_sql, + "constraints_schema.yml": constrained_model_schema_yml, + } + + @pytest.fixture(scope="class") + def expected_sql(self): + return """ + EXEC(' create view as -- depends_on: select ''blue'' as color, 1 as id, ''2019-01-01'' as date_day; ') EXEC(' CREATE TABLE ( id int not null , color varchar(100), date_day varchar(100) ) INSERT INTO ( [id], [color], [date_day] ) SELECT [id], [color], [date_day] FROM ') EXEC('DROP VIEW IF EXISTS + """ + + def test__model_constraints_ddl(self, project, expected_sql): + unformatted_constraint_schema_yml = read_file("models", "constraints_schema.yml") + write_file( + unformatted_constraint_schema_yml.format(schema=project.test_schema), + "models", + "constraints_schema.yml", + ) + + results = run_dbt(["run", "-s", "+my_model"]) + assert len(results) >= 1 + generated_sql = read_file("target", "run", "test", "models", "my_model.sql") + + generated_sql_generic = _find_and_replace(generated_sql, "my_model", "") + generated_sql_generic = _find_and_replace( + generated_sql_generic, "foreign_key_model", "" + ) + generated_sql_wodb = generated_sql_generic.replace("USE [" + project.database + "];", "") + generated_sql_option_clause = generated_sql_wodb.replace( + "OPTION (LABEL = ''dbt-sqlserver'');", "" + ) + assert _normalize_whitespace(expected_sql.strip()) == _normalize_whitespace( + generated_sql_option_clause.strip() + ) + + +class TestModelConstraintsRuntimeEnforcement(BaseModelConstraintsRuntimeEnforcement): + pass + + +class BaseConstraintsRollback: + @pytest.fixture(scope="class") + def models(self): + return { + "my_model.sql": my_model_sql, + "constraints_schema.yml": model_schema_yml, + } + + @pytest.fixture(scope="class") + def null_model_sql(self): + return my_model_with_nulls_sql + + @pytest.fixture(scope="class") + def expected_color(self): + return "blue" + + @pytest.fixture(scope="class") + def expected_error_messages(self): + return [ + "Cannot insert the value NULL into column", + "column does not allow nulls", + "There is already an object", + ] + + def assert_expected_error_messages(self, error_message, expected_error_messages): + assert any(msg in error_message for msg in expected_error_messages) + + def test__constraints_enforcement_rollback( + self, project, expected_color, expected_error_messages, null_model_sql + ): + results = run_dbt(["run", "-s", "my_model"]) + assert len(results) == 1 + + # Make a contract-breaking change to the model + write_file(null_model_sql, "models", "my_model.sql") + + failing_results = run_dbt(["run", "-s", "my_model"], expect_pass=False) + assert len(failing_results) == 1 + + # Verify the previous table still exists + relation = relation_from_name(project.adapter, "my_model") + # table materialization does not rename existing relation to back)up relation + # Rather, a new relation is created with __dbt_temp suffix. + # If model creation is successful, then the existing model is renamed as backup and then dropped + model_backup = str(relation).replace("my_model", "my_model") + old_model_exists_sql = f"select * from {model_backup}" + old_model_exists = project.run_sql(old_model_exists_sql, fetch="all") + assert len(old_model_exists) == 1 + assert old_model_exists[0][1] == expected_color + + # Confirm this model was contracted + # TODO: is this step really necessary? + manifest = get_manifest(project.project_root) + model_id = "model.test.my_model" + my_model_config = manifest.nodes[model_id].config + contract_actual_config = my_model_config.contract + assert contract_actual_config.enforced is True + + # Its result includes the expected error messages + self.assert_expected_error_messages(failing_results[0].message, expected_error_messages) + + +class BaseIncrementalConstraintsRollback(BaseConstraintsRollback): + my_incremental_model_sql = """ + {{ + config( + materialized = "incremental", + on_schema_change='append_new_columns', + as_columnstore = False + ) + }} + + select + 1 as id, + 'blue' as color, + '2019-01-01' as date_day + """ + + @pytest.fixture(scope="class") + def models(self): + return { + "my_model.sql": my_incremental_model_sql, + "constraints_schema.yml": model_schema_yml, + } + + @pytest.fixture(scope="class") + def null_model_sql(self): + return my_model_incremental_with_nulls_sql + + +class TestTableConstraintsRollback(BaseConstraintsRollback): + pass + + +class TestIncrementalConstraintsRollback(BaseIncrementalConstraintsRollback): + def test__constraints_enforcement_rollback( + self, project, expected_color, expected_error_messages, null_model_sql + ): + results = run_dbt(["run", "-s", "my_model"]) + assert len(results) == 1 + + # Make a contract-breaking change to the model + write_file(null_model_sql, "models", "my_model.sql") + # drops the previous table before + # when there is an exception, cant rollback + failing_results = run_dbt(["run", "-s", "my_model"], expect_pass=False) + assert len(failing_results) == 1 + + # Verify the previous table still exists, + # for incremental we are not creating backups, because its not a create replace + relation = relation_from_name(project.adapter, "my_model") + old_model_exists_sql = f"select * from {relation}" + old_model_exists = project.run_sql(old_model_exists_sql, fetch="all") + assert len(old_model_exists) == 1 + assert old_model_exists[0][1] == expected_color + + # Confirm this model was contracted + # TODO: is this step really necessary? + manifest = get_manifest(project.project_root) + model_id = "model.test.my_model" + my_model_config = manifest.nodes[model_id].config + contract_actual_config = my_model_config.contract + assert contract_actual_config.enforced is True + + # Its result includes the expected error messages + self.assert_expected_error_messages(failing_results[0].message, expected_error_messages) diff --git a/tests/functional/adapter/dbt/test_dbt_clone.py b/tests/functional/adapter/dbt/test_dbt_clone.py new file mode 100644 index 00000000..e55e5e18 --- /dev/null +++ b/tests/functional/adapter/dbt/test_dbt_clone.py @@ -0,0 +1,12 @@ +import pytest +from dbt.tests.adapter.dbt_clone.test_dbt_clone import BaseCloneNotPossible, BaseClonePossible + + +@pytest.mark.skip(reason="SQLServer does not support cloning") +class TestCloneNotPossible(BaseCloneNotPossible): + pass + + +@pytest.mark.skip(reason="SQLServer does not support cloning") +class TestClonePossible(BaseClonePossible): + pass diff --git a/tests/functional/adapter/dbt/test_dbt_debug.py b/tests/functional/adapter/dbt/test_dbt_debug.py new file mode 100644 index 00000000..4159f399 --- /dev/null +++ b/tests/functional/adapter/dbt/test_dbt_debug.py @@ -0,0 +1,12 @@ +from dbt.tests.adapter.dbt_debug.test_dbt_debug import ( + BaseDebugInvalidProjectPostgres, + BaseDebugPostgres, +) + + +class TestDebugProfileVariable(BaseDebugPostgres): + pass + + +class TestDebugInvalidProject(BaseDebugInvalidProjectPostgres): + pass diff --git a/tests/functional/adapter/dbt/test_empty.py b/tests/functional/adapter/dbt/test_empty.py new file mode 100644 index 00000000..52ea88de --- /dev/null +++ b/tests/functional/adapter/dbt/test_empty.py @@ -0,0 +1,56 @@ +import pytest + +# switch for 1.9 +# from dbt.tests.adapter.empty import _models +from dbt.tests.adapter.empty.test_empty import ( # MetadataWithEmptyFlag + BaseTestEmpty, + BaseTestEmptyInlineSourceRef, + model_input_sql, + schema_sources_yml, +) +from dbt.tests.util import run_dbt + +model_sql_sqlserver = """ +select * +from {{ ref('model_input') }} +union all +select * +from {{ source('seed_sources', 'raw_source') }} +""" + +model_inline_sql_sqlserver = """ +select * from {{ source('seed_sources', 'raw_source') }} +""" + + +class TestEmpty(BaseTestEmpty): + @pytest.fixture(scope="class") + def models(self): + return { + "model_input.sql": model_input_sql, + # # no support for ephemeral models in SQLServer + # "ephemeral_model_input.sql": _models.ephemeral_model_input_sql, + "model.sql": model_sql_sqlserver, + "sources.yml": schema_sources_yml, + } + + def test_run_with_empty(self, project): + # create source from seed + run_dbt(["seed"]) + + # run without empty - 3 expected rows in output - 1 from each input + run_dbt(["run"]) + self.assert_row_count(project, "model", 2) + + # run with empty - 0 expected rows in output + run_dbt(["run", "--empty"]) + self.assert_row_count(project, "model", 0) + + +class TestemptyInlineSourceRef(BaseTestEmptyInlineSourceRef): + @pytest.fixture(scope="class") + def models(self): + return { + "model.sql": model_inline_sql_sqlserver, + "sources.yml": schema_sources_yml, + } diff --git a/tests/functional/adapter/dbt/test_ephemeral.py b/tests/functional/adapter/dbt/test_ephemeral.py new file mode 100644 index 00000000..fd077d8c --- /dev/null +++ b/tests/functional/adapter/dbt/test_ephemeral.py @@ -0,0 +1,21 @@ +import pytest +from dbt.tests.adapter.ephemeral.test_ephemeral import ( + BaseEphemeralErrorHandling, + BaseEphemeralMulti, + BaseEphemeralNested, +) + + +@pytest.mark.skip(reason="Epemeral models are not supported in SQLServer") +class TestEphemeral(BaseEphemeralMulti): + pass + + +@pytest.mark.skip(reason="Epemeral models are not supported in SQLServer") +class TestEphemeralNested(BaseEphemeralNested): + pass + + +@pytest.mark.skip(reason="Epemeral models are not supported in SQLServer") +class TestEphemeralErrorHandling(BaseEphemeralErrorHandling): + pass diff --git a/tests/functional/adapter/dbt/test_grants.py b/tests/functional/adapter/dbt/test_grants.py new file mode 100644 index 00000000..52a20803 --- /dev/null +++ b/tests/functional/adapter/dbt/test_grants.py @@ -0,0 +1,26 @@ +from dbt.tests.adapter.grants.test_incremental_grants import BaseIncrementalGrants +from dbt.tests.adapter.grants.test_invalid_grants import BaseInvalidGrants +from dbt.tests.adapter.grants.test_model_grants import BaseModelGrants +from dbt.tests.adapter.grants.test_seed_grants import BaseSeedGrants +from dbt.tests.adapter.grants.test_snapshot_grants import BaseSnapshotGrants + + +class TestIncrementalGrants(BaseIncrementalGrants): + pass + + +class TestInvalidGrants(BaseInvalidGrants): + def privilege_does_not_exist_error(self): + return "Incorrect syntax near" + + +class TestModelGrants(BaseModelGrants): + pass + + +class TestSeedGrants(BaseSeedGrants): + pass + + +class TestSnapshotGrants(BaseSnapshotGrants): + pass diff --git a/tests/functional/adapter/dbt/test_hooks.py b/tests/functional/adapter/dbt/test_hooks.py new file mode 100644 index 00000000..9daaaa51 --- /dev/null +++ b/tests/functional/adapter/dbt/test_hooks.py @@ -0,0 +1,235 @@ +import pytest +from dbt.tests.adapter.hooks import fixtures +from dbt.tests.util import run_dbt + +seed_model_sql = """ +drop table if exists {schema}.on_model_hook; + +create table {schema}.on_model_hook ( + test_state VARCHAR(8000), -- start|end + target_dbname VARCHAR(8000), + target_host VARCHAR(8000), + target_name VARCHAR(8000), + target_schema VARCHAR(8000), + target_type VARCHAR(8000), + target_user VARCHAR(8000), + target_pass VARCHAR(8000), + target_threads INTEGER, + run_started_at VARCHAR(8000), + invocation_id VARCHAR(8000), + thread_id VARCHAR(8000) +); +""".strip() + +MODEL_PRE_HOOK = """ + insert into {{this.schema}}.on_model_hook ( + test_state, + target_dbname, + target_host, + target_name, + target_schema, + target_type, + target_user, + target_pass, + target_threads, + run_started_at, + invocation_id, + thread_id + ) VALUES ( + 'start', + '{{ target.database }}', + '{{ target.server }}', + '{{ target.name }}', + '{{ target.schema }}', + '{{ target.type }}', + '{{ target.user }}', + '{{ target.get("pass", "") }}', + {{ target.threads }}, + '{{ run_started_at }}', + '{{ invocation_id }}', + '{{ thread_id }}' + ) +""" + +MODEL_POST_HOOK = """ + insert into {{this.schema}}.on_model_hook ( + test_state, + target_dbname, + target_host, + target_name, + target_schema, + target_type, + target_user, + target_pass, + target_threads, + run_started_at, + invocation_id, + thread_id + ) VALUES ( + 'end', + '{{ target.database }}', + '{{ target.server }}', + '{{ target.name }}', + '{{ target.schema }}', + '{{ target.type }}', + '{{ target.user }}', + '{{ target.get("pass", "") }}', + {{ target.threads }}, + '{{ run_started_at }}', + '{{ invocation_id }}', + '{{ thread_id }}' + ) +""" + + +class BaseTestPrePost: + @pytest.fixture(scope="class", autouse=True) + def setUp(self, project): + project.run_sql(seed_model_sql) + + def get_ctx_vars(self, state, count, project): + fields = [ + "test_state", + "target_dbname", + "target_host", + "target_name", + "target_schema", + "target_threads", + "target_type", + "target_user", + "target_pass", + "run_started_at", + "invocation_id", + "thread_id", + ] + field_list = ", ".join(['"{}"'.format(f) for f in fields]) + query = f""" + select + {field_list} + from + {project.test_schema}.on_model_hook where test_state = '{state}'""" + + vals = project.run_sql(query, fetch="all") + assert len(vals) != 0, "nothing inserted into hooks table" + assert len(vals) >= count, "too few rows in hooks table" + assert len(vals) <= count, "too many rows in hooks table" + return [{k: v for k, v in zip(fields, val)} for val in vals] + + def check_hooks(self, state, project, host, count=1): + ctxs = self.get_ctx_vars(state, count=count, project=project) + for ctx in ctxs: + assert ctx["test_state"] == state + # assert ctx["target_dbname"] == "TestDB" + # assert ctx["target_host"] == host + assert ctx["target_name"] == "default" + assert ctx["target_schema"] == project.test_schema + assert ctx["target_threads"] == 1 + assert ctx["target_type"] == project.adapter_type + # assert ctx["target_user"] == "root" + # assert ctx["target_pass"] == "" + + assert ( + ctx["run_started_at"] is not None and len(ctx["run_started_at"]) > 0 + ), "run_started_at was not set" + assert ( + ctx["invocation_id"] is not None and len(ctx["invocation_id"]) > 0 + ), "invocation_id was not set" + assert ctx["thread_id"].startswith("Thread-") + + +class BasePrePostModelHooks(BaseTestPrePost): + @pytest.fixture(scope="class") + def project_config_update(self): + return { + "models": { + "test": { + "pre-hook": [ + # inside transaction (runs second) + MODEL_PRE_HOOK, + ], + "post-hook": [ + # inside transaction (runs first) + MODEL_POST_HOOK, + ], + } + } + } + + @pytest.fixture(scope="class") + def models(self): + return {"hooks.sql": fixtures.models__hooks} + + def test_pre_and_post_run_hooks(self, project, dbt_profile_target): + run_dbt() + self.check_hooks("start", project, dbt_profile_target.get("host", None)) + self.check_hooks("end", project, dbt_profile_target.get("host", None)) + + +class TestPrePostModelHooks(BasePrePostModelHooks): + pass + + +class TestPrePostModelHooksUnderscores(BasePrePostModelHooks): + @pytest.fixture(scope="class") + def project_config_update(self): + return { + "models": { + "test": { + "pre_hook": [ + # inside transaction (runs second) + MODEL_PRE_HOOK, + ], + "post_hook": [ + # inside transaction (runs first) + MODEL_POST_HOOK, + ], + } + } + } + + +class BaseHookRefs(BaseTestPrePost): + @pytest.fixture(scope="class") + def project_config_update(self): + return { + "models": { + "test": { + "hooked": { + "post-hook": [ + """ + insert into {{this.schema}}.on_model_hook select + test_state, + '{{ target.dbname }}' as target_dbname, + '{{ target.host }}' as target_host, + '{{ target.name }}' as target_name, + '{{ target.schema }}' as target_schema, + '{{ target.type }}' as target_type, + '{{ target.user }}' as target_user, + '{{ target.get(pass, "") }}' as target_pass, + {{ target.threads }} as target_threads, + '{{ run_started_at }}' as run_started_at, + '{{ invocation_id }}' as invocation_id, + '{{ thread_id }}' as thread_id + from {{ ref('post') }}""".strip() + ], + } + }, + } + } + + @pytest.fixture(scope="class") + def models(self): + return { + "hooked.sql": fixtures.models__hooked, + "post.sql": fixtures.models__post, + "pre.sql": fixtures.models__pre, + } + + def test_pre_post_model_hooks_refed(self, project, dbt_profile_target): + run_dbt() + self.check_hooks("start", project, dbt_profile_target.get("host", None)) + self.check_hooks("end", project, dbt_profile_target.get("host", None)) + + +class TestHookRefs(BaseHookRefs): + pass diff --git a/tests/functional/adapter/dbt/test_incremental.py b/tests/functional/adapter/dbt/test_incremental.py new file mode 100644 index 00000000..959a81d0 --- /dev/null +++ b/tests/functional/adapter/dbt/test_incremental.py @@ -0,0 +1,107 @@ +import pytest +from dbt.tests.adapter.incremental import fixtures +from dbt.tests.adapter.incremental.test_incremental_merge_exclude_columns import ( + BaseMergeExcludeColumns, +) +from dbt.tests.adapter.incremental.test_incremental_on_schema_change import ( + BaseIncrementalOnSchemaChange, +) +from dbt.tests.adapter.incremental.test_incremental_predicates import ( + TestIncrementalPredicatesDeleteInsert, + TestPredicatesDeleteInsert, +) + +_MODELS__INCREMENTAL_IGNORE_SQLServer = """ +{{ + config( + materialized='incremental', + unique_key='id', + on_schema_change='ignore' + ) +}} + +WITH source_data AS (SELECT * FROM {{ ref('model_a') }} ) + +{% if is_incremental() %} + +SELECT id, field1, field2, field3, field4 +FROM source_data WHERE id NOT IN (SELECT id from {{ this }} ) + +{% else %} + +SELECT TOP 3 id, field1, field2 FROM source_data + +{% endif %} +""" + +_MODELS__INCREMENTAL_SYNC_REMOVE_ONLY_TARGET_SQLServer = """ +{{ + config(materialized='table') +}} + +with source_data as ( + + select * from {{ ref('model_a') }} + +) + +{% set string_type = dbt.type_string() %} + +select id + ,cast(field1 as {{string_type}}) as field1 + +from source_data +""" + +_MODELS__INCREMENTAL_SYNC_ALL_COLUMNS_TARGET_SQLServer = """ +{{ + config(materialized='table') +}} + +with source_data as ( + + select * from {{ ref('model_a') }} + +) + +{% set string_type = dbt.type_string() %} + +select id + ,cast(field1 as {{string_type}}) as field1 + --,field2 + ,cast(case when id <= 3 then null else field3 end as {{string_type}}) as field3 + ,cast(case when id <= 3 then null else field4 end as {{string_type}}) as field4 + +from source_data +""" + + +class TestIncrementalMergeExcludeColumns(BaseMergeExcludeColumns): + pass + + +class TestIncrementalOnSchemaChange(BaseIncrementalOnSchemaChange): + @pytest.fixture(scope="class") + def models(self): + return { + "incremental_sync_remove_only.sql": fixtures._MODELS__INCREMENTAL_SYNC_REMOVE_ONLY, + "incremental_ignore.sql": _MODELS__INCREMENTAL_IGNORE_SQLServer, + "incremental_sync_remove_only_target.sql": _MODELS__INCREMENTAL_SYNC_REMOVE_ONLY_TARGET_SQLServer, # noqa: E501 + "incremental_ignore_target.sql": fixtures._MODELS__INCREMENTAL_IGNORE_TARGET, + "incremental_fail.sql": fixtures._MODELS__INCREMENTAL_FAIL, + "incremental_sync_all_columns.sql": fixtures._MODELS__INCREMENTAL_SYNC_ALL_COLUMNS, + "incremental_append_new_columns_remove_one.sql": fixtures._MODELS__INCREMENTAL_APPEND_NEW_COLUMNS_REMOVE_ONE, # noqa: E501 + "model_a.sql": fixtures._MODELS__A, + "incremental_append_new_columns_target.sql": fixtures._MODELS__INCREMENTAL_APPEND_NEW_COLUMNS_TARGET, # noqa: E501 + "incremental_append_new_columns.sql": fixtures._MODELS__INCREMENTAL_APPEND_NEW_COLUMNS, # noqa: E501 + "incremental_sync_all_columns_target.sql": _MODELS__INCREMENTAL_SYNC_ALL_COLUMNS_TARGET_SQLServer, # noqa: E501 + "incremental_append_new_columns_remove_one_target.sql": fixtures._MODELS__INCREMENTAL_APPEND_NEW_COLUMNS_REMOVE_ONE_TARGET, # noqa: E501 + } + + +class TestIncrementalPredicatesDeleteInsert(TestIncrementalPredicatesDeleteInsert): + pass + + +class TestPredicatesDeleteInsert(TestPredicatesDeleteInsert): + pass diff --git a/tests/functional/adapter/dbt/test_materialized_views.py b/tests/functional/adapter/dbt/test_materialized_views.py new file mode 100644 index 00000000..c45e752b --- /dev/null +++ b/tests/functional/adapter/dbt/test_materialized_views.py @@ -0,0 +1,7 @@ +import pytest +from dbt.tests.adapter.materialized_view.basic import MaterializedViewBasic + + +@pytest.mark.skip(reason="Materialized views are not supported in SQLServer") +class TestMaterializedViews(MaterializedViewBasic): + pass diff --git a/tests/functional/adapter/dbt/test_persist_docs.py b/tests/functional/adapter/dbt/test_persist_docs.py new file mode 100644 index 00000000..d0b0fdce --- /dev/null +++ b/tests/functional/adapter/dbt/test_persist_docs.py @@ -0,0 +1,12 @@ +import pytest +from dbt.tests.adapter.persist_docs.test_persist_docs import BasePersistDocs + + +@pytest.mark.skip( + reason=""" + Persisted docs are not implemented in SQLServer. + Could be implemented with sp_addextendedproperty + """ +) +class TestPersistDocs(BasePersistDocs): + pass diff --git a/tests/functional/adapter/dbt/test_python_model.py b/tests/functional/adapter/dbt/test_python_model.py new file mode 100644 index 00000000..66f3b065 --- /dev/null +++ b/tests/functional/adapter/dbt/test_python_model.py @@ -0,0 +1,21 @@ +import pytest +from dbt.tests.adapter.python_model.test_python_model import ( + BasePythonIncrementalTests, + BasePythonModelTests, +) +from dbt.tests.adapter.python_model.test_spark import BasePySparkTests + + +@pytest.mark.skip(reason="Python models are not supported in SQLServer") +class TestPythonModel(BasePythonModelTests): + pass + + +@pytest.mark.skip(reason="Python models are not supported in SQLServer") +class TestPythonIncremental(BasePythonIncrementalTests): + pass + + +@pytest.mark.skip(reason="Python models are not supported in SQLServer") +class TestPySpark(BasePySparkTests): + pass diff --git a/tests/functional/adapter/dbt/test_query_comment.py b/tests/functional/adapter/dbt/test_query_comment.py new file mode 100644 index 00000000..8c376c88 --- /dev/null +++ b/tests/functional/adapter/dbt/test_query_comment.py @@ -0,0 +1,32 @@ +from dbt.tests.adapter.query_comment.test_query_comment import ( + BaseEmptyQueryComments, + BaseMacroArgsQueryComments, + BaseMacroInvalidQueryComments, + BaseMacroQueryComments, + BaseNullQueryComments, + BaseQueryComments, +) + + +class TestQueryComments(BaseQueryComments): + pass + + +class TestMacroQueryComments(BaseMacroQueryComments): + pass + + +class TestMacroArgsQueryComments(BaseMacroArgsQueryComments): + pass + + +class TestMacroInvalidQueryComments(BaseMacroInvalidQueryComments): + pass + + +class TestNullQueryComments(BaseNullQueryComments): + pass + + +class TestEmptyQueryComments(BaseEmptyQueryComments): + pass diff --git a/tests/functional/adapter/dbt/test_relations.py b/tests/functional/adapter/dbt/test_relations.py new file mode 100644 index 00000000..45ebe161 --- /dev/null +++ b/tests/functional/adapter/dbt/test_relations.py @@ -0,0 +1,18 @@ +import pytest +from dbt.tests.adapter.relations.test_changing_relation_type import BaseChangeRelationTypeValidator +from dbt.tests.adapter.relations.test_dropping_schema_named import BaseDropSchemaNamed + + +class TestChangeRelationTypeValidator(BaseChangeRelationTypeValidator): + pass + + +@pytest.mark.xfail( + reason=""" + Test fails as its not passing Use[] properly. + `Use[None]` is called, should be `User[TestDB]` + Unclear why the macro doens't pass it properly. + """ +) +class TestDropSchemaNamed(BaseDropSchemaNamed): + pass diff --git a/tests/functional/adapter/dbt/test_simple_seed.py b/tests/functional/adapter/dbt/test_simple_seed.py new file mode 100644 index 00000000..776d1186 --- /dev/null +++ b/tests/functional/adapter/dbt/test_simple_seed.py @@ -0,0 +1,292 @@ +import pytest +from dbt.tests.adapter.simple_seed.seeds import seeds__expected_sql +from dbt.tests.adapter.simple_seed.test_seed import ( + BaseBasicSeedTests, + BaseSeedConfigFullRefreshOff, + BaseSeedConfigFullRefreshOn, + BaseSeedCustomSchema, + BaseSeedWithEmptyDelimiter, + BaseSeedWithUniqueDelimiter, + BaseSeedWithWrongDelimiter, + BaseSimpleSeedEnabledViaConfig, +) +from dbt.tests.util import check_table_does_exist, check_table_does_not_exist, run_dbt + +seeds__expected_sql = seeds__expected_sql.replace( + "TIMESTAMP WITHOUT TIME ZONE", "DATETIME2(6)" +).replace("TEXT", "VARCHAR(8000)") + +properties__schema_yml = """ +version: 2 +seeds: +- name: seed_enabled + columns: + - name: birthday + data_tests: + - column_type: + type: date + - name: seed_id + data_tests: + - column_type: + type: varchar(8000) + +- name: seed_tricky + columns: + - name: seed_id + data_tests: + - column_type: + type: integer + - name: seed_id_str + data_tests: + - column_type: + type: varchar(8000) + - name: a_bool + data_tests: + - column_type: + type: boolean + - name: looks_like_a_bool + data_tests: + - column_type: + type: varchar(8000) + - name: a_date + data_tests: + - column_type: + type: datetime2(6) + - name: looks_like_a_date + data_tests: + - column_type: + type: varchar(8000) + - name: relative + data_tests: + - column_type: + type: varchar(8000) + - name: weekday + data_tests: + - column_type: + type: varchar(8000) +""" + + +class TestBasicSeedTests(BaseBasicSeedTests): + @pytest.fixture(scope="class", autouse=True) + def setUp(self, project): + """Create table for ensuring seeds and models used in tests build correctly""" + project.run_sql(seeds__expected_sql) + + def test_simple_seed_full_refresh_flag(self, project): + """ + Drop the seed_actual table and re-create. + Verifies correct behavior by the absence of the + model which depends on seed_actual.""" + self._build_relations_for_test(project) + self._check_relation_end_state( + run_result=run_dbt(["seed", "--full-refresh"]), project=project, exists=True + ) + + +class TestSeedConfigFullRefreshOn(BaseSeedConfigFullRefreshOn): + @pytest.fixture(scope="class", autouse=True) + def setUp(self, project): + """Create table for ensuring seeds and models used in tests build correctly""" + project.run_sql(seeds__expected_sql) + + def test_simple_seed_full_refresh_config(self, project): + """config option should drop current model and cascade drop to downstream models""" + self._build_relations_for_test(project) + self._check_relation_end_state(run_result=run_dbt(["seed"]), project=project, exists=True) + + +class TestSeedConfigFullRefreshOff(BaseSeedConfigFullRefreshOff): + @pytest.fixture(scope="class", autouse=True) + def setUp(self, project): + """Create table for ensuring seeds and models used in tests build correctly""" + project.run_sql(seeds__expected_sql) + + +class TestSeedCustomSchema(BaseSeedCustomSchema): + @pytest.fixture(scope="class", autouse=True) + def setUp(self, project): + """Create table for ensuring seeds and models used in tests build correctly""" + project.run_sql(seeds__expected_sql) + + +class TestSeedWithUniqueDelimiter(BaseSeedWithUniqueDelimiter): + @pytest.fixture(scope="class", autouse=True) + def setUp(self, project): + """Create table for ensuring seeds and models used in tests build correctly""" + project.run_sql(seeds__expected_sql) + + +class TestSeedWithWrongDelimiter(BaseSeedWithWrongDelimiter): + @pytest.fixture(scope="class", autouse=True) + def setUp(self, project): + """Create table for ensuring seeds and models used in tests build correctly""" + project.run_sql(seeds__expected_sql) + + def test_seed_with_wrong_delimiter(self, project): + """Testing failure of running dbt seed with a wrongly configured delimiter""" + seed_result = run_dbt(["seed"], expect_pass=False) + assert "incorrect syntax" in seed_result.results[0].message.lower() + + +class TestSeedWithEmptyDelimiter(BaseSeedWithEmptyDelimiter): + @pytest.fixture(scope="class", autouse=True) + def setUp(self, project): + """Create table for ensuring seeds and models used in tests build correctly""" + project.run_sql(seeds__expected_sql) + + +class TestSimpleSeedEnabledViaConfig__seed_with_disabled(BaseSimpleSeedEnabledViaConfig): + @pytest.fixture(scope="function") + def clear_test_schema(self, project): + yield + project.run_sql( + f"drop table if exists {project.database}.{project.test_schema}.seed_enabled" + ) + project.run_sql( + f"drop table if exists {project.database}.{project.test_schema}.seed_disabled" + ) + project.run_sql( + f"drop table if exists {project.database}.{project.test_schema}.seed_tricky" + ) + project.run_sql(f"drop view if exists {project.test_schema}.seed_enabled") + project.run_sql(f"drop view if exists {project.test_schema}.seed_disabled") + project.run_sql(f"drop view if exists {project.test_schema}.seed_tricky") + project.run_sql(f"drop schema if exists {project.test_schema}") + + def test_simple_seed_with_disabled(self, clear_test_schema, project): + results = run_dbt(["seed"]) + assert len(results) == 2 + check_table_does_exist(project.adapter, "seed_enabled") + check_table_does_not_exist(project.adapter, "seed_disabled") + check_table_does_exist(project.adapter, "seed_tricky") + + @pytest.mark.skip( + reason=""" + Running all the tests in the same schema causes the tests to fail + as they all share the same schema across the tests + """ + ) + def test_simple_seed_selection(self, clear_test_schema, project): + results = run_dbt(["seed", "--select", "seed_enabled"]) + assert len(results) == 1 + check_table_does_exist(project.adapter, "seed_enabled") + check_table_does_not_exist(project.adapter, "seed_disabled") + check_table_does_not_exist(project.adapter, "seed_tricky") + + @pytest.mark.skip( + reason=""" + Running all the tests in the same schema causes the tests to fail + as they all share the same schema across the tests + """ + ) + def test_simple_seed_exclude(self, clear_test_schema, project): + results = run_dbt(["seed", "--exclude", "seed_enabled"]) + assert len(results) == 1 + check_table_does_not_exist(project.adapter, "seed_enabled") + check_table_does_not_exist(project.adapter, "seed_disabled") + check_table_does_exist(project.adapter, "seed_tricky") + + +class TestSimpleSeedEnabledViaConfig__seed_selection(BaseSimpleSeedEnabledViaConfig): + @pytest.fixture(scope="function") + def clear_test_schema(self, project): + yield + project.run_sql( + f"drop table if exists {project.database}.{project.test_schema}.seed_enabled" + ) + project.run_sql( + f"drop table if exists {project.database}.{project.test_schema}.seed_disabled" + ) + project.run_sql( + f"drop table if exists {project.database}.{project.test_schema}.seed_tricky" + ) + project.run_sql(f"drop view if exists {project.test_schema}.seed_enabled") + project.run_sql(f"drop view if exists {project.test_schema}.seed_disabled") + project.run_sql(f"drop view if exists {project.test_schema}.seed_tricky") + project.run_sql(f"drop schema if exists {project.test_schema}") + + @pytest.mark.skip( + reason=""" + Running all the tests in the same schema causes the tests to fail + as they all share the same schema across the tests + """ + ) + def test_simple_seed_with_disabled(self, clear_test_schema, project): + results = run_dbt(["seed"]) + assert len(results) == 2 + check_table_does_exist(project.adapter, "seed_enabled") + check_table_does_not_exist(project.adapter, "seed_disabled") + check_table_does_exist(project.adapter, "seed_tricky") + + def test_simple_seed_selection(self, clear_test_schema, project): + results = run_dbt(["seed", "--select", "seed_enabled"]) + assert len(results) == 1 + check_table_does_exist(project.adapter, "seed_enabled") + check_table_does_not_exist(project.adapter, "seed_disabled") + check_table_does_not_exist(project.adapter, "seed_tricky") + + @pytest.mark.skip( + reason=""" + Running all the tests in the same schema causes the tests to fail + as they all share the same schema across the tests + """ + ) + def test_simple_seed_exclude(self, clear_test_schema, project): + results = run_dbt(["seed", "--exclude", "seed_enabled"]) + assert len(results) == 1 + check_table_does_not_exist(project.adapter, "seed_enabled") + check_table_does_not_exist(project.adapter, "seed_disabled") + check_table_does_exist(project.adapter, "seed_tricky") + + +class TestSimpleSeedEnabledViaConfig__seed_exclude(BaseSimpleSeedEnabledViaConfig): + @pytest.fixture(scope="function") + def clear_test_schema(self, project): + yield + project.run_sql( + f"drop table if exists {project.database}.{project.test_schema}.seed_enabled" + ) + project.run_sql( + f"drop table if exists {project.database}.{project.test_schema}.seed_disabled" + ) + project.run_sql( + f"drop table if exists {project.database}.{project.test_schema}.seed_tricky" + ) + project.run_sql(f"drop view if exists {project.test_schema}.seed_enabled") + project.run_sql(f"drop view if exists {project.test_schema}.seed_disabled") + project.run_sql(f"drop view if exists {project.test_schema}.seed_tricky") + project.run_sql(f"drop schema if exists {project.test_schema}") + + @pytest.mark.skip( + reason=""" + Running all the tests in the same schema causes the tests to fail + as they all share the same schema across the tests + """ + ) + def test_simple_seed_with_disabled(self, clear_test_schema, project): + results = run_dbt(["seed"]) + assert len(results) == 2 + check_table_does_exist(project.adapter, "seed_enabled") + check_table_does_not_exist(project.adapter, "seed_disabled") + check_table_does_exist(project.adapter, "seed_tricky") + + @pytest.mark.skip( + reason=""" + Running all the tests in the same schema causes the tests to fail + as they all share the same schema across the tests + """ + ) + def test_simple_seed_selection(self, clear_test_schema, project): + results = run_dbt(["seed", "--select", "seed_enabled"]) + assert len(results) == 1 + check_table_does_exist(project.adapter, "seed_enabled") + check_table_does_not_exist(project.adapter, "seed_disabled") + check_table_does_not_exist(project.adapter, "seed_tricky") + + def test_simple_seed_exclude(self, clear_test_schema, project): + results = run_dbt(["seed", "--exclude", "seed_enabled"]) + assert len(results) == 1 + check_table_does_not_exist(project.adapter, "seed_enabled") + check_table_does_not_exist(project.adapter, "seed_disabled") + check_table_does_exist(project.adapter, "seed_tricky") diff --git a/tests/functional/adapter/test_snapshot.py b/tests/functional/adapter/dbt/test_simple_snapshot.py similarity index 63% rename from tests/functional/adapter/test_snapshot.py rename to tests/functional/adapter/dbt/test_simple_snapshot.py index 3e5d9257..52128f69 100644 --- a/tests/functional/adapter/test_snapshot.py +++ b/tests/functional/adapter/dbt/test_simple_snapshot.py @@ -1,11 +1,13 @@ from typing import Iterable -from dbt.tests.adapter.simple_snapshot.test_snapshot import BaseSimpleSnapshot, BaseSnapshotCheck -from dbt.tests.fixtures.project import TestProjInfo as ProjInfo +from dbt.tests.adapter.simple_snapshot.test_snapshot import BaseSimpleSnapshot +from dbt.tests.fixtures.project import TestProjInfo from dbt.tests.util import relation_from_name, run_dbt -def clone_table(project: ProjInfo, to_table: str, from_table: str, select: str, where: str = None): +def clone_table_sqlserver( + project: TestProjInfo, to_table: str, from_table: str, select: str, where: str = None +): """ Creates a new table based on another table in a dbt project @@ -15,8 +17,9 @@ def clone_table(project: ProjInfo, to_table: str, from_table: str, select: str, from_table: the name of the table, without a schema, to be cloned select: the selection clause to apply on `from_table`; defaults to all columns (*) where: the where clause to apply on `from_table`, if any; defaults to all records + + We override this for sqlserver as its using `select into` instead of `create table as select` """ - print(project) to_table_name = relation_from_name(project.adapter, to_table) from_table_name = relation_from_name(project.adapter, from_table) select_clause = select or "*" @@ -32,7 +35,7 @@ def clone_table(project: ProjInfo, to_table: str, from_table: str, select: str, project.run_sql(sql) -def add_column(project: ProjInfo, table: str, column: str, definition: str): +def add_column_sqlserver(project: TestProjInfo, table: str, column: str, definition: str): """ Applies updates to a table in a dbt project @@ -53,77 +56,19 @@ def add_column(project: ProjInfo, table: str, column: str, definition: str): project.run_sql(sql) -class SnapshotSQLServer: - def _assert_results( - self, - ids_with_current_snapshot_records: Iterable, - ids_with_closed_out_snapshot_records: Iterable, - ): - """ - All test cases are checked by considering whether a source record's id has a value - in `dbt_valid_to` in `snapshot`. Each id can fall into one of the following cases: - - - The id has only one record in `snapshot`; it has a value in `dbt_valid_to` - - the record was hard deleted in the source - - The id has only one record in `snapshot`; it does not have a value in `dbt_valid_to` - - the record was not updated in the source - - the record was updated in the source, but not in a way that is tracked - (e.g. via `strategy='check'`) - - The id has two records in `snapshot`; one has a value in `dbt_valid_to`, - the other does not - - the record was altered in the source in a way that is tracked - - the record was hard deleted and revived - - Note: Because of the third scenario, ids may show up in both arguments of this method. - - Args: - ids_with_current_snapshot_records: a list/set/etc. of ids which aren't end-dated - ids_with_closed_out_snapshot_records: a list/set/etc. of ids which are end-dated - """ - records = set( - self.get_snapshot_records( - """id, CASE WHEN dbt_valid_to is null then cast(1 as bit) - ELSE CAST(0 as bit) END as is_current""" - ) - ) - expected_records = set().union( - {(i, True) for i in ids_with_current_snapshot_records}, - {(i, False) for i in ids_with_closed_out_snapshot_records}, - ) - for record in records: - assert record in expected_records - +class TestSimpleSnapshot(BaseSimpleSnapshot): def create_fact_from_seed(self, where: str = None): # type: ignore - # overwrite clone table - clone_table(self.project, "fact", "seed", "*", where) + clone_table_sqlserver(self.project, "fact", "seed", "*", where) - def add_fact_column(self, column: str = None, definition: str = None): - add_column(self.project, "fact", column, definition) - - def test_column_selection_is_reflected_in_snapshot(self, project): - """ - Update the first 10 records on a non-tracked column. - Update the middle 10 records on a tracked column. - (hence records 6-10 are updated on both) - Show that all ids are current, and only the tracked column updates are reflected in - `snapshot`. - """ - self.update_fact_records( - {"last_name": "left(last_name, 3)"}, "id between 1 and 10" - ) # not tracked - self.update_fact_records({"email": "left(email, 3)"}, "id between 6 and 15") # tracked - run_dbt(["snapshot"]) - self._assert_results( - ids_with_current_snapshot_records=range(1, 21), - ids_with_closed_out_snapshot_records=range(6, 16), - ) + def add_fact_column(self, column: str = None, definition: str = None): # type: ignore + add_column_sqlserver(self.project, "fact", column, definition) def test_updates_are_captured_by_snapshot(self, project): """ Update the last 5 records. Show that all ids are current, but the last 5 reflect updates. """ self.update_fact_records( - {"updated_at": "DATEADD(day, 1, [updated_at])"}, "id between 16 and 20" + {"updated_at": "DATEADD(DAY, 1, updated_at)"}, "id between 16 and 20" ) run_dbt(["snapshot"]) self._assert_results( @@ -141,7 +86,7 @@ def test_new_column_captured_by_snapshot(self, project): self.update_fact_records( { "full_name": "first_name + ' ' + last_name", - "updated_at": "DATEADD(day, 1, [updated_at])", + "updated_at": "DATEADD(DAY, 1, updated_at)", }, "id between 11 and 20", ) @@ -151,10 +96,42 @@ def test_new_column_captured_by_snapshot(self, project): ids_with_closed_out_snapshot_records=range(11, 21), ) + def _assert_results( + self, + ids_with_current_snapshot_records: Iterable, + ids_with_closed_out_snapshot_records: Iterable, + ): + """ + All test cases are checked by considering whether a + source record's id has a value in `dbt_valid_to` + in `snapshot`. Each id can fall into one of the following cases: -class TestSnapshotSQLServer(SnapshotSQLServer, BaseSimpleSnapshot): - pass + - The id has only one record in `snapshot`; it has a value in `dbt_valid_to` + - the record was hard deleted in the source + - The id has only one record in `snapshot`; + it does not have a value in `dbt_valid_to` + - the record was not updated in the source + - the record was updated in the source, + but not in a way that is tracked (e.g. via `strategy='check'`) + - The id has two records in `snapshot`; + one has a value in `dbt_valid_to`, the other does not + - the record was altered in the source in a way that is tracked + - the record was hard deleted and revived + Note: Because of the third scenario, ids may show up in both arguments of this method. -class TestSnapshotCheckSQLServer(SnapshotSQLServer, BaseSnapshotCheck): - pass + Args: + ids_with_current_snapshot_records: a list/set/etc. of ids which aren't end-dated + ids_with_closed_out_snapshot_records: a list/set/etc. of ids which are end-dated + """ + records = set( + self.get_snapshot_records( + "id, CASE WHEN dbt_valid_to is null then 1 else 0 END as is_current" + ) + ) + expected_records = set().union( + {(i, 1) for i in ids_with_current_snapshot_records}, + {(i, 0) for i in ids_with_closed_out_snapshot_records}, + ) + for record in records: + assert record in expected_records diff --git a/tests/functional/adapter/dbt/test_utils.py b/tests/functional/adapter/dbt/test_utils.py new file mode 100644 index 00000000..b5ec8d30 --- /dev/null +++ b/tests/functional/adapter/dbt/test_utils.py @@ -0,0 +1,363 @@ +import pytest +from dbt.tests.adapter.utils import fixture_cast_bool_to_text, fixture_dateadd, fixture_listagg +from dbt.tests.adapter.utils.test_any_value import BaseAnyValue +from dbt.tests.adapter.utils.test_array_append import BaseArrayAppend +from dbt.tests.adapter.utils.test_array_concat import BaseArrayConcat +from dbt.tests.adapter.utils.test_array_construct import BaseArrayConstruct +from dbt.tests.adapter.utils.test_bool_or import BaseBoolOr +from dbt.tests.adapter.utils.test_cast import BaseCast +from dbt.tests.adapter.utils.test_cast_bool_to_text import BaseCastBoolToText +from dbt.tests.adapter.utils.test_concat import BaseConcat +from dbt.tests.adapter.utils.test_current_timestamp import ( + BaseCurrentTimestampAware, + BaseCurrentTimestampNaive, +) +from dbt.tests.adapter.utils.test_date import BaseDate +from dbt.tests.adapter.utils.test_date_spine import BaseDateSpine +from dbt.tests.adapter.utils.test_date_trunc import BaseDateTrunc +from dbt.tests.adapter.utils.test_dateadd import BaseDateAdd +from dbt.tests.adapter.utils.test_datediff import BaseDateDiff +from dbt.tests.adapter.utils.test_equals import BaseEquals +from dbt.tests.adapter.utils.test_escape_single_quotes import ( + BaseEscapeSingleQuotesBackslash, + BaseEscapeSingleQuotesQuote, +) +from dbt.tests.adapter.utils.test_except import BaseExcept +from dbt.tests.adapter.utils.test_generate_series import BaseGenerateSeries +from dbt.tests.adapter.utils.test_get_intervals_between import BaseGetIntervalsBetween +from dbt.tests.adapter.utils.test_get_powers_of_two import BaseGetPowersOfTwo +from dbt.tests.adapter.utils.test_hash import BaseHash +from dbt.tests.adapter.utils.test_intersect import BaseIntersect +from dbt.tests.adapter.utils.test_last_day import BaseLastDay +from dbt.tests.adapter.utils.test_length import BaseLength +from dbt.tests.adapter.utils.test_listagg import BaseListagg +from dbt.tests.adapter.utils.test_null_compare import BaseMixedNullCompare, BaseNullCompare +from dbt.tests.adapter.utils.test_position import BasePosition +from dbt.tests.adapter.utils.test_replace import BaseReplace +from dbt.tests.adapter.utils.test_right import BaseRight +from dbt.tests.adapter.utils.test_safe_cast import BaseSafeCast +from dbt.tests.adapter.utils.test_split_part import BaseSplitPart +from dbt.tests.adapter.utils.test_string_literal import BaseStringLiteral +from dbt.tests.adapter.utils.test_timestamps import BaseCurrentTimestamps +from dbt.tests.adapter.utils.test_validate_sql import BaseValidateSqlMethod + +# flake8: noqa: E501 + + +class TestAnyValue(BaseAnyValue): + pass + + +@pytest.mark.skip(reason="Not supported/Not implemented") +class TestArrayAppend(BaseArrayAppend): + pass + + +@pytest.mark.skip(reason="Not supported/Not implemented") +class TestArrayConcat(BaseArrayConcat): + pass + + +@pytest.mark.skip(reason="Not supported/Not implemented") +class TestArrayConstruct(BaseArrayConstruct): + pass + + +@pytest.mark.skip(reason="Not supported/Not implemented") +class TestBoolOr(BaseBoolOr): + pass + + +class TestCast(BaseCast): + pass + + +models__test_cast_bool_to_text_sql = """ +with data as ( + + select 0 as input, 'false' as expected union all + select 1 as input, 'true' as expected union all + select null as input, null as expected + +) + +select + + {{ cast_bool_to_text("input") }} as actual, + expected + +from data +""" + + +class TestCastBoolToText(BaseCastBoolToText): + @pytest.fixture(scope="class") + def models(self): + return { + "test_cast_bool_to_text.yml": fixture_cast_bool_to_text.models__test_cast_bool_to_text_yml, # noqa: E501 + "test_cast_bool_to_text.sql": self.interpolate_macro_namespace( + models__test_cast_bool_to_text_sql, "cast_bool_to_text" + ), + } + + +class TestConcat(BaseConcat): + pass + + +@pytest.mark.skip( + reason="Only should implement Aware or Naive. Opted for Naive to align with fabric." +) +class TestCurrentTimestampAware(BaseCurrentTimestampAware): + pass + + +class TestCurrentTimestampNaive(BaseCurrentTimestampNaive): + pass + + +@pytest.mark.skip(reason="Date spine relies on recursive CTES which are not supported.") +class TestDate(BaseDate): + pass + + +@pytest.mark.skip(reason="Date spine relies on recursive CTES which are not supported.") +class TestDateSpine(BaseDateSpine): + pass + + +class TestDateTrunc(BaseDateTrunc): + pass + + +class TestDateAdd(BaseDateAdd): + models__test_dateadd_sql = """ + with data as ( + + select * from {{ ref('data_dateadd') }} + + ) + + select + case + when datepart = 'hour' then cast({{ dateadd('hour', 'interval_length', 'from_time') }} as {{ api.Column.translate_type('timestamp') }}) + when datepart = 'day' then cast({{ dateadd('day', 'interval_length', 'from_time') }} as {{ api.Column.translate_type('timestamp') }}) + when datepart = 'month' then cast({{ dateadd('month', 'interval_length', 'from_time') }} as {{ api.Column.translate_type('timestamp') }}) + when datepart = 'year' then cast({{ dateadd('year', 'interval_length', 'from_time') }} as {{ api.Column.translate_type('timestamp') }}) + else null + end as actual, + result as expected + + from data + """ + + @pytest.fixture(scope="class") + def project_config_update(self): + return { + "name": "test", + # this is only needed for BigQuery, right? + # no harm having it here until/unless there's an adapter that doesn't support the 'timestamp' type + "seeds": { + "test": { + "data_dateadd": { + "+column_types": { + "from_time": "datetime2(6)", + "result": "datetime2(6)", + }, + }, + }, + }, + } + + @pytest.fixture(scope="class") + def seeds(self): + return {"data_dateadd.csv": fixture_dateadd.seeds__data_dateadd_csv} + + @pytest.fixture(scope="class") + def models(self): + return { + "test_dateadd.yml": fixture_dateadd.models__test_dateadd_yml, + "test_dateadd.sql": self.interpolate_macro_namespace( + self.models__test_dateadd_sql, "dateadd" + ), + } + + +class TestDateDiff(BaseDateDiff): + pass + + +class TestEquals(BaseEquals): + pass + + +class TestEscapeSingleQuotesQuote(BaseEscapeSingleQuotesQuote): + pass + + +@pytest.mark.skip(reason="SQLServer applies escaping with double of values") +class TestEscapeSingleQuotesBackslash(BaseEscapeSingleQuotesBackslash): + pass + + +class TestExcept(BaseExcept): + pass + + +@pytest.mark.skip( + reason="Only newer versions of SQLServer support Generate Series. Skipping for back compat" +) +class TestGenerateSeries(BaseGenerateSeries): + pass + + +class TestGetIntervalsBetween(BaseGetIntervalsBetween): + pass + + +class TestGetPowersOfTwo(BaseGetPowersOfTwo): + pass + + +class TestHash(BaseHash): + pass + + +class TestIntersect(BaseIntersect): + pass + + +class TestLastDay(BaseLastDay): + pass + + +class TestLength(BaseLength): + pass + + +seeds__data_listagg_output_csv = """group_col,expected,version +1,"a_|_b_|_c",bottom_ordered +2,"1_|_a_|_p",bottom_ordered +3,"g_|_g_|_g",bottom_ordered +1,"c_|_b_|_a",reverse_order +2,"p_|_a_|_1",reverse_order +3,"g_|_g_|_g",reverse_order +3,"g, g, g",comma_whitespace_unordered +""" + + +models__test_listagg_sql = """ +with data as ( + + select * from {{ ref('data_listagg') }} + +), + +data_output as ( + + select * from {{ ref('data_listagg_output') }} + +), + +calculate as ( + + select + group_col, + {{ listagg('string_text', "'_|_'", "order by order_col") }} as actual, + 'bottom_ordered' as version + from data + group by group_col + + union all + + select + group_col, + {{ listagg('string_text', "'_|_'", "order by order_col desc", 2) }} as actual, + 'reverse_order' as version + from data + group by group_col + + union all + + select + group_col, + {{ listagg('string_text', "', '") }} as actual, + 'comma_whitespace_unordered' as version + from data + where group_col = 3 + group by group_col + +) + +select + calculate.actual, + data_output.expected +from calculate +left join data_output +on calculate.group_col = data_output.group_col +and calculate.version = data_output.version +""" + + +class TestListagg(BaseListagg): + @pytest.fixture(scope="class") + def seeds(self): + return { + "data_listagg.csv": fixture_listagg.seeds__data_listagg_csv, + "data_listagg_output.csv": seeds__data_listagg_output_csv, + } + + @pytest.fixture(scope="class") + def models(self): + return { + "test_listagg.yml": fixture_listagg.models__test_listagg_yml, + "test_listagg.sql": self.interpolate_macro_namespace( + models__test_listagg_sql, "listagg" + ), + } + + +class TestMixedNullCompare(BaseMixedNullCompare): + pass + + +class TestNullCompare(BaseNullCompare): + pass + + +class TestPosition(BasePosition): + pass + + +class TestReplace(BaseReplace): + pass + + +class TestRight(BaseRight): + pass + + +class TestSafeCast(BaseSafeCast): + pass + + +class TestSplitPart(BaseSplitPart): + pass + + +class TestStringLiteral(BaseStringLiteral): + pass + + +@pytest.mark.skip( + reason=""" + comment here about why this is skipped. + https://github.com/dbt-labs/dbt-adapters/blob/f1987d4313cc94bac9906963dff1337ee0bffbc6/dbt/include/global_project/macros/adapters/timestamps.sql#L39 + """ +) +class TestCurrentTimestamps(BaseCurrentTimestamps): + pass + + +class TestValidateSqlMethod(BaseValidateSqlMethod): + pass diff --git a/tests/functional/adapter/mssql/test_index.py b/tests/functional/adapter/mssql/test_index.py new file mode 100644 index 00000000..37328acd --- /dev/null +++ b/tests/functional/adapter/mssql/test_index.py @@ -0,0 +1,145 @@ +import pytest +from dbt.tests.util import get_connection, run_dbt + +# flake8: noqa: E501 + +index_seed_csv = """id_col,data,secondary_data,tertiary_data +1,'a'",122,20 +""" + +index_schema_base_yml = """ +version: 2 +seeds: + - name: raw_data + config: + column_types: + id_col: integer + data: nvarchar(20) + secondary_data: integer + tertiary_data: bigint +""" + +model_yml = """ +version: 2 +models: + - name: index_model + - name: index_ccs_model +""" + +model_sql = """ +{{ + config({ + "materialized": 'table', + "as_columnstore": False, + "post-hook": [ + "{{ create_clustered_index(columns = ['id_col'], unique=True) }}", + "{{ create_nonclustered_index(columns = ['data']) }}", + "{{ create_nonclustered_index(columns = ['secondary_data'], includes = ['tertiary_data']) }}", + ] + }) +}} + select * from {{ ref('raw_data') }} +""" + +model_sql_ccs = """ +{{ + config({ + "materialized": 'table', + "post-hook": [ + "{{ create_nonclustered_index(columns = ['data']) }}", + "{{ create_nonclustered_index(columns = ['secondary_data'], includes = ['tertiary_data']) }}", + ] + }) +}} + select * from {{ ref('raw_data') }} +""" + +base_validation = """ +with base_query AS ( +select i.[name] as index_name, + substring(column_names, 1, len(column_names)-1) as [columns], + case when i.[type] = 1 then 'Clustered index' + when i.[type] = 2 then 'Nonclustered unique index' + when i.[type] = 3 then 'XML index' + when i.[type] = 4 then 'Spatial index' + when i.[type] = 5 then 'Clustered columnstore index' + when i.[type] = 6 then 'Nonclustered columnstore index' + when i.[type] = 7 then 'Nonclustered hash index' + end as index_type, + case when i.is_unique = 1 then 'Unique' + else 'Not unique' end as [unique], + schema_name(t.schema_id) + '.' + t.[name] as table_view, + case when t.[type] = 'U' then 'Table' + when t.[type] = 'V' then 'View' + end as [object_type], + s.name as schema_name +from sys.objects t + inner join sys.schemas s + on + t.schema_id = s.schema_id + inner join sys.indexes i + on t.object_id = i.object_id + cross apply (select col.[name] + ', ' + from sys.index_columns ic + inner join sys.columns col + on ic.object_id = col.object_id + and ic.column_id = col.column_id + where ic.object_id = t.object_id + and ic.index_id = i.index_id + order by key_ordinal + for xml path ('') ) D (column_names) +where t.is_ms_shipped <> 1 +and index_id > 0 +) +""" + +index_count = ( + base_validation + + """ +select + index_type, + count(*) index_count +from + base_query +WHERE + schema_name='{schema_name}' +group by index_type +""" +) + + +class TestIndex: + @pytest.fixture(scope="class") + def project_config_update(self): + return {"name": "generic_tests"} + + @pytest.fixture(scope="class") + def seeds(self): + return { + "raw_data.csv": index_seed_csv, + "schema.yml": index_schema_base_yml, + } + + @pytest.fixture(scope="class") + def models(self): + return { + "index_model.sql": model_sql, + "index_ccs_model.sql": model_sql_ccs, + "schema.yml": model_yml, + } + + def test_create_index(self, project): + run_dbt(["seed"]) + run_dbt(["run"]) + + with get_connection(project.adapter): + result, table = project.adapter.execute( + index_count.format(schema_name=project.created_schemas[0]), fetch=True + ) + schema_dict = {_[0]: _[1] for _ in table.rows} + expected = { + "Clustered columnstore index": 1, + "Clustered index": 1, + "Nonclustered unique index": 4, + } + assert schema_dict == expected diff --git a/tests/functional/adapter/test_provision_users.py b/tests/functional/adapter/mssql/test_provision_users.py similarity index 100% rename from tests/functional/adapter/test_provision_users.py rename to tests/functional/adapter/mssql/test_provision_users.py diff --git a/tests/functional/adapter/mssql/test_temp_relation_cleanup.py b/tests/functional/adapter/mssql/test_temp_relation_cleanup.py new file mode 100644 index 00000000..5ed2660d --- /dev/null +++ b/tests/functional/adapter/mssql/test_temp_relation_cleanup.py @@ -0,0 +1,58 @@ +import pytest +from dbt.tests.util import get_connection, run_dbt + +table_model = """ +{{ +config({ + "materialized": 'table' +}) +}} + +SELECT 1 as data +""" + +model_yml = """ +version: 2 +models: + - name: table_model +""" + +validation_sql = """ +SELECT + * +FROM + {database}.INFORMATION_SCHEMA.TABLES +WHERE + TABLE_SCHEMA = '{schema}' + AND + TABLE_NAME LIKE '%__dbt_tmp_vw' +""" + + +class TestTempRelationCleanup: + """ + This tests to validate that the temporary relations, + created by the `create_table` statement is cleaned up after a set of runs. + """ + + view_name = "__dbt_tmp_vw" + + @pytest.fixture(scope="class") + def models(self): + return { + "table_model.sql": table_model, + "schema.yml": model_yml, + } + + def test_drops_temp_view_object(self, project): + run_dbt(["seed"]) + run_dbt(["run"]) + + with get_connection(project.adapter): + result, table = project.adapter.execute( + validation_sql.format( + database=project.database, schema=project.created_schemas[0] + ), + fetch=True, + ) + assert len(table.rows) == 0 diff --git a/tests/functional/adapter/mssql/test_xml_index.py b/tests/functional/adapter/mssql/test_xml_index.py new file mode 100644 index 00000000..a53b5a8f --- /dev/null +++ b/tests/functional/adapter/mssql/test_xml_index.py @@ -0,0 +1,49 @@ +import pytest + +xml_seed = """id_col,xml_data +1,1" +""" + +xml_schema_base_yml = """ +version: 2 +seeds: + - name: xml_data + config: + column_types: + id_col: integer + xml_data: xml +""" + +xml_model_yml = """ +version: 2 +models: + - name: xml_model + columns: + - name: id + - name: xml_data +""" + +xml_sql = """ +{{ config(materialized="table") }} + select * from {{ ref('xml_data') }} +""" + + +class TestIndex: + @pytest.fixture(scope="class") + def project_config_update(self): + return {"name": "generic_tests"} + + @pytest.fixture(scope="class") + def seeds(self): + return { + "xml_data.csv": xml_seed, + "schema.yml": xml_schema_base_yml, + } + + @pytest.fixture(scope="class") + def models(self): + return { + "xml_model.sql": xml_sql, + "schema.yml": xml_model_yml, + } diff --git a/tests/functional/adapter/test_aliases.py b/tests/functional/adapter/test_aliases.py deleted file mode 100644 index bacd2d1e..00000000 --- a/tests/functional/adapter/test_aliases.py +++ /dev/null @@ -1,32 +0,0 @@ -import pytest -from dbt.tests.adapter.aliases.fixtures import MACROS__EXPECT_VALUE_SQL -from dbt.tests.adapter.aliases.test_aliases import ( - BaseAliasErrors, - BaseAliases, - BaseSameAliasDifferentDatabases, - BaseSameAliasDifferentSchemas, -) - - -class TestAliasesSQLServer(BaseAliases): - @pytest.fixture(scope="class") - def macros(self): - return {"expect_value.sql": MACROS__EXPECT_VALUE_SQL} - - -class TestAliasErrorsSQLServer(BaseAliasErrors): - @pytest.fixture(scope="class") - def macros(self): - return {"expect_value.sql": MACROS__EXPECT_VALUE_SQL} - - -class TestSameAliasDifferentSchemasSQLServer(BaseSameAliasDifferentSchemas): - @pytest.fixture(scope="class") - def macros(self): - return {"expect_value.sql": MACROS__EXPECT_VALUE_SQL} - - -class TestSameAliasDifferentDatabasesSQLServer(BaseSameAliasDifferentDatabases): - @pytest.fixture(scope="class") - def macros(self): - return {"expect_value.sql": MACROS__EXPECT_VALUE_SQL} diff --git a/tests/functional/adapter/test_basic.py b/tests/functional/adapter/test_basic.py deleted file mode 100644 index f55e8e0b..00000000 --- a/tests/functional/adapter/test_basic.py +++ /dev/null @@ -1,76 +0,0 @@ -import pytest -from dbt.tests.adapter.basic.files import incremental_not_schema_change_sql -from dbt.tests.adapter.basic.test_adapter_methods import BaseAdapterMethod -from dbt.tests.adapter.basic.test_base import BaseSimpleMaterializations -from dbt.tests.adapter.basic.test_empty import BaseEmpty -from dbt.tests.adapter.basic.test_ephemeral import BaseEphemeral -from dbt.tests.adapter.basic.test_generic_tests import BaseGenericTests -from dbt.tests.adapter.basic.test_incremental import ( - BaseIncremental, - BaseIncrementalNotSchemaChange, -) -from dbt.tests.adapter.basic.test_singular_tests import BaseSingularTests -from dbt.tests.adapter.basic.test_singular_tests_ephemeral import BaseSingularTestsEphemeral -from dbt.tests.adapter.basic.test_snapshot_check_cols import BaseSnapshotCheckCols -from dbt.tests.adapter.basic.test_snapshot_timestamp import BaseSnapshotTimestamp -from dbt.tests.adapter.basic.test_table_materialization import BaseTableMaterialization -from dbt.tests.adapter.basic.test_validate_connection import BaseValidateConnection - - -class TestSimpleMaterializationsSQLServer(BaseSimpleMaterializations): - pass - - -class TestSingularTestsSQLServer(BaseSingularTests): - pass - - -@pytest.mark.skip(reason="ephemeral not supported") -class TestSingularTestsEphemeralSQLServer(BaseSingularTestsEphemeral): - pass - - -class TestEmptySQLServer(BaseEmpty): - pass - - -class TestEphemeralSQLServer(BaseEphemeral): - pass - - -class TestIncrementalSQLServer(BaseIncremental): - pass - - -class TestIncrementalNotSchemaChangeSQLServer(BaseIncrementalNotSchemaChange): - @pytest.fixture(scope="class") - def models(self): - return { - "incremental_not_schema_change.sql": incremental_not_schema_change_sql.replace( - "||", "+" - ) - } - - -class TestGenericTestsSQLServer(BaseGenericTests): - pass - - -class TestSnapshotCheckColsSQLServer(BaseSnapshotCheckCols): - pass - - -class TestSnapshotTimestampSQLServer(BaseSnapshotTimestamp): - pass - - -class TestBaseCachingSQLServer(BaseAdapterMethod): - pass - - -class TestValidateConnectionSQLServer(BaseValidateConnection): - pass - - -class TestTableMaterializationSQLServer(BaseTableMaterialization): - ... diff --git a/tests/functional/adapter/test_changing_relation_type.py b/tests/functional/adapter/test_changing_relation_type.py deleted file mode 100644 index f135cf8f..00000000 --- a/tests/functional/adapter/test_changing_relation_type.py +++ /dev/null @@ -1,7 +0,0 @@ -import pytest -from dbt.tests.adapter.relations.test_changing_relation_type import BaseChangeRelationTypeValidator - - -@pytest.mark.skip(reason="CTAS is not supported without a underlying table definition.") -class TestChangeRelationTypesSQLServer(BaseChangeRelationTypeValidator): - pass diff --git a/tests/functional/adapter/test_concurrency.py b/tests/functional/adapter/test_concurrency.py deleted file mode 100644 index f846f07a..00000000 --- a/tests/functional/adapter/test_concurrency.py +++ /dev/null @@ -1,36 +0,0 @@ -from dbt.tests.adapter.concurrency.test_concurrency import BaseConcurrency, seeds__update_csv -from dbt.tests.util import ( - check_relations_equal, - check_table_does_not_exist, - rm_file, - run_dbt, - run_dbt_and_capture, - write_file, -) - - -class TestConcurenncySQLServer(BaseConcurrency): - def test_concurrency(self, project): - run_dbt(["seed", "--select", "seed"]) - results = run_dbt(["run"], expect_pass=False) - assert len(results) == 7 - check_relations_equal(project.adapter, ["seed", "view_model"]) - check_relations_equal(project.adapter, ["seed", "dep"]) - check_relations_equal(project.adapter, ["seed", "table_a"]) - check_relations_equal(project.adapter, ["seed", "table_b"]) - check_table_does_not_exist(project.adapter, "invalid") - check_table_does_not_exist(project.adapter, "skip") - - rm_file(project.project_root, "seeds", "seed.csv") - write_file(seeds__update_csv, project.project_root, "seeds", "seed.csv") - - results, output = run_dbt_and_capture(["run"], expect_pass=False) - assert len(results) == 7 - check_relations_equal(project.adapter, ["seed", "view_model"]) - check_relations_equal(project.adapter, ["seed", "dep"]) - check_relations_equal(project.adapter, ["seed", "table_a"]) - check_relations_equal(project.adapter, ["seed", "table_b"]) - check_table_does_not_exist(project.adapter, "invalid") - check_table_does_not_exist(project.adapter, "skip") - - assert "PASS=5 WARN=0 ERROR=1 SKIP=1 TOTAL=7" in output diff --git a/tests/functional/adapter/test_data_types.py b/tests/functional/adapter/test_data_types.py deleted file mode 100644 index 28d67889..00000000 --- a/tests/functional/adapter/test_data_types.py +++ /dev/null @@ -1,59 +0,0 @@ -import pytest -from dbt.tests.adapter.utils.data_types.test_type_bigint import BaseTypeBigInt -from dbt.tests.adapter.utils.data_types.test_type_boolean import BaseTypeBoolean -from dbt.tests.adapter.utils.data_types.test_type_float import BaseTypeFloat -from dbt.tests.adapter.utils.data_types.test_type_int import BaseTypeInt -from dbt.tests.adapter.utils.data_types.test_type_numeric import BaseTypeNumeric -from dbt.tests.adapter.utils.data_types.test_type_string import BaseTypeString -from dbt.tests.adapter.utils.data_types.test_type_timestamp import ( - BaseTypeTimestamp, - seeds__expected_csv, -) - - -@pytest.mark.skip(reason="SQL Server shows 'numeric' if you don't explicitly cast it to bigint") -class TestTypeBigIntSQLServer(BaseTypeBigInt): - pass - - -class TestTypeFloatSQLServer(BaseTypeFloat): - pass - - -class TestTypeIntSQLServer(BaseTypeInt): - pass - - -class TestTypeNumericSQLServer(BaseTypeNumeric): - pass - - -class TestTypeStringSQLServer(BaseTypeString): - def assert_columns_equal(self, project, expected_cols, actual_cols): - # ignore the size of the varchar since we do - # an optimization to not use varchar(max) all the time - assert ( - expected_cols[:-1] == actual_cols[:-1] - ), f"Type difference detected: {expected_cols} vs. {actual_cols}" - - -class TestTypeTimestampSQLServer(BaseTypeTimestamp): - @pytest.fixture(scope="class") - def seeds(self): - seeds__expected_yml = """ -version: 2 -seeds: - - name: expected - config: - column_types: - timestamp_col: "datetime2" - """ - - return { - "expected.csv": seeds__expected_csv, - "expected.yml": seeds__expected_yml, - } - - -class TestTypeBooleanSQLServer(BaseTypeBoolean): - pass diff --git a/tests/functional/adapter/test_debug.py b/tests/functional/adapter/test_debug.py deleted file mode 100644 index 7e5457cb..00000000 --- a/tests/functional/adapter/test_debug.py +++ /dev/null @@ -1,67 +0,0 @@ -import os -import re - -import pytest -import yaml -from dbt.cli.exceptions import DbtUsageException -from dbt.tests.adapter.dbt_debug.test_dbt_debug import BaseDebug, BaseDebugProfileVariable -from dbt.tests.util import run_dbt - - -class TestDebugSQLServer(BaseDebug): - def test_ok(self, project): - run_dbt(["debug"]) - assert "ERROR" not in self.capsys.readouterr().out - - def test_nopass(self, project): - run_dbt(["debug", "--target", "nopass"], expect_pass=False) - self.assertGotValue(re.compile(r"\s+profiles\.yml file"), "ERROR invalid") - - def test_wronguser(self, project): - run_dbt(["debug", "--target", "wronguser"], expect_pass=False) - self.assertGotValue(re.compile(r"\s+Connection test"), "ERROR") - - def test_empty_target(self, project): - run_dbt(["debug", "--target", "none_target"], expect_pass=False) - self.assertGotValue(re.compile(r"\s+output 'none_target'"), "misconfigured") - - -class TestDebugProfileVariableSQLServer(BaseDebugProfileVariable): - pass - - -class TestDebugInvalidProjectSQLServer(BaseDebug): - def test_empty_project(self, project): - with open("dbt_project.yml", "w") as f: # noqa: F841 - pass - - run_dbt(["debug", "--profile", "test"], expect_pass=False) - splitout = self.capsys.readouterr().out.split("\n") - self.check_project(splitout) - - def test_badproject(self, project): - update_project = {"invalid-key": "not a valid key so this is bad project"} - - with open("dbt_project.yml", "w") as f: - yaml.safe_dump(update_project, f) - - run_dbt(["debug", "--profile", "test"], expect_pass=False) - splitout = self.capsys.readouterr().out.split("\n") - self.check_project(splitout) - - def test_not_found_project(self, project): - with pytest.raises(DbtUsageException) as dbt_exeption: - run_dbt(["debug", "--project-dir", "nopass"], expect_pass=False) - dbt_exeption = dbt_exeption - splitout = self.capsys.readouterr().out.split("\n") - self.check_project(splitout, msg="ERROR not found") - - def test_invalid_project_outside_current_dir(self, project): - # create a dbt_project.yml - project_config = {"invalid-key": "not a valid key in this project"} - os.makedirs("custom", exist_ok=True) - with open("custom/dbt_project.yml", "w") as f: - yaml.safe_dump(project_config, f, default_flow_style=True) - run_dbt(["debug", "--project-dir", "custom"], expect_pass=False) - splitout = self.capsys.readouterr().out.split("\n") - self.check_project(splitout) diff --git a/tests/functional/adapter/test_docs.py b/tests/functional/adapter/test_docs.py deleted file mode 100644 index 1130295e..00000000 --- a/tests/functional/adapter/test_docs.py +++ /dev/null @@ -1,91 +0,0 @@ -import os - -import pytest -from dbt.tests.adapter.basic.expected_catalog import ( - base_expected_catalog, - expected_references_catalog, - no_stats, -) -from dbt.tests.adapter.basic.test_docs_generate import ( - BaseDocsGenerate, - BaseDocsGenReferences, - ref_models__docs_md, - ref_models__ephemeral_copy_sql, - ref_models__schema_yml, - ref_sources__schema_yml, -) - - -class TestDocsGenerateSQLServer(BaseDocsGenerate): - @staticmethod - @pytest.fixture(scope="class") - def dbt_profile_target_update(): - return {"schema_authorization": "{{ env_var('DBT_TEST_USER_1') }}"} - - @pytest.fixture(scope="class") - def expected_catalog(self, project): - return base_expected_catalog( - project, - role=os.getenv("DBT_TEST_USER_1"), - id_type="int", - text_type="varchar", - time_type="datetime2", - view_type="VIEW", - table_type="BASE TABLE", - model_stats=no_stats(), - ) - - -class TestDocsGenReferencesSQLServer(BaseDocsGenReferences): - @staticmethod - @pytest.fixture(scope="class") - def dbt_profile_target_update(): - return {"schema_authorization": "{{ env_var('DBT_TEST_USER_1') }}"} - - @pytest.fixture(scope="class") - def expected_catalog(self, project): - return expected_references_catalog( - project, - role=os.getenv("DBT_TEST_USER_1"), - id_type="int", - text_type="varchar", - time_type="datetime2", - bigint_type="int", - view_type="VIEW", - table_type="BASE TABLE", - model_stats=no_stats(), - ) - - @pytest.fixture(scope="class") - def models(self): - ref_models__ephemeral_summary_sql_no_order_by = """ - {{ - config( - materialized = "table" - ) - }} - - select first_name, count(*) as ct from {{ref('ephemeral_copy')}} - group by first_name - """ - - ref_models__view_summary_sql_no_order_by = """ - {{ - config( - materialized = "view" - ) - }} - - select first_name, ct from {{ref('ephemeral_summary')}} - """ - - return { - "schema.yml": ref_models__schema_yml, - "sources.yml": ref_sources__schema_yml, - # order by not allowed in VIEWS - "view_summary.sql": ref_models__view_summary_sql_no_order_by, - # order by not allowed in CTEs - "ephemeral_summary.sql": ref_models__ephemeral_summary_sql_no_order_by, - "ephemeral_copy.sql": ref_models__ephemeral_copy_sql, - "docs.md": ref_models__docs_md, - } diff --git a/tests/functional/adapter/test_ephemeral.py b/tests/functional/adapter/test_ephemeral.py deleted file mode 100644 index 2c11d6d6..00000000 --- a/tests/functional/adapter/test_ephemeral.py +++ /dev/null @@ -1,26 +0,0 @@ -import pytest -from dbt.tests.adapter.ephemeral.test_ephemeral import ( - BaseEphemeral, - ephemeral_errors__base__base_copy_sql, - ephemeral_errors__base__base_sql, - ephemeral_errors__dependent_sql, -) -from dbt.tests.util import run_dbt - - -class TestEphemeralErrorHandling(BaseEphemeral): - @pytest.fixture(scope="class") - def models(self): - return { - "dependent.sql": ephemeral_errors__dependent_sql, - "base": { - "base.sql": ephemeral_errors__base__base_sql, - "base_copy.sql": ephemeral_errors__base__base_copy_sql, - }, - } - - def test_ephemeral_error_handling(self, project): - results = run_dbt(["run"], expect_pass=False) - assert len(results) == 1 - assert results[0].status == "skipped" - assert "Compilation Error" in results[0].message diff --git a/tests/functional/adapter/test_grants.py b/tests/functional/adapter/test_grants.py deleted file mode 100644 index 1f297dbd..00000000 --- a/tests/functional/adapter/test_grants.py +++ /dev/null @@ -1,63 +0,0 @@ -from dbt.tests.adapter.grants.test_incremental_grants import BaseIncrementalGrants -from dbt.tests.adapter.grants.test_invalid_grants import BaseInvalidGrants -from dbt.tests.adapter.grants.test_model_grants import BaseModelGrants -from dbt.tests.adapter.grants.test_seed_grants import BaseSeedGrants -from dbt.tests.adapter.grants.test_snapshot_grants import ( - BaseSnapshotGrants, - user2_snapshot_schema_yml, -) -from dbt.tests.util import get_manifest, run_dbt, run_dbt_and_capture, write_file - - -class TestIncrementalGrantsSQLServer(BaseIncrementalGrants): - pass - - -class TestInvalidGrantsSQLServer(BaseInvalidGrants): - def grantee_does_not_exist_error(self): - return "Cannot find the user" - - def privilege_does_not_exist_error(self): - return "Incorrect syntax near" - - -class TestModelGrantsSQLServer(BaseModelGrants): - pass - - -class TestSeedGrantsSQLServer(BaseSeedGrants): - pass - - -class TestSnapshotGrantsSQLServer(BaseSnapshotGrants): - def test_snapshot_grants(self, project, get_test_users): - test_users = get_test_users - select_privilege_name = self.privilege_grantee_name_overrides()["select"] - - # run the snapshot - results = run_dbt(["snapshot"]) - assert len(results) == 1 - manifest = get_manifest(project.project_root) - snapshot_id = "snapshot.test.my_snapshot" - snapshot = manifest.nodes[snapshot_id] - expected = {select_privilege_name: [test_users[0]]} - assert snapshot.config.grants == expected - self.assert_expected_grants_match_actual(project, "my_snapshot", expected) - - # run it again, nothing should have changed - # we do expect to see the grant again. - # dbt selects into a temporary table, drops existing, selects into original table name - # this means we need to grant select again, so we will see the grant again - (results, log_output) = run_dbt_and_capture(["--debug", "snapshot"]) - assert len(results) == 1 - assert "revoke " not in log_output - assert "grant " in log_output - self.assert_expected_grants_match_actual(project, "my_snapshot", expected) - - # change the grantee, assert it updates - updated_yaml = self.interpolate_name_overrides(user2_snapshot_schema_yml) - write_file(updated_yaml, project.project_root, "snapshots", "schema.yml") - (results, log_output) = run_dbt_and_capture(["--debug", "snapshot"]) - assert len(results) == 1 - expected = {select_privilege_name: [test_users[1]]} - self.assert_expected_grants_match_actual(project, "my_snapshot", expected) diff --git a/tests/functional/adapter/test_incremental.py b/tests/functional/adapter/test_incremental.py deleted file mode 100644 index 0c3356dc..00000000 --- a/tests/functional/adapter/test_incremental.py +++ /dev/null @@ -1,125 +0,0 @@ -import pytest -from dbt.tests.adapter.incremental.fixtures import ( - _MODELS__A, - _MODELS__INCREMENTAL_APPEND_NEW_COLUMNS, - _MODELS__INCREMENTAL_APPEND_NEW_COLUMNS_REMOVE_ONE, - _MODELS__INCREMENTAL_APPEND_NEW_COLUMNS_REMOVE_ONE_TARGET, - _MODELS__INCREMENTAL_APPEND_NEW_COLUMNS_TARGET, - _MODELS__INCREMENTAL_FAIL, - _MODELS__INCREMENTAL_IGNORE_TARGET, - _MODELS__INCREMENTAL_SYNC_ALL_COLUMNS, - _MODELS__INCREMENTAL_SYNC_REMOVE_ONLY, -) -from dbt.tests.adapter.incremental.test_incremental_on_schema_change import ( - BaseIncrementalOnSchemaChange, -) -from dbt.tests.adapter.incremental.test_incremental_predicates import BaseIncrementalPredicates -from dbt.tests.adapter.incremental.test_incremental_unique_id import BaseIncrementalUniqueKey - -_MODELS__INCREMENTAL_IGNORE = """ -{{ - config( - materialized='incremental', - unique_key='id', - on_schema_change='ignore' - ) -}} - -WITH source_data AS (SELECT * FROM {{ ref('model_a') }} ) - -{% if is_incremental() %} - -SELECT - id, - field1, - field2, - field3, - field4 -FROM source_data -WHERE id NOT IN (SELECT id from {{ this }} ) - -{% else %} - -SELECT TOP 3 id, field1, field2 FROM source_data - -{% endif %} -""" - -_MODELS__INCREMENTAL_SYNC_REMOVE_ONLY_TARGET = """ -{{ - config(materialized='table') -}} - -with source_data as ( - - select * from {{ ref('model_a') }} - -) - -{% set string_type = dbt.type_string() %} - -select id - ,cast(field1 as {{string_type}}) as field1 - -from source_data -""" - -_MODELS__INCREMENTAL_SYNC_ALL_COLUMNS_TARGET = """ -{{ - config(materialized='table') -}} - -with source_data as ( - - select * from {{ ref('model_a') }} - -) - -{% set string_type = dbt.type_string() %} - -select id - ,cast(field1 as {{string_type}}) as field1 - --,field2 - ,cast(case when id <= 3 then null else field3 end as {{string_type}}) as field3 - ,cast(case when id <= 3 then null else field4 end as {{string_type}}) as field4 - -from source_data -""" - - -class TestBaseIncrementalUniqueKeySQLServer(BaseIncrementalUniqueKey): - pass - - -class TestIncrementalOnSchemaChangeSQLServer(BaseIncrementalOnSchemaChange): - @pytest.fixture(scope="class") - def models(self): - return { - "incremental_sync_remove_only.sql": _MODELS__INCREMENTAL_SYNC_REMOVE_ONLY, - "incremental_ignore.sql": _MODELS__INCREMENTAL_IGNORE, - "incremental_sync_remove_only_target.sql": _MODELS__INCREMENTAL_SYNC_REMOVE_ONLY_TARGET, # noqa: E501 - "incremental_ignore_target.sql": _MODELS__INCREMENTAL_IGNORE_TARGET, - "incremental_fail.sql": _MODELS__INCREMENTAL_FAIL, - "incremental_sync_all_columns.sql": _MODELS__INCREMENTAL_SYNC_ALL_COLUMNS, - "incremental_append_new_columns_remove_one.sql": _MODELS__INCREMENTAL_APPEND_NEW_COLUMNS_REMOVE_ONE, # noqa: E501 - "model_a.sql": _MODELS__A, - "incremental_append_new_columns_target.sql": _MODELS__INCREMENTAL_APPEND_NEW_COLUMNS_TARGET, # noqa: E501 - "incremental_append_new_columns.sql": _MODELS__INCREMENTAL_APPEND_NEW_COLUMNS, - "incremental_sync_all_columns_target.sql": _MODELS__INCREMENTAL_SYNC_ALL_COLUMNS_TARGET, # noqa: E501 - "incremental_append_new_columns_remove_one_target.sql": _MODELS__INCREMENTAL_APPEND_NEW_COLUMNS_REMOVE_ONE_TARGET, # noqa: E501 - } - - -class TestIncrementalPredicatesDeleteInsertSQLServer(BaseIncrementalPredicates): - pass - - -class TestPredicatesDeleteInsertSQLServer(BaseIncrementalPredicates): - @pytest.fixture(scope="class") - def project_config_update(self): - return { - "models": { - "+predicates": ["id != 2"], - "+incremental_strategy": "delete+insert", - } - } diff --git a/tests/functional/adapter/test_new_project.py b/tests/functional/adapter/test_new_project.py deleted file mode 100644 index b5eef440..00000000 --- a/tests/functional/adapter/test_new_project.py +++ /dev/null @@ -1,88 +0,0 @@ -import pytest -from dbt.tests.util import run_dbt - -schema_yml = """ - -version: 2 - -models: - - name: my_first_dbt_model - description: "A starter dbt model" - columns: - - name: id - description: "The primary key for this table" - tests: - - unique - - - name: my_second_dbt_model - description: "A starter dbt model" - columns: - - name: id - description: "The primary key for this table" - tests: - - unique - - not_null -""" - -my_first_dbt_model_sql = """ -/* - Welcome to your first dbt model! - Did you know that you can also configure models directly within SQL files? - This will override configurations stated in dbt_project.yml - - Try changing "table" to "view" below -*/ - -{{ config(materialized='table') }} - -with source_data as ( - - select 1 as id - union all - select null as id - -) - -select * -from source_data - -/* - Uncomment the line below to remove records with null `id` values -*/ - --- where id is not null -""" - -my_second_dbt_model_sql = """ --- Use the `ref` function to select from other models - -select * -from {{ ref('my_first_dbt_model') }} -where id = 1 -""" - - -class TestNewProjectSQLServer: - @pytest.fixture(scope="class") - def project_config_update(self): - return {"name": "my_new_project"} - - @pytest.fixture(scope="class") - def models(self): - return { - "my_first_dbt_model.sql": my_first_dbt_model_sql, - "my_second_dbt_model.sql": my_second_dbt_model_sql, - "schema.yml": schema_yml, - } - - def test_new_project(self, project): - results = run_dbt(["build"]) - assert len(results) == 5 - - def test_run_same_model_multiple_times(self, project): - results = run_dbt(["run"]) - assert len(results) == 2 - - for i in range(10): - run_dbt(["run", "-s", "my_second_dbt_model"]) - assert len(results) == 2 diff --git a/tests/functional/adapter/test_query_comment.py b/tests/functional/adapter/test_query_comment.py deleted file mode 100644 index 434d93d1..00000000 --- a/tests/functional/adapter/test_query_comment.py +++ /dev/null @@ -1,32 +0,0 @@ -from dbt.tests.adapter.query_comment.test_query_comment import ( - BaseEmptyQueryComments, - BaseMacroArgsQueryComments, - BaseMacroInvalidQueryComments, - BaseMacroQueryComments, - BaseNullQueryComments, - BaseQueryComments, -) - - -class TestQueryCommentsSQLServer(BaseQueryComments): - pass - - -class TestMacroQueryCommentsSQLServer(BaseMacroQueryComments): - pass - - -class TestMacroArgsQueryCommentsSQLServer(BaseMacroArgsQueryComments): - pass - - -class TestMacroInvalidQueryCommentsSQLServer(BaseMacroInvalidQueryComments): - pass - - -class TestNullQueryCommentsSQLServer(BaseNullQueryComments): - pass - - -class TestEmptyQueryCommentsSQLServer(BaseEmptyQueryComments): - pass diff --git a/tests/functional/adapter/test_schema.py b/tests/functional/adapter/test_schema.py deleted file mode 100644 index 90570610..00000000 --- a/tests/functional/adapter/test_schema.py +++ /dev/null @@ -1,37 +0,0 @@ -import os - -import pytest -from dbt.tests.util import run_dbt - - -class TestSchemaCreation: - @pytest.fixture(scope="class") - def models(self): - return { - "dummy.sql": """ -{{ config(schema='with_custom_auth') }} -select 1 as id -""", - } - - @staticmethod - @pytest.fixture(scope="class") - def dbt_profile_target_update(): - return {"schema_authorization": "{{ env_var('DBT_TEST_USER_1') }}"} - - @staticmethod - def _verify_schema_owner(schema_name, owner, project): - get_schema_owner = f""" -select SCHEMA_OWNER from INFORMATION_SCHEMA.SCHEMATA where SCHEMA_NAME = '{schema_name}' - """ - result = project.run_sql(get_schema_owner, fetch="one")[0] - assert result == owner - - def test_schema_creation(self, project, unique_schema): - res = run_dbt(["run"]) - assert len(res) == 1 - - self._verify_schema_owner(unique_schema, os.getenv("DBT_TEST_USER_1"), project) - self._verify_schema_owner( - unique_schema + "_with_custom_auth", os.getenv("DBT_TEST_USER_1"), project - ) diff --git a/tests/functional/adapter/test_seed.py b/tests/functional/adapter/test_seed.py deleted file mode 100644 index 5739e3c1..00000000 --- a/tests/functional/adapter/test_seed.py +++ /dev/null @@ -1,291 +0,0 @@ -import os - -import pytest -from dbt.tests.adapter.simple_seed.fixtures import models__downstream_from_seed_actual -from dbt.tests.adapter.simple_seed.seeds import seed__actual_csv, seeds__expected_sql -from dbt.tests.adapter.simple_seed.test_seed import SeedConfigBase -from dbt.tests.adapter.simple_seed.test_seed import TestBasicSeedTests as BaseBasicSeedTests -from dbt.tests.adapter.simple_seed.test_seed import ( - TestSeedConfigFullRefreshOff as BaseSeedConfigFullRefreshOff, -) -from dbt.tests.adapter.simple_seed.test_seed import ( - TestSeedConfigFullRefreshOn as BaseSeedConfigFullRefreshOn, -) -from dbt.tests.adapter.simple_seed.test_seed import TestSeedCustomSchema as BaseSeedCustomSchema -from dbt.tests.adapter.simple_seed.test_seed import TestSeedParsing as BaseSeedParsing -from dbt.tests.adapter.simple_seed.test_seed import ( - TestSeedSpecificFormats as BaseSeedSpecificFormats, -) -from dbt.tests.adapter.simple_seed.test_seed import ( - TestSimpleSeedEnabledViaConfig as BaseSimpleSeedEnabledViaConfig, -) -from dbt.tests.adapter.simple_seed.test_seed_type_override import ( - BaseSimpleSeedColumnOverride, - seeds__disabled_in_config_csv, - seeds__enabled_in_config_csv, -) -from dbt.tests.util import check_relations_equal, check_table_does_exist, get_connection, run_dbt - -from dbt.adapters.sqlserver import SQLServerAdapter - -fixed_setup_sql = seeds__expected_sql.replace( - "TIMESTAMP WITHOUT TIME ZONE", "DATETIME2(6)" -).replace("TEXT", "VARCHAR(255)") - -seeds__tricky_csv = """ -seed_id,seed_id_str,a_bool,looks_like_a_bool,a_date,looks_like_a_date,relative,weekday -1,1,1,1,2019-01-01 12:32:30,2019-01-01 12:32:30,tomorrow,Saturday -2,2,1,1,2019-01-01 12:32:31,2019-01-01 12:32:31,today,Sunday -3,3,1,1,2019-01-01 12:32:32,2019-01-01 12:32:32,yesterday,Monday -4,4,0,0,2019-01-01 01:32:32,2019-01-01 01:32:32,tomorrow,Saturday -5,5,0,0,2019-01-01 01:32:32,2019-01-01 01:32:32,today,Sunday -6,6,0,0,2019-01-01 01:32:32,2019-01-01 01:32:32,yesterday,Monday -""".lstrip() - -macros__schema_test = """ -{% test column_type(model, column_name, type) %} - - {% set cols = adapter.get_columns_in_relation(model) %} - - {% set col_types = {} %} - {% for col in cols %} - {% do col_types.update({col.name: col.data_type}) %} - {% endfor %} - - {% set col_type = col_types.get(column_name) %} - {% set col_type = 'text' if col_type and 'varchar' in col_type else col_type %} - - {% set validation_message = 'Got a column type of ' ~ col_type ~ ', expected ' ~ type %} - - {% set val = 0 if col_type == type else 1 %} - {% if val == 1 and execute %} - {{ log(validation_message, info=True) }} - {% endif %} - - select '{{ validation_message }}' as validation_error - from (select 1 as empty) as nothing - where {{ val }} = 1 - -{% endtest %} - -""" - -properties__schema_yml = """ -version: 2 -seeds: -- name: seed_enabled - columns: - - name: birthday - tests: - - column_type: - type: date - - name: seed_id - tests: - - column_type: - type: text - -- name: seed_tricky - columns: - - name: seed_id - tests: - - column_type: - type: int - - name: seed_id_str - tests: - - column_type: - type: text - - name: a_bool - tests: - - column_type: - type: int - - name: looks_like_a_bool - tests: - - column_type: - type: text - - name: a_date - tests: - - column_type: - type: datetime2 - - name: looks_like_a_date - tests: - - column_type: - type: text - - name: relative - tests: - - column_type: - type: text - - name: weekday - tests: - - column_type: - type: text -""" - - -class TestSimpleSeedColumnOverrideSQLServer(BaseSimpleSeedColumnOverride): - @pytest.fixture(scope="class") - def seeds(self): - return { - "seed_enabled.csv": seeds__enabled_in_config_csv, - "seed_disabled.csv": seeds__disabled_in_config_csv, - "seed_tricky.csv": seeds__tricky_csv, - } - - @pytest.fixture(scope="class") - def macros(self): - return {"schema_test.sql": macros__schema_test} - - @pytest.fixture(scope="class") - def models(self): - return { - "schema.yml": properties__schema_yml, - } - - -class TestBasicSeedTestsSQLServer(BaseBasicSeedTests): - @pytest.fixture(scope="class", autouse=True) - def setUp(self, project): - project.run_sql(fixed_setup_sql) - - def test_simple_seed(self, project): - """Build models and observe that run truncates a seed and re-inserts rows""" - self._build_relations_for_test(project) - self._check_relation_end_state(run_result=run_dbt(["seed"]), project=project, exists=True) - - def test_simple_seed_full_refresh_flag(self, project): - """Drop the seed_actual table and re-create. - Verifies correct behavior by the absence of the - model which depends on seed_actual.""" - self._build_relations_for_test(project) - self._check_relation_end_state( - run_result=run_dbt(["seed", "--full-refresh"]), project=project, exists=True - ) - - -class TestSeedConfigFullRefreshOnSQLServer(BaseSeedConfigFullRefreshOn): - @pytest.fixture(scope="class", autouse=True) - def setUp(self, project): - project.run_sql(fixed_setup_sql) - - def test_simple_seed_full_refresh_config(self, project): - """Drop the seed_actual table and re-create. - Verifies correct behavior by the absence of the - model which depends on seed_actual.""" - self._build_relations_for_test(project) - self._check_relation_end_state( - run_result=run_dbt(["seed", "--full-refresh"]), project=project, exists=True - ) - - -class TestSeedConfigFullRefreshOffSQLServer(BaseSeedConfigFullRefreshOff): - @pytest.fixture(scope="class", autouse=True) - def setUp(self, project): - project.run_sql(fixed_setup_sql) - - -class TestSeedCustomSchemaSQLServer(BaseSeedCustomSchema): - @pytest.fixture(scope="class", autouse=True) - def setUp(self, project): - project.run_sql(fixed_setup_sql) - - -class TestSimpleSeedEnabledViaConfigSQLServer(BaseSimpleSeedEnabledViaConfig): - @pytest.fixture(scope="function") - def clear_test_schema(self, project): - yield - adapter = project.adapter - assert isinstance(project.adapter, SQLServerAdapter) - with get_connection(project.adapter): - rel = adapter.Relation.create(database=project.database, schema=project.test_schema) - adapter.drop_schema(rel) - - -class TestSeedParsingSQLServer(BaseSeedParsing): - @pytest.fixture(scope="class", autouse=True) - def setUp(self, project): - project.run_sql(fixed_setup_sql) - - -class TestSeedSpecificFormatsSQLServer(BaseSeedSpecificFormats): - pass - - -class TestSeedBatchSizeMaxSQLServer(SeedConfigBase): - @pytest.fixture(scope="class") - def seeds(self, test_data_dir): - return { - "five_columns.csv": """seed_id,first_name,email,ip_address,birthday -1,Larry,lking0@miitbeian.gov.cn,69.135.206.194,2008-09-12 19:08:31 -2,Larry,lperkins1@toplist.cz,64.210.133.162,1978-05-09 04:15:14 -3,Anna,amontgomery2@miitbeian.gov.cn,168.104.64.114,2011-10-16 04:07:57""" - } - - def test_max_batch_size(self, project, logs_dir): - run_dbt(["seed"]) - with open(os.path.join(logs_dir, "dbt.log"), "r") as fp: - logs = "".join(fp.readlines()) - - assert "Inserting batches of 400 records" in logs - - -class TestSeedBatchSizeCustomSQLServer(SeedConfigBase): - @pytest.fixture(scope="class") - def seeds(self, test_data_dir): - return { - "six_columns.csv": """seed_id,first_name,last_name,email,ip_address,birthday -1,Larry,King,lking0@miitbeian.gov.cn,69.135.206.194,2008-09-12 19:08:31 -2,Larry,Perkins,lperkins1@toplist.cz,64.210.133.162,1978-05-09 04:15:14 -3,Anna,Montgomery,amontgomery2@miitbeian.gov.cn,168.104.64.114,2011-10-16 04:07:57""" - } - - def test_custom_batch_size(self, project, logs_dir): - run_dbt(["seed"]) - with open(os.path.join(logs_dir, "dbt.log"), "r") as fp: - logs = "".join(fp.readlines()) - # this is changed from 350. - # Fabric goes -1 of min batch of (2100/number of columns -1) or 400 - assert "Inserting batches of 349 records" in logs - - -class SeedConfigBase: - @pytest.fixture(scope="class") - def project_config_update(self): - return { - "seeds": { - "quote_columns": False, - }, - } - - -class SeedTestBase(SeedConfigBase): - @pytest.fixture(scope="class", autouse=True) - def setUp(self, project): - """Create table for ensuring seeds and models used in tests build correctly""" - project.run_sql(seeds__expected_sql) - - @pytest.fixture(scope="class") - def seeds(self, test_data_dir): - return {"seed_actual.csv": seed__actual_csv} - - @pytest.fixture(scope="class") - def models(self): - return { - "models__downstream_from_seed_actual.sql": models__downstream_from_seed_actual, - } - - def _build_relations_for_test(self, project): - """The testing environment needs seeds and models to interact with""" - seed_result = run_dbt(["seed"]) - assert len(seed_result) == 1 - check_relations_equal(project.adapter, ["seed_expected", "seed_actual"]) - - run_result = run_dbt() - assert len(run_result) == 1 - check_relations_equal( - project.adapter, ["models__downstream_from_seed_actual", "seed_expected"] - ) - - def _check_relation_end_state(self, run_result, project, exists: bool): - assert len(run_result) == 1 - check_relations_equal(project.adapter, ["seed_actual", "seed_expected"]) - if exists: - check_table_does_exist(project.adapter, "models__downstream_from_seed_actual") diff --git a/tests/functional/adapter/test_sources.py b/tests/functional/adapter/test_sources.py deleted file mode 100644 index 91c34db8..00000000 --- a/tests/functional/adapter/test_sources.py +++ /dev/null @@ -1,65 +0,0 @@ -import pytest -from dbt.tests.adapter.basic.files import config_materialized_table, config_materialized_view -from dbt.tests.util import run_dbt - -source_regular = """ -version: 2 -sources: -- name: regular - schema: INFORMATION_SCHEMA - tables: - - name: VIEWS - columns: - - name: TABLE_NAME - tests: - - not_null -""" - -source_space_in_name = """ -version: 2 -sources: -- name: 'space in name' - schema: INFORMATION_SCHEMA - tables: - - name: VIEWS - columns: - - name: TABLE_NAME - tests: - - not_null -""" - -select_from_source_regular = """ -select * from {{ source("regular", "VIEWS") }} with (nolock) -""" - -select_from_source_space_in_name = """ -select * from {{ source("space in name", "VIEWS") }} with (nolock) -""" - - -class TestSourcesSQLServer: - @pytest.fixture(scope="class") - def models(self): - return { - "source_regular.yml": source_regular, - "source_space_in_name.yml": source_space_in_name, - "v_select_from_source_regular.sql": config_materialized_view - + select_from_source_regular, - "v_select_from_source_space_in_name.sql": config_materialized_view - + select_from_source_space_in_name, - "t_select_from_source_regular.sql": config_materialized_table - + select_from_source_regular, - "t_select_from_source_space_in_name.sql": config_materialized_table - + select_from_source_space_in_name, - } - - def test_dbt_run(self, project): - run_dbt(["compile"]) - - ls = run_dbt(["list"]) - assert len(ls) == 8 - ls_sources = [src for src in ls if src.startswith("source:")] - assert len(ls_sources) == 2 - - run_dbt(["run"]) - run_dbt(["test"]) diff --git a/tests/functional/adapter/test_timestamps.py b/tests/functional/adapter/test_timestamps.py deleted file mode 100644 index acea51cd..00000000 --- a/tests/functional/adapter/test_timestamps.py +++ /dev/null @@ -1,18 +0,0 @@ -import pytest -from dbt.tests.adapter.utils.test_timestamps import BaseCurrentTimestamps - - -class TestCurrentTimestampSQLServer(BaseCurrentTimestamps): - @pytest.fixture(scope="class") - def models(self): - return { - "get_current_timestamp.sql": 'select {{ current_timestamp() }} as "current_timestamp"' - } - - @pytest.fixture(scope="class") - def expected_schema(self): - return {"current_timestamp": "datetime2"} - - @pytest.fixture(scope="class") - def expected_sql(self): - return '''select SYSDATETIME() as "current_timestamp"''' diff --git a/tests/functional/adapter/test_utils.py b/tests/functional/adapter/test_utils.py deleted file mode 100644 index be166f29..00000000 --- a/tests/functional/adapter/test_utils.py +++ /dev/null @@ -1,278 +0,0 @@ -import pytest -from dbt.tests.adapter.utils.fixture_cast_bool_to_text import models__test_cast_bool_to_text_yml -from dbt.tests.adapter.utils.fixture_listagg import ( - models__test_listagg_yml, - seeds__data_listagg_csv, -) -from dbt.tests.adapter.utils.test_any_value import BaseAnyValue -from dbt.tests.adapter.utils.test_array_append import BaseArrayAppend -from dbt.tests.adapter.utils.test_array_concat import BaseArrayConcat -from dbt.tests.adapter.utils.test_array_construct import BaseArrayConstruct -from dbt.tests.adapter.utils.test_bool_or import BaseBoolOr -from dbt.tests.adapter.utils.test_cast_bool_to_text import BaseCastBoolToText -from dbt.tests.adapter.utils.test_concat import BaseConcat -from dbt.tests.adapter.utils.test_current_timestamp import BaseCurrentTimestampNaive -from dbt.tests.adapter.utils.test_date_trunc import BaseDateTrunc -from dbt.tests.adapter.utils.test_dateadd import BaseDateAdd -from dbt.tests.adapter.utils.test_datediff import BaseDateDiff -from dbt.tests.adapter.utils.test_escape_single_quotes import BaseEscapeSingleQuotesQuote -from dbt.tests.adapter.utils.test_except import BaseExcept -from dbt.tests.adapter.utils.test_hash import BaseHash -from dbt.tests.adapter.utils.test_intersect import BaseIntersect -from dbt.tests.adapter.utils.test_last_day import BaseLastDay -from dbt.tests.adapter.utils.test_length import BaseLength -from dbt.tests.adapter.utils.test_listagg import BaseListagg -from dbt.tests.adapter.utils.test_position import BasePosition -from dbt.tests.adapter.utils.test_replace import BaseReplace -from dbt.tests.adapter.utils.test_right import BaseRight -from dbt.tests.adapter.utils.test_safe_cast import BaseSafeCast -from dbt.tests.adapter.utils.test_split_part import BaseSplitPart -from dbt.tests.adapter.utils.test_string_literal import BaseStringLiteral - - -class BaseFixedMacro: - @pytest.fixture(scope="class") - def macros(self): - return { - "test_assert_equal.sql": """ - {% test assert_equal(model, actual, expected) %} - select * from {{ model }} - where {{ actual }} != {{ expected }} - or ({{ actual }} is null and {{ expected }} is not null) - or ({{ expected }} is null and {{ actual }} is not null) - {% endtest %} - """ - } - - -class TestAnyValueSQLServer(BaseAnyValue): - pass - - -@pytest.mark.skip("bool_or not supported in this adapter") -class TestBoolOrSQLServer(BaseBoolOr): - pass - - -class TestCastBoolToTextSQLServer(BaseCastBoolToText): - @pytest.fixture(scope="class") - def models(self): - models__test_cast_bool_to_text_sql = """ - with data as ( - - select 0 as input, 'false' as expected union all - select 1 as input, 'true' as expected union all - select null as input, null as expected - - ) - - select - - {{ cast_bool_to_text("input") }} as actual, - expected - - from data - """ - - return { - "test_cast_bool_to_text.yml": models__test_cast_bool_to_text_yml, - "test_cast_bool_to_text.sql": self.interpolate_macro_namespace( - models__test_cast_bool_to_text_sql, "cast_bool_to_text" - ), - } - - -class TestConcatSQLServer(BaseConcat): - @pytest.fixture(scope="class") - def seeds(self): - return { - "data_concat.csv": """input_1,input_2,output -a,b,ab -a,,a -,b,b -""" - } - - -class TestDateTruncSQLServer(BaseDateTrunc): - pass - - -seeds__data_hash_csv = """input_1,output -ab,187ef4436122d1cc2f40dc2b92f0eba0 -a,0cc175b9c0f1b6a831c399e269772661 -1,c4ca4238a0b923820dcc509a6f75849b -,d41d8cd98f00b204e9800998ecf8427e""" - - -class TestHashSQLServer(BaseHash): - @pytest.fixture(scope="class") - def seeds(self): - return {"data_hash.csv": seeds__data_hash_csv} - - -class TestStringLiteralSQLServer(BaseStringLiteral): - pass - - -class TestSplitPartSQLServer(BaseSplitPart): - pass - - -class TestDateDiffSQLServer(BaseDateDiff): - pass - - -class TestEscapeSingleQuotesSQLServer(BaseEscapeSingleQuotesQuote): - pass - - -class TestIntersectSQLServer(BaseIntersect): - pass - - -class TestLastDaySQLServer(BaseLastDay): - pass - - -class TestLengthSQLServer(BaseLength): - pass - - -class TestListaggSQLServer(BaseListagg): - # Only supported in SQL Server 2017 and later or cloud versions - # DISTINCT not supported - # limit not supported - @pytest.fixture(scope="class") - def seeds(self): - seeds__data_listagg_output_csv = """group_col,expected,version -1,"a_|_b_|_c",bottom_ordered -2,"1_|_a_|_p",bottom_ordered -3,"g_|_g_|_g",bottom_ordered -3,"g, g, g",comma_whitespace_unordered -3,"g,g,g",no_params - """ - - return { - "data_listagg.csv": seeds__data_listagg_csv, - "data_listagg_output.csv": seeds__data_listagg_output_csv, - } - - @pytest.fixture(scope="class") - def models(self): - models__test_listagg_sql = """ -with data as ( - - select * from {{ ref('data_listagg') }} - -), - -data_output as ( - - select * from {{ ref('data_listagg_output') }} - -), - -calculate as ( - - select - group_col, - {{ listagg('string_text', "'_|_'", "order by order_col") }} as actual, - 'bottom_ordered' as version - from data - group by group_col - - union all - - select - group_col, - {{ listagg('string_text', "', '") }} as actual, - 'comma_whitespace_unordered' as version - from data - where group_col = 3 - group by group_col - - union all - - select - group_col, - {{ listagg('string_text') }} as actual, - 'no_params' as version - from data - where group_col = 3 - group by group_col - -) - -select - calculate.actual, - data_output.expected -from calculate -left join data_output -on calculate.group_col = data_output.group_col -and calculate.version = data_output.version -""" - - return { - "test_listagg.yml": models__test_listagg_yml, - "test_listagg.sql": self.interpolate_macro_namespace( - models__test_listagg_sql, "listagg" - ), - } - - -class TestRightSQLServer(BaseRight): - pass - - -class TestSafeCastSQLServer(BaseSafeCast): - pass - - -class TestDateAddSQLServer(BaseDateAdd): - @pytest.fixture(scope="class") - def project_config_update(self): - return { - "name": "test", - "seeds": { - "test": { - "data_dateadd": { - "+column_types": { - "from_time": "datetimeoffset", - "result": "datetimeoffset", - }, - }, - }, - }, - } - - -class TestExceptSQLServer(BaseExcept): - pass - - -class TestPositionSQLServer(BasePosition): - pass - - -class TestReplaceSQLServer(BaseReplace): - pass - - -class TestCurrentTimestampSQLServer(BaseCurrentTimestampNaive): - pass - - -@pytest.mark.skip(reason="arrays not supported") -class TestArrayAppendSQLServer(BaseArrayAppend): - pass - - -@pytest.mark.skip(reason="arrays not supporteTd") -class TestArrayConcatSQLServer(BaseArrayConcat): - pass - - -@pytest.mark.skip(reason="arrays not supported") -class TestArrayConstructSQLServer(BaseArrayConstruct): - pass diff --git a/tests/unit/adapters/mssql/test_sqlserver_connection_manager.py b/tests/unit/adapters/mssql/test_sqlserver_connection_manager.py new file mode 100644 index 00000000..20c5031b --- /dev/null +++ b/tests/unit/adapters/mssql/test_sqlserver_connection_manager.py @@ -0,0 +1,41 @@ +import pytest +from azure.identity import AzureCliCredential + +from dbt.adapters.sqlserver.sqlserver_connections import ( # byte_array_to_datetime, + bool_to_connection_string_arg, + get_pyodbc_attrs_before, +) +from dbt.adapters.sqlserver.sqlserver_credentials import SQLServerCredentials + +# See +# https://github.com/Azure/azure-sdk-for-python/blob/azure-identity_1.5.0/sdk/identity/azure-identity/tests/test_cli_credential.py +CHECK_OUTPUT = AzureCliCredential.__module__ + ".subprocess.check_output" + + +@pytest.fixture +def credentials() -> SQLServerCredentials: + credentials = SQLServerCredentials( + driver="ODBC Driver 17 for SQL Server", + host="fake.sql.sqlserver.net", + database="dbt", + schema="sqlserver", + ) + return credentials + + +def test_get_pyodbc_attrs_before_empty_dict_when_service_principal( + credentials: SQLServerCredentials, +) -> None: + """ + When the authentication is set to sql we expect an empty attrs before. + """ + attrs_before = get_pyodbc_attrs_before(credentials) + assert attrs_before == {} + + +@pytest.mark.parametrize( + "key, value, expected", + [("somekey", False, "somekey=No"), ("somekey", True, "somekey=Yes")], +) +def test_bool_to_connection_string_arg(key: str, value: bool, expected: str) -> None: + assert bool_to_connection_string_arg(key, value) == expected diff --git a/tests/unit/adapters/sqlserver/test_sql_server_connection_manager.py b/tests/unit/adapters/sqlserver/test_sql_server_connection_manager.py deleted file mode 100644 index 010ecd1a..00000000 --- a/tests/unit/adapters/sqlserver/test_sql_server_connection_manager.py +++ /dev/null @@ -1,128 +0,0 @@ -import pytest -from azure.identity import AzureCliCredential - -from dbt.adapters.sqlserver.sql_server_connection_manager import ( # byte_array_to_datetime, - bool_to_connection_string_arg, - get_pyodbc_attrs_before, -) -from dbt.adapters.sqlserver.sql_server_credentials import SQLServerCredentials - -# See -# https://github.com/Azure/azure-sdk-for-python/blob/azure-identity_1.5.0/sdk/identity/azure-identity/tests/test_cli_credential.py -CHECK_OUTPUT = AzureCliCredential.__module__ + ".subprocess.check_output" - - -@pytest.fixture -def credentials() -> SQLServerCredentials: - credentials = SQLServerCredentials( - driver="ODBC Driver 17 for SQL Server", - host="fake.sql.sqlserver.net", - database="dbt", - schema="sqlserver", - ) - return credentials - - -# @pytest.fixture -# def mock_cli_access_token() -> str: -# access_token = "access token" -# expected_expires_on = 1602015811 -# successful_output = json.dumps( -# { -# "expiresOn": dt.datetime.fromtimestamp(expected_expires_on).strftime( -# "%Y-%m-%d %H:%M:%S.%f" -# ), -# "accessToken": access_token, -# "subscription": "some-guid", -# "tenant": "some-guid", -# "tokenType": "Bearer", -# } -# ) -# return successful_output - - -def test_get_pyodbc_attrs_before_empty_dict_when_service_principal( - credentials: SQLServerCredentials, -) -> None: - """ - When the authentication is set to sql we expect an empty attrs before. - """ - attrs_before = get_pyodbc_attrs_before(credentials) - assert attrs_before == {} - - -# @pytest.mark.parametrize("authentication", ["CLI", "cli", "cLi"]) -# def test_get_pyodbc_attrs_before_contains_access_token_key_for_cli_authentication( -# credentials: SQLServerCredentials, -# authentication: str, -# mock_cli_access_token: str, -# ) -> None: -# """ -# When the cli authentication is used, the attrs before should contain an -# access token key. -# """ -# credentials.authentication = authentication -# with mock.patch(CHECK_OUTPUT, mock.Mock(return_value=mock_cli_access_token)): -# attrs_before = get_pyodbc_attrs_before(credentials) -# assert 1256 in attrs_before.keys() - - -@pytest.mark.parametrize( - "key, value, expected", - [("somekey", False, "somekey=No"), ("somekey", True, "somekey=Yes")], -) -def test_bool_to_connection_string_arg(key: str, value: bool, expected: str) -> None: - assert bool_to_connection_string_arg(key, value) == expected - - -# @pytest.mark.parametrize( -# "value, expected_datetime, expected_str", -# [ -# ( -# bytes( -# [ -# 0xE6, -# 0x07, # 2022 year unsigned short -# 0x0C, -# 0x00, # 12 month unsigned short -# 0x11, -# 0x00, # 17 day unsigned short -# 0x11, -# 0x00, # 17 hour unsigned short -# 0x34, -# 0x00, # 52 minute unsigned short -# 0x12, -# 0x00, # 18 second unsigned short -# 0xBC, -# 0xCC, -# 0x5B, -# 0x07, # 123456700 10⁻⁷ second unsigned long -# 0xFE, -# 0xFF, # -2 offset hour signed short -# 0xE2, -# 0xFF, # -30 offset minute signed short -# ] -# ), -# dt.datetime( -# year=2022, -# month=12, -# day=17, -# hour=17, -# minute=52, -# second=18, -# microsecond=123456700 // 1000, # 10⁻⁶ second -# tzinfo=dt.timezone(dt.timedelta(hours=-2, minutes=-30)), -# ), -# "2022-12-17 17:52:18.123456-02:30", -# ) -# ], -# ) -# def test_byte_array_to_datetime( -# value: bytes, expected_datetime: dt.datetime, expected_str: str -# ) -> None: -# """ -# Assert SQL_SS_TIMESTAMPOFFSET_STRUCT bytes are converted to datetime and str -# https://learn.microsoft.com/sql/relational-databases/native-client-odbc-date-time/data-type-support-for-odbc-date-and-time-improvements#sql_ss_timestampoffset_struct -# """ -# assert byte_array_to_datetime(value) == expected_datetime -# assert str(byte_array_to_datetime(value)) == expected_str