Merge pull request #368 from dbt-msft/create-as

Fix concurrency issues and document create as

Author: sdebruyn

dbt/include/sqlserver/macros/adapters/columns.sql

@@ -1,48 +1,27 @@
 {% macro sqlserver__get_columns_in_relation(relation) -%}
   {% call statement('get_columns_in_relation', fetch_result=True) %}
 
-    with
-        regular_db_cols as (
-            select
-                ordinal_position,
-                column_name,
-                data_type,
-                character_maximum_length,
-                numeric_precision,
-                numeric_scale
-            from [{{ relation.database }}].INFORMATION_SCHEMA.COLUMNS
-            where table_name = '{{ relation.identifier }}'
-              and table_schema = '{{ relation.schema }}'
-        ),
-
-        temp_db_cols as (
-            select
-                ordinal_position,
-                column_name collate database_default as column_name,
-                data_type collate database_default as data_type,
-                character_maximum_length,
-                numeric_precision,
-                numeric_scale
-            from tempdb.INFORMATION_SCHEMA.COLUMNS
-            where table_name like '{{ relation.identifier }}%'
-        ),
-
-        all_cols as (
-            select *
-            from regular_db_cols
-            union
-            select *
-            from temp_db_cols
-        )
+    with mapping as (
+        select
+            row_number() over (partition by object_name(c.object_id) order by c.column_id) as ordinal_position,
+            c.name collate database_default as column_name,
+            t.name as data_type,
+            c.max_length as character_maximum_length,
+            c.precision as numeric_precision,
+            c.scale as numeric_scale
+        from [{{ 'tempdb' if '#' in relation.identifier else relation.database }}].sys.columns c
+        inner join sys.types t
+        on c.user_type_id = t.user_type_id
+        where c.object_id = object_id('{{ 'tempdb..' ~ relation.include(database=false, schema=false) if '#' in relation.identifier else relation }}')
+    )
 
     select
         column_name,
         data_type,
         character_maximum_length,
         numeric_precision,
         numeric_scale
-    from
-        all_cols
+    from mapping
     order by ordinal_position
 
   {% endcall %}

dbt/include/sqlserver/macros/adapters/relation.sql

@@ -1,9 +1,9 @@
 {% macro sqlserver__make_temp_relation(base_relation, suffix) %}
-    {% set tmp_identifier = '#' ~  base_relation.identifier ~ suffix %}
-    {% set tmp_relation = base_relation.incorporate(
-                                path={"identifier": tmp_identifier}) -%}
+    {%- set temp_identifier = '#' ~ base_relation.identifier ~ suffix -%}
+    {%- set temp_relation = base_relation.incorporate(
+                                path={"identifier": temp_identifier}) -%}
 
-    {% do return(tmp_relation) %}
+    {{ return(temp_relation) }}
 {% endmacro %}
 
 {% macro sqlserver__drop_relation(relation) -%}

dbt/include/sqlserver/macros/materializations/models/table/create_table_as.sql

@@ -1,26 +1,35 @@
 {% macro sqlserver__create_table_as(temporary, relation, sql) -%}
+   {#- TODO: add contracts here when in dbt 1.5 -#}
+   {%- set sql_header = config.get('sql_header', none) -%}
    {%- set as_columnstore = config.get('as_columnstore', default=true) -%}
-   {% set tmp_relation = relation.incorporate(
-   path={"identifier": relation.identifier.replace("#", "") ~ '_temp_view'},
-   type='view')-%}
    {%- set temp_view_sql = sql.replace("'", "''") -%}
+   {%- set tmp_relation = relation.incorporate(
+        path={"identifier": relation.identifier.replace("#", "") ~ '_temp_view'},
+        type='view') -%}
 
-   {{ sqlserver__drop_relation_script(tmp_relation) }}
+   {{- sql_header if sql_header is not none -}}
 
-   {{ sqlserver__drop_relation_script(relation) }}
+    -- drop previous temp view
+   {{- sqlserver__drop_relation_script(tmp_relation) }}
 
+    -- create temp view
    USE [{{ relation.database }}];
    EXEC('create view {{ tmp_relation.include(database=False) }} as
     {{ temp_view_sql }}
     ');
 
-   SELECT * INTO {{ relation }} FROM
-    {{ tmp_relation }}
+   -- select into the table and create it that way
+   {# TempDB schema is ignored, always goes to dbo #}
+   SELECT *
+   INTO {{ relation.include(database=False, schema=(not temporary))  }}
+   FROM {{ tmp_relation }}
 
+   -- drop temp view
    {{ sqlserver__drop_relation_script(tmp_relation) }}
 
-   {% if not temporary and as_columnstore -%}
-   {{ sqlserver__create_clustered_columnstore_index(relation) }}
-   {% endif %}
+   {%- if not temporary and as_columnstore -%}
+        -- add columnstore index
+        {{ sqlserver__create_clustered_columnstore_index(relation) }}
+   {%- endif -%}
 
 {% endmacro %}

dbt/include/sqlserver/macros/materializations/models/view/create_view_as.sql

@@ -1,12 +1,14 @@
+{% macro sqlserver__create_view_as(relation, sql) -%}
+    {%- set sql_header = config.get('sql_header', none) -%}
+    {{ sql_header if sql_header is not none }}
+    USE [{{ relation.database }}];
+    {{ sqlserver__create_view_exec(relation, sql) }}
+{% endmacro %}
+
 {% macro sqlserver__create_view_exec(relation, sql) -%}
+    {#- TODO: add contracts here when in dbt 1.5 -#}
     {%- set temp_view_sql = sql.replace("'", "''") -%}
     execute('create view {{ relation.include(database=False) }} as
-    {{ temp_view_sql }}
+        {{ temp_view_sql }}
     ');
 {% endmacro %}
-
-
-{% macro sqlserver__create_view_as(relation, sql) -%}
-    USE [{{ relation.database }}];
-    {{ sqlserver__create_view_exec(relation, sql) }}
-{% endmacro %}

tests/functional/adapter/test_basic.py

@@ -1,4 +1,5 @@
 import pytest
+from dbt.tests.adapter.basic.files import incremental_not_schema_change_sql
 from dbt.tests.adapter.basic.test_adapter_methods import BaseAdapterMethod
 from dbt.tests.adapter.basic.test_base import BaseSimpleMaterializations
 from dbt.tests.adapter.basic.test_empty import BaseEmpty
@@ -43,20 +44,11 @@ class TestIncrementalSQLServer(BaseIncremental):
 class TestIncrementalNotSchemaChangeSQLServer(BaseIncrementalNotSchemaChange):
     @pytest.fixture(scope="class")
     def models(self):
-        incremental_not_schema_change_sql = """
-{{ config(
-    materialized="incremental",
-    unique_key="user_id_current_time",
-    on_schema_change="sync_all_columns") }}
-select
-    1 + '-' + current_timestamp as user_id_current_time,
-    {% if is_incremental() %}
-        'thisis18characters' as platform
-    {% else %}
-        'okthisis20characters' as platform
-    {% endif %}
-            """
-        return {"incremental_not_schema_change.sql": incremental_not_schema_change_sql}
+        return {
+            "incremental_not_schema_change.sql": incremental_not_schema_change_sql.replace(
+                "||", "+"
+            )
+        }
 
 
 class TestGenericTestsSQLServer(BaseGenericTests):