V1.8.8 draft (#251)

* configuring local tests to run on user credentials. Dropping a relation correctly based on its type

* Addressing #243, #221, #228, #229, #232, #235 issues

* Updated test helper with ephemeral

* Updated unit tests

* Updated get_pyodbc_attrs_before_credentials method

* include only lines that start with order by

* Addressed issue #249,#240,#238,#233,#180,#168,#186,#52

* Ensure Testview Name Uniqueness with MD5 of Model Name and Invocation

* Updating integration tests

Author: prdpsvs

.github/workflows/integration-tests-azure.yml

@@ -1,3 +1,4 @@
+---
 name: Integration tests on Fabric DW
 on:   # yamllint disable-line rule:truthy
   workflow_dispatch:

dbt/adapters/fabric/__version__.py

@@ -1 +1 @@
-version = "1.8.8"
+version = "1.8.9"

dbt/adapters/fabric/fabric_adapter.py

@@ -150,6 +150,9 @@ def get_rows_different_sql(
             names = sorted((self.quote(n) for n in column_names))
         columns_csv = ", ".join(names)
 
+        if columns_csv == "":
+            columns_csv = "*"
+
         sql = COLUMNS_EQUAL_SQL.format(
             columns=columns_csv,
             relation_a=str(relation_a),

dbt/adapters/fabric/fabric_connection_manager.py

@@ -352,6 +352,7 @@ def open(cls, connection: Connection) -> Connection:
             con_str.append(f"SERVER={credentials.host}")
 
         con_str.append(f"Database={credentials.database}")
+        con_str.append("Pooling=true")
 
         # Enabling trace flag
         if credentials.trace_flag:
@@ -395,8 +396,8 @@ def open(cls, connection: Connection) -> Connection:
         con_str.append(f"APP={application_name}")
 
         try:
-            if int(credentials.retries) > 0:
-                con_str.append(f"ConnectRetryCount={credentials.retries}")
+            con_str.append("ConnectRetryCount=3")
+            con_str.append("ConnectRetryInterval=10")
 
         except Exception as e:
             logger.debug(
@@ -427,7 +428,7 @@ def open(cls, connection: Connection) -> Connection:
 
         def connect():
             logger.debug(f"Using connection string: {con_str_display}")
-
+            pyodbc.pooling = True
             if credentials.authentication == "ActiveDirectoryAccessToken":
                 attrs_before = get_pyodbc_attrs_before_accesstoken(credentials.access_token)
             else:
@@ -567,3 +568,4 @@ def execute(
         while cursor.nextset():
             pass
         return response, table
+      

dbt/adapters/fabric/fabric_credentials.py

@@ -21,7 +21,7 @@ class FabricCredentials(Credentials):
     authentication: Optional[str] = "ActiveDirectoryServicePrincipal"
     encrypt: Optional[bool] = True  # default value in MS ODBC Driver 18 as well
     trust_cert: Optional[bool] = False  # default value in MS ODBC Driver 18 as well
-    retries: int = 1
+    retries: int = 3
     schema_authorization: Optional[str] = None
     login_timeout: Optional[int] = 0
     query_timeout: Optional[int] = 0

dbt/include/fabric/macros/adapters/catalog.sql

@@ -1,4 +1,5 @@
 {% macro fabric__get_catalog(information_schemas, schemas) -%}
+
     {% set query_label = apply_label() %}
     {%- call statement('catalog', fetch_result=True) -%}
 
@@ -126,144 +127,153 @@
 {%- endmacro %}
 
 {% macro fabric__get_catalog_relations(information_schema, relations) -%}
+
     {% set query_label = apply_label() %}
-    {%- call statement('catalog', fetch_result=True) -%}
+    {%- set distinct_databases = relations | map(attribute='database') | unique | list -%}
 
-    with
-    principals as (
-        select
-            name as principal_name,
-            principal_id as principal_id
-        from
-            sys.database_principals {{ information_schema_hints() }}
-    ),
+    {%- if distinct_databases | length == 1 -%}
+        {%- call statement('catalog', fetch_result=True) -%}
+            {{ get_use_database_sql(distinct_databases[0]) }}
+            with
+            principals as (
+                select
+                    name as principal_name,
+                    principal_id as principal_id
+                from
+                    sys.database_principals {{ information_schema_hints() }}
+            ),
 
-    schemas as (
-        select
-            name as schema_name,
-            schema_id as schema_id,
-            principal_id as principal_id
-        from
-            sys.schemas {{ information_schema_hints() }}
-    ),
+            schemas as (
+                select
+                    name as schema_name,
+                    schema_id as schema_id,
+                    principal_id as principal_id
+                from
+                    sys.schemas {{ information_schema_hints() }}
+            ),
 
-    tables as (
-        select
-            object_id,
-            name as table_name,
-            schema_id as schema_id,
-            principal_id as principal_id,
-            'BASE TABLE' as table_type
-        from
-            sys.tables {{ information_schema_hints() }}
-    ),
+            tables as (
+                select
+                    object_id,
+                    name as table_name,
+                    schema_id as schema_id,
+                    principal_id as principal_id,
+                    'BASE TABLE' as table_type
+                from
+                    sys.tables {{ information_schema_hints() }}
+            ),
 
-    tables_with_metadata as (
-        select
-            object_id,
-            table_name,
-            schema_name,
-            coalesce(tables.principal_id, schemas.principal_id) as owner_principal_id,
-            table_type
-        from
-            tables
-        join schemas on tables.schema_id = schemas.schema_id
-    ),
+            tables_with_metadata as (
+                select
+                    object_id,
+                    table_name,
+                    schema_name,
+                    coalesce(tables.principal_id, schemas.principal_id) as owner_principal_id,
+                    table_type
+                from
+                    tables
+                join schemas on tables.schema_id = schemas.schema_id
+            ),
 
-    views as (
-        select
-            object_id,
-            name as table_name,
-            schema_id as schema_id,
-            principal_id as principal_id,
-            'VIEW' as table_type
-        from
-            sys.views {{ information_schema_hints() }}
-    ),
+            views as (
+                select
+                    object_id,
+                    name as table_name,
+                    schema_id as schema_id,
+                    principal_id as principal_id,
+                    'VIEW' as table_type
+                from
+                    sys.views {{ information_schema_hints() }}
+            ),
 
-    views_with_metadata as (
-        select
-            object_id,
-            table_name,
-            schema_name,
-            coalesce(views.principal_id, schemas.principal_id) as owner_principal_id,
-            table_type
-        from
-            views
-        join schemas on views.schema_id = schemas.schema_id
-    ),
+            views_with_metadata as (
+                select
+                    object_id,
+                    table_name,
+                    schema_name,
+                    coalesce(views.principal_id, schemas.principal_id) as owner_principal_id,
+                    table_type
+                from
+                    views
+                join schemas on views.schema_id = schemas.schema_id
+            ),
 
-    tables_and_views as (
-        select
-            object_id,
-            table_name,
-            schema_name,
-            principal_name,
-            table_type
-        from
-            tables_with_metadata
-        join principals on tables_with_metadata.owner_principal_id = principals.principal_id
-        union all
-        select
-            object_id,
-            table_name,
-            schema_name,
-            principal_name,
-            table_type
-        from
-            views_with_metadata
-        join principals on views_with_metadata.owner_principal_id = principals.principal_id
-    ),
+            tables_and_views as (
+                select
+                    object_id,
+                    table_name,
+                    schema_name,
+                    principal_name,
+                    table_type
+                from
+                    tables_with_metadata
+                join principals on tables_with_metadata.owner_principal_id = principals.principal_id
+                union all
+                select
+                    object_id,
+                    table_name,
+                    schema_name,
+                    principal_name,
+                    table_type
+                from
+                    views_with_metadata
+                join principals on views_with_metadata.owner_principal_id = principals.principal_id
+            ),
 
-    cols as (
+            cols as (
 
-        select
-            c.object_id,
-            c.name as column_name,
-            c.column_id as column_index,
-            t.name as column_type
-        from sys.columns as c {{ information_schema_hints() }}
-        left join sys.types as t on c.system_type_id = t.system_type_id
-    )
+                select
+                    c.object_id,
+                    c.name as column_name,
+                    c.column_id as column_index,
+                    t.name as column_type
+                from sys.columns as c {{ information_schema_hints() }}
+                left join sys.types as t on c.system_type_id = t.system_type_id
+            )
 
-    select
-        DB_NAME() as table_database,
-        tv.schema_name as table_schema,
-        tv.table_name,
-        tv.table_type,
-        null as table_comment,
-        tv.principal_name as table_owner,
-        cols.column_name,
-        cols.column_index,
-        cols.column_type,
-        null as column_comment
-    from tables_and_views tv
-    join cols on tv.object_id = cols.object_id
-    where (
-        {%- for relation in relations -%}
-            {% if relation.schema and relation.identifier %}
-                (
-                    upper(tv.schema_name) = upper('{{ relation.schema }}')
-                    and upper(tv.table_name) = upper('{{ relation.identifier }}')
-                )
-            {% elif relation.schema %}
-                (
-                    upper(tv.schema_name) = upper('{{ relation.schema }}')
-                )
-            {% else %}
-                {% do exceptions.raise_compiler_error(
-                    '`get_catalog_relations` requires a list of relations, each with a schema'
-                ) %}
-            {% endif %}
+            select
+                DB_NAME() as table_database,
+                tv.schema_name as table_schema,
+                tv.table_name,
+                tv.table_type,
+                null as table_comment,
+                tv.principal_name as table_owner,
+                cols.column_name,
+                cols.column_index,
+                cols.column_type,
+                null as column_comment
+            from tables_and_views tv
+            join cols on tv.object_id = cols.object_id
+            where (
+                {%- for relation in relations -%}
+                    {% if relation.schema and relation.identifier %}
+                        (
+                            upper(tv.schema_name) = upper('{{ relation.schema }}')
+                            and upper(tv.table_name) = upper('{{ relation.identifier }}')
+                        )
+                    {% elif relation.schema %}
+                        (
+                            upper(tv.schema_name) = upper('{{ relation.schema }}')
+                        )
+                    {% else %}
+                        {% do exceptions.raise_compiler_error(
+                            '`get_catalog_relations` requires a list of relations, each with a schema'
+                        ) %}
+                    {% endif %}
 
-            {%- if not loop.last %} or {% endif -%}
-        {%- endfor -%}
-    )
+                    {%- if not loop.last %} or {% endif -%}
+                {%- endfor -%}
+            )
 
-    order by column_index
-    {{ query_label }}
-    {%- endcall -%}
+            order by column_index
+            {{ query_label }}
 
-  {{ return(load_result('catalog').table) }}
+        {%- endcall -%}
+        {{ return(load_result('catalog').table) }}
+    {% else %}
+        {% do exceptions.raise_compiler_error(
+            '`get_catalog_relations` can catalog one database at a time'
+        ) %}
+    {% endif %}
 
 {%- endmacro %}

dbt/include/fabric/macros/adapters/metadata.sql

@@ -21,11 +21,12 @@
 {% endmacro %}
 
 {%- macro fabric__get_use_database_sql(database) -%}
-  USE [{{database}}];
+  USE [{{database | replace('"', '')}}];
 {%- endmacro -%}
 
 {% macro fabric__list_schemas(database) %}
   {% call statement('list_schemas', fetch_result=True, auto_begin=False) -%}
+    {{ get_use_database_sql(database) }}
     select  name as [schema]
     from sys.schemas {{ information_schema_hints() }} {{ apply_label() }}
   {% endcall %}

dbt/include/fabric/macros/adapters/schema.sql

@@ -31,6 +31,7 @@
   {%- endfor %}
 
   {% call statement('drop_schema') -%}
-      EXEC('DROP SCHEMA IF EXISTS {{ relation.schema }}')
+    {{ get_use_database_sql(relation.database) }}
+    EXEC('DROP SCHEMA IF EXISTS {{ relation.schema }}')
   {% endcall %}
 {% endmacro %}