Merge branch 'master' into token_timeout_fix

Author: Alieu

.circleci/config.yml

@@ -42,7 +42,8 @@ jobs:
       - checkout
       - run: &install-dbt-sqlserver
           name: "install dbt-sqlserver"
-          command: pip install .
+          command: |
+            pip install .
       - run:
           name: wait for SQL Server container to set up
           command: sleep 30

.gitignore

@@ -96,4 +96,4 @@ env/
 venv/
 ENV/
 env.bak/
-venv.bak/
+venv.bak/

CHANGELOG.md

@@ -1,15 +1,47 @@
 # Changelog
 
-### v0.19.0.3
+### v0.20.0
+
+#### features:
+
+- dbt-sqlserver will now work with dbt `v0.20.0`. Please see dbt's [upgrading to `v0.20.0` docs](https://docs.getdbt.com/docs/guides/migration-guide/upgrading-to-0-20-0) for more info.
+- users can now declare a custom `max_batch_size` in the project configuration to set the batch size used by the seed file loader. [#127](https://github.com/dbt-msft/dbt-sqlserver/issues/127) and [#151](https://github.com/dbt-msft/dbt-sqlserver/pull/151) thanks [@jacobm001](https://github.com/jacobm001)
+
+#### under the hood
+
+- `sqlserver__load_csv_rows` now has a safety provided by `calc_batch_size()` to ensure the insert statements won't exceed SQL Server's 2100 parameter limit. [#127](https://github.com/dbt-msft/dbt-sqlserver/issues/127) and [#151](https://github.com/dbt-msft/dbt-sqlserver/pull/151) thanks [@jacobm001](https://github.com/jacobm001)
+- switched to using a `MANIFEST.in` to declare which files should be included
+- updated `pyodbc` and `azure-identity` dependencies to their latest versions 
+### v0.19.2
+
+#### fixes
+
+- fixing and issue with empty seed table that dbt-redshift already addressed with [fishtown-analytics/dbt#2255](https://github.com/fishtown-analytics/dbt/pull/2255) [#147](https://github.com/dbt-msft/dbt-sqlserver/pull/147)
+- drop unneeded debugging code that only was run when "Active Directory integrated" was given as the auth method [#149](https://github.com/dbt-msft/dbt-sqlserver/pull/149)
+- hotfix for regression introduced by [#126](https://github.com/dbt-msft/dbt-sqlserver/issues/126) that wouldn't surface syntax errors from the SQL engine [#140](https://github.com/dbt-msft/dbt-sqlserver/issues/140) thanks [@jeroen-mostert](https://github.com/jeroen-mostert)!
+
+#### under the hood:
+
+- ensure that macros are not recreated for incremental models [#116](https://github.com/dbt-msft/dbt-sqlserver/issues/116) thanks [@infused-kim](https://github.com/infused-kim)
+- authentication now is case-insensitive and accepts both `CLI` and `cli` as options. [#100](https://github.com/dbt-msft/dbt-sqlserver/issues/100) thanks (@JCZuurmond)[https://github.com/JCZuurmond]
+- add unit tests for azure-identity related token fetching
+
+### v0.19.1
+
+#### features:
+
+- users can now delcare a model's database to be other than the one specified in the profile. This will only work for on-premise SQL Server and Azure SQL Managed Instance. [#126](https://github.com/dbt-msft/dbt-sqlserver/issues/126) thanks [@semcha](https://github.com/semcha)!
 
 #### under the hood
+
+- abandon four-part version names (`v0.19.0.2`) in favor of three-part version names because it isn't [SemVer](https://semver.org/) and it causes problems with the `~=` pip operator used dbt-synapse, a pacakge that depends on dbt-sqlserver
 - allow CI to work with the lower-cost serverless Azure SQL [#132](https://github.com/dbt-msft/dbt-sqlserver/pull/132)
+
 ### v0.19.0.2
 
 #### fixes
 - solved a bug in snapshots introduced in v0.19.0. Fixes: [#108](https://github.com/dbt-msft/dbt-sqlserver/issues/108), [#117](https://github.com/dbt-msft/dbt-sqlserver/issues/117). 
 
-
 ### v0.19.0.1
 
 #### fixes

MANIFEST.in

@@ -0,0 +1 @@
+recursive-include dbt/include *.sql *.yml *.md

README.md

@@ -137,6 +137,15 @@ client_secret: clientsecret
 
 ### Seeds
 
+By default, dbt-sqlserver will attempt to insert seed files in batches of 400 rows. If this exceeds SQL Server's 2100 parameter limit, the adapter will automatically limit to the highest safe value possible.
+
+To set a different default seed value, you can set the variable `max_batch_size` in your project configuration.
+
+```yaml
+vars:
+  max_batch_size: 200 # Any integer less than  or equal to 2100 will do.
+```
+
 ### Hooks
 
 ### Custom schemas

dbt/adapters/sqlserver/__version__.py

@@ -1 +1 @@
-version = '0.19.0.2'
+version = '0.20.0'

dbt/adapters/sqlserver/connections.py

@@ -5,7 +5,7 @@
 import time
 import struct
 from itertools import chain, repeat
-from typing import Callable, Mapping
+from typing import Callable, Dict, Mapping, Optional
 
 import dbt.exceptions
 from dbt.adapters.base import Credentials
@@ -22,6 +22,7 @@
 
 
 AZURE_CREDENTIAL_SCOPE = "https://database.windows.net//.default"
+_TOKEN: Optional[AccessToken] = None
 
 
 @dataclass
@@ -162,12 +163,47 @@ def get_sp_access_token(credentials: SQLServerCredentials) -> AccessToken:
     token = DefaultAzureCredential().get_token(AZURE_CREDENTIAL_SCOPE)
     return token
 
-MAX_QUERY_TIME_IN_SECONDS = 3300
-AZURE_AUTH_FUNCTION_TYPE = Callable[[SQLServerCredentials], AccessToken]
-AZURE_AUTH_FUNCTIONS: Mapping[str, AZURE_AUTH_FUNCTION_TYPE] = {
-    "ServicePrincipal": get_sp_access_token,
-    "CLI": get_cli_access_token,
-}
+
+def get_pyodbc_attrs_before(credentials: SQLServerCredentials) -> Dict:
+    """
+    Get the pyodbc attrs before.
+
+    Parameters
+    ----------
+    credentials : SQLServerCredentials
+        Credentials.
+
+    Returns
+    -------
+    out : Dict
+        The pyodbc attrs before.
+
+    Source
+    ------
+    Authentication for SQL server with an access token:
+    https://docs.microsoft.com/en-us/sql/connect/odbc/using-azure-active-directory?view=sql-server-ver15#authenticating-with-an-access-token
+    """
+    global _TOKEN
+    attrs_before: Dict
+    MAX_QUERY_TIME_IN_SECONDS = 3300
+
+    azure_auth_function_type = Callable[[SQLServerCredentials], AccessToken]
+    azure_auth_functions: Mapping[str, azure_auth_function_type] = {
+        "serviceprincipal": get_sp_access_token,
+        "cli": get_cli_access_token,
+    }
+
+    authentication = credentials.authentication.lower()
+    if authentication not in azure_auth_functions:
+        attrs_before = {}
+    elif _TOKEN is None or (_TOKEN.expires_on - time.time()) < MAX_QUERY_TIME_IN_SECONDS:
+        azure_auth_function = azure_auth_functions[authentication]
+        _TOKEN = _TOKEN or azure_auth_function(credentials)
+        token_bytes = convert_access_token_to_mswindows_byte_string(_TOKEN)
+
+        sql_copt_ss_access_token = 1256  # see source in docstring
+        attrs_before = {sql_copt_ss_access_token: token_bytes}
+    return attrs_before
 
 
 class SQLServerConnectionManager(SQLConnectionManager):
@@ -235,9 +271,6 @@ def open(cls, connection):
                     con_str.append(f"PWD={{{credentials.PWD}}}")
                 elif type_auth == "ActiveDirectoryInteractive":
                     con_str.append(f"UID={{{credentials.UID}}}")
-                elif type_auth == "ActiveDirectoryIntegrated":
-                    # why is this necessary???
-                    con_str.remove("UID={None}")
                 elif type_auth == "ActiveDirectoryMsi":
                     raise ValueError("ActiveDirectoryMsi is not supported yet")
 
@@ -276,25 +309,9 @@ def open(cls, connection):
 
             con_str_display = ';'.join(con_str)
 
-            logger.debug(f'Using connection string: {con_str_display}')
-
-            if type_auth in AZURE_AUTH_FUNCTIONS.keys():
-                # create token if it does not exist
-                if cls.TOKEN is None or (cls.TOKEN.expires_on - time.time()) < MAX_QUERY_TIME_IN_SECONDS:
-                    azure_auth_function = AZURE_AUTH_FUNCTIONS[type_auth]
-                    token = azure_auth_function(credentials)
-                    cls.TOKEN = convert_access_token_to_mswindows_byte_string(
-                        token
-                    )
-
-                # Source:
-                # https://docs.microsoft.com/en-us/sql/connect/odbc/using-azure-active-directory?view=sql-server-ver15#authenticating-with-an-access-token
-                SQL_COPT_SS_ACCESS_TOKEN = 1256
-
-                attrs_before = {SQL_COPT_SS_ACCESS_TOKEN: cls.TOKEN}
-            else:
-                attrs_before = {}
+            logger.debug(f"Using connection string: {con_str_display}")
 
+            attrs_before = get_pyodbc_attrs_before(credentials)
             handle = pyodbc.connect(
                 con_str_concat,
                 attrs_before=attrs_before,
@@ -356,7 +373,7 @@ def add_query(self, sql, auto_begin=True, bindings=None, abridge_sql_log=False):
                     self.get_response(cursor), (time.time() - pre)
                 )
             )
-
+            
             return connection, cursor
 
     @classmethod
@@ -383,9 +400,16 @@ def get_response(cls, cursor) -> AdapterResponse:
 
     def execute(self, sql, auto_begin=True, fetch=False):
         _, cursor = self.add_query(sql, auto_begin)
-        status = self.get_response(cursor)
+        response = self.get_response(cursor)
         if fetch:
+            # Get the result of the first non-empty result set (if any)
+            while cursor.description is None:
+                if not cursor.nextset(): 
+                    break
             table = self.get_result_from_cursor(cursor)
         else:
             table = dbt.clients.agate_helper.empty_table()
-        return status, table
+        # Step through all result sets so we process all errors
+        while cursor.nextset(): 
+            pass
+        return response, table

dbt/adapters/sqlserver/impl.py

@@ -18,7 +18,8 @@ def date_function(cls):
     @classmethod
     def convert_text_type(cls, agate_table, col_idx):
         column = agate_table.columns[col_idx]
-        lens = (len(d.encode("utf-8")) for d in column.values_without_nulls())
+        # see https://github.com/fishtown-analytics/dbt/pull/2255
+        lens = [len(d.encode("utf-8")) for d in column.values_without_nulls()]
         max_len = max(lens) if lens else 64
         length = max_len if max_len > 16 else 16
         return "varchar({})".format(length)

dbt/include/sqlserver/macros/adapters.sql

@@ -26,15 +26,15 @@
            else table_type
       end as table_type
 
-    from information_schema.tables
+    from [{{ schema_relation.database }}].information_schema.tables
     where table_schema like '{{ schema_relation.schema }}'
-      and table_catalog like '{{ schema_relation.database }}'
   {% endcall %}
   {{ return(load_result('list_relations_without_caching').table) }}
 {% endmacro %}
 
 {% macro sqlserver__list_schemas(database) %}
   {% call statement('list_schemas', fetch_result=True, auto_begin=False) -%}
+    USE {{ database }};
     select  name as [schema]
     from sys.schemas
   {% endcall %}
@@ -43,7 +43,7 @@
 
 {% macro sqlserver__create_schema(relation) -%}
   {% call statement('create_schema') -%}
-    USE [{{ relation.database }}]
+    USE [{{ relation.database }}];
     IF NOT EXISTS (SELECT * FROM sys.schemas WHERE name = '{{ relation.without_identifier().schema }}')
     BEGIN
     EXEC('CREATE SCHEMA {{ relation.without_identifier().schema }}')
@@ -72,17 +72,8 @@
 {% endmacro %}
 
 {% macro sqlserver__drop_relation(relation) -%}
-  {% if relation.type == 'view' -%}
-   {% set object_id_type = 'V' %}
-   {% elif relation.type == 'table'%}
-   {% set object_id_type = 'U' %}
-   {%- else -%} invalid target name
-   {% endif %}
   {% call statement('drop_relation', auto_begin=False) -%}
-    if object_id ('{{ relation.include(database=False) }}','{{ object_id_type }}') is not null
-      begin
-      drop {{ relation.type }} {{ relation.include(database=False) }}
-      end
+    {{ sqlserver__drop_relation_script(relation) }}
   {%- endcall %}
 {% endmacro %}
 
@@ -93,6 +84,7 @@
    {% set object_id_type = 'U' %}
    {%- else -%} invalid target name
    {% endif %}
+  USE [{{ relation.database }}];
   if object_id ('{{ relation.include(database=False) }}','{{ object_id_type }}') is not null
       begin
       drop {{ relation.type }} {{ relation.include(database=False) }}
@@ -107,14 +99,24 @@
   {{ return(load_result('check_schema_exists').table) }}
 {% endmacro %}
 
+
+{% macro sqlserver__create_view_exec(relation, sql) -%}
+    {%- set temp_view_sql = sql.replace("'", "''") -%}
+    execute('create view {{ relation.include(database=False) }} as
+    {{ temp_view_sql }}
+    ');
+{% endmacro %}
+
+
 {% macro sqlserver__create_view_as(relation, sql) -%}
-  create view {{ relation.schema }}.{{ relation.identifier }} as
-    {{ sql }}
+    USE [{{ relation.database }}];
+    {{ sqlserver__create_view_exec(relation, sql) }}
 {% endmacro %}
 
 
 {% macro sqlserver__rename_relation(from_relation, to_relation) -%}
   {% call statement('rename_relation') -%}
+    USE [{{ to_relation.database }}];
     EXEC sp_rename '{{ from_relation.schema }}.{{ from_relation.identifier }}', '{{ to_relation.identifier }}'
     IF EXISTS(
     SELECT *
@@ -128,6 +130,7 @@
   {%- set cci_name = relation.schema ~ '_' ~ relation.identifier ~ '_cci' -%}
   {%- set relation_name = relation.schema ~ '_' ~ relation.identifier -%}
   {%- set full_relation = relation.schema ~ '.' ~ relation.identifier -%}
+  use [{{ relation.database }}];
   if EXISTS (
         SELECT * FROM
         sys.indexes WHERE name = '{{cci_name}}'
@@ -149,12 +152,13 @@
 
    {{ sqlserver__drop_relation_script(relation) }}
 
-   EXEC('create view {{ tmp_relation.schema }}.{{ tmp_relation.identifier }} as
+   USE [{{ relation.database }}];
+   EXEC('create view {{ tmp_relation.include(database=False) }} as
     {{ temp_view_sql }}
     ');
 
-   SELECT * INTO {{ relation.schema }}.{{ relation.identifier }} FROM
-    {{ tmp_relation.schema }}.{{ tmp_relation.identifier }}
+   SELECT * INTO {{ relation }} FROM
+    {{ tmp_relation }}
 
    {{ sqlserver__drop_relation_script(tmp_relation) }}
 
@@ -165,11 +169,7 @@
 {% endmacro %}_
 
 {% macro sqlserver__insert_into_from(to_relation, from_relation) -%}
-  {%- set full_to_relation = to_relation.schema ~ '.' ~ to_relation.identifier -%}
-  {%- set full_from_relation = from_relation.schema ~ '.' ~ from_relation.identifier -%}
-
-  SELECT * INTO {{full_to_relation}} FROM {{full_from_relation}}
-
+  SELECT * INTO {{ to_relation }} FROM {{ from_relation }}
 {% endmacro %}
 
 {% macro sqlserver__current_timestamp() -%}
@@ -192,7 +192,7 @@
               character_maximum_length,
               numeric_precision,
               numeric_scale
-          from INFORMATION_SCHEMA.COLUMNS
+          from [{{ relation.database }}].INFORMATION_SCHEMA.COLUMNS
           where table_name = '{{ relation.identifier }}'
             and table_schema = '{{ relation.schema }}'
           UNION ALL