dbt 1.10 migration (#310)

99Lys · web-flow · commit 500d4951698b · 2025-10-27T20:55:53.000Z
### Summary Update dbt-dremio to v1.10, supporting sample mode. ### Description - Update to v1.10 and implemented sample mode - Some test fixes ### Test Results - Added `test_sample_mode.py` - All tests behave as expected ### Changelog - [x] Added a summary of what this PR accomplishes to CHANGELOG.md ### Related Issue #303
diff --git a/.github/scripts/compare_test_failures.sh b/.github/scripts/compare_test_failures.sh
@@ -21,7 +21,7 @@ echo "Comparing actual test failures with expected failures from: $expected_fail
 shopt -s globstar
 
 # Extract actual failures from test reports
-actual_failures=$(grep -E "(FAILED tests|ERROR tests)" reports/**/*.txt | awk '{print $2}' | sort)
+actual_failures=$(grep -E "(FAILED tests|ERROR tests)" reports/**/*.txt | awk '{print $2}' | sort -u)
 
 # Read expected failures
 expected_failures=$(sort "$expected_failures_file")
@@ -49,7 +49,7 @@ if [ -n "$unexpected_failures" ]; then
 fi
 
 if [ -n "$missing_failures" ]; then
-  echo "::warning::Expected test failures that did not occur (they passed):"
+  echo "⚠️ Expected test failures that did not occur (they passed):"
   echo "$missing_failures"
   exit_code=1
 fi
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -112,7 +112,7 @@ jobs:
         run: |
           mkdir reports/
           report_file="reports/${{ env.sanitized_test_dir }}.txt"
-          pytest ${{ matrix.test_dir }} | tee $report_file
+          pytest -s ${{ matrix.test_dir }} | tee $report_file
           exit ${PIPESTATUS[0]}
 
       - name: Upload test report as artifact
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,12 +1,21 @@
-# dbt-dremio v1.9.1
+# dbt-dremio v1.10.0
 
 ## Changes
 
-- [#299](https://github.com/dremio/dbt-dremio/pull/299) Enhance persist_docs macro to wrap model and column metadata (including descriptions, tags and tests) into a Markdown wiki for Dremio.
+- Updated dbt-dremio to match dbt-core v1.10 with sample mode
+- Enhanced persist_docs macro to wrap model and column metadata (including descriptions, tags and tests) into a Markdown wiki for Dremio.
 - Refactored CI
 - Fixed tests for hooks and grants
 - Added Dremio Enterprise Catalog tests
 
+## Features
+
+- [#310](https://github.com/dremio/dbt-dremio/pull/310) Sample mode
+
+## Dependency
+
+- Upgraded dbt-core to 1.10.0 and dbt-tests-adapter to 1.16.0
+
 # dbt-dremio v1.9.0
 
 ## Changes
diff --git a/README.md b/README.md
@@ -12,16 +12,16 @@ The `dbt-dremio` package contains all of the code enabling dbt to work with [Dre
 
 The dbt-dremio package supports both Dremio Cloud and Dremio Software (versions 22.0 and later).
 
-## dbt-dremio version 1.9.0
+## dbt-dremio version 1.10.0
 
-Version 1.9.0 of the dbt-dremio adapter is compatible with dbt-core versions 1.9.*.
+Version 1.10.0 of the dbt-dremio adapter is compatible with dbt-core versions 1.10.*.
 
 > Prior to version 1.1.0b, dbt-dremio was created and maintained by [Fabrice Etanchaud](https://github.com/fabrice-etanchaud) on [their GitHub repo](https://github.com/fabrice-etanchaud/dbt-dremio). Code for using Dremio REST APIs was originally authored by [Ryan Murray](https://github.com/rymurr). Contributors in this repo are credited for laying the groundwork and maintaining the adapter till version 1.0.6.5. The dbt-dremio adapter is maintained and distributed by Dremio starting with version 1.1.0b.
 
 ## Getting started
 
 -   [Install dbt-dremio](https://docs.getdbt.com/reference/warehouse-setups/dremio-setup)
-    -   Version 1.9.0 of dbt-dremio requires dbt-core >= 1.9.*.
+    -   Version 1.10.0 of dbt-dremio requires dbt-core >= 1.10.*.
 -   Read the [introduction](https://docs.getdbt.com/docs/introduction/) and [viewpoint](https://docs.getdbt.com/docs/about/viewpoint/)
 
 ## Join the dbt Community
diff --git a/dbt/adapters/dremio/__version__.py b/dbt/adapters/dremio/__version__.py
@@ -9,4 +9,4 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-version = "1.9.0"
+version = "1.10.0"
diff --git a/dbt/adapters/dremio/relation.py b/dbt/adapters/dremio/relation.py
@@ -18,6 +18,7 @@
     BaseRelation,
     Policy,
     ComponentName,
+    EventTimeFilter,
 )
 from typing import Optional, Tuple, Iterator
 
@@ -45,6 +46,7 @@ class DremioRelation(BaseRelation):
     no_schema = "no_schema"
     format: Optional[str] = None
     format_clause: Optional[str] = None
+    event_time_filter: Optional[EventTimeFilter] = None
 
     def quoted_by_component(self, identifier, componentName):
         dot_char = "."
@@ -85,3 +87,42 @@ def _render_iterator(
                 ):  # or key == ComponentName.Schema):
                     path_part = self.quoted_by_component(path_part, key)
             yield key, path_part
+
+    def _format_timestamp(self, timestamp_str: str) -> str:
+        """
+        Convert timestamp with timezone info to Dremio-compatible format.
+        Removes timezone information and limits microseconds precision since Dremio
+        doesn't support timezone info in timestamp literals and has limited microsecond precision.
+
+        Example: '2025-10-02 11:45:01.142708+00:00' -> '2025-10-02 11:45:01.142'
+        """
+        if timestamp_str is None:
+            return timestamp_str
+
+        # Remove timezone information (e.g., +00:00, -05:00, Z)
+        # This regex matches timezone patterns at the end of the string
+        timestamp_str = str(timestamp_str)
+        timestamp_str = re.sub(r'[+-]\d{2}:\d{2}$|Z$', '', timestamp_str)
+
+        # Limit microseconds to 3 digits (milliseconds) as Dremio does not support full 6-digit microseconds
+        # Pattern: YYYY-MM-DD HH:MM:SS.ssssss -> YYYY-MM-DD HH:MM:SS.sss
+        timestamp_str = re.sub(r'(\.\d{3})\d{3}$', r'\1', timestamp_str)
+
+        return timestamp_str
+
+    # Override in order to apply _format_timestamp
+    def _render_event_time_filtered(self, event_time_filter: EventTimeFilter) -> str:
+        """
+        Returns "" if start and end are both None
+        """
+        filter = ""
+        start_ts = self._format_timestamp(event_time_filter.start)
+        end_ts = self._format_timestamp(event_time_filter.end)
+        if event_time_filter.start and event_time_filter.end:
+            filter = f"{event_time_filter.field_name} >= '{start_ts}' and {event_time_filter.field_name} < '{end_ts}'"
+        elif event_time_filter.start:
+            filter = f"{event_time_filter.field_name} >= '{start_ts}'"
+        elif event_time_filter.end:
+            filter = f"{event_time_filter.field_name} < '{end_ts}'"
+
+        return filter
diff --git a/dbt/include/dremio/macros/builtins/builtins.sql b/dbt/include/dremio/macros/builtins/builtins.sql
@@ -24,7 +24,7 @@ limitations under the License.*/
       and model.config.format is defined
       else none -%}
     {%- set format_clause = format_clause_from_node(model.config) if format is not none else none -%}
-    {%- set relation2 = api.Relation.create(database=relation.database, schema=relation.schema, identifier=relation.identifier, format=format, format_clause=format_clause, limit=relation.limit) -%}
+    {%- set relation2 = api.Relation.create(database=relation.database, schema=relation.schema, identifier=relation.identifier, format=format, format_clause=format_clause, limit=relation.limit, event_time_filter=relation.event_time_filter) -%}
       {{ return (relation2) }}
   {%- else -%}
     {{ return (relation) }}
@@ -40,7 +40,7 @@ limitations under the License.*/
       and source.external.format is defined
       else none -%}
     {%- set format_clause = format_clause_from_node(source.external) if format is not none else none -%}
-    {%- set relation2 = api.Relation.create(database=relation.database, schema=relation.schema, identifier=relation.identifier, format=format, format_clause=format_clause, limit=relation.limit) -%}
+    {%- set relation2 = api.Relation.create(database=relation.database, schema=relation.schema, identifier=relation.identifier, format=format, format_clause=format_clause, limit=relation.limit, event_time_filter=relation.event_time_filter) -%}
       {{ return (relation2) }}
   {%- else -%}
     {{ return (relation) }}
diff --git a/dev_requirements.txt b/dev_requirements.txt
@@ -3,8 +3,8 @@ Babel==2.12.1
 betterproto==1.2.5
 certifi==2023.7.22
 charset-normalizer==3.1.0
-dbt-core==1.9.0
-dbt-tests-adapter==1.11.0
+dbt-core==1.10.0
+dbt-tests-adapter==1.16.0
 python-dotenv==1.0.1
 exceptiongroup==1.1.1
 future==0.18.3
@@ -13,11 +13,9 @@ h2==4.1.0
 hpack==4.0.0
 hyperframe==6.0.1
 iniconfig==2.0.0
-jsonschema==4.17.3
 leather==0.3.4
 MarkupSafe==2.1.2
 msgpack==1.0.5
-multidict==6.0.4
 parsedatetime==2.4
 pip-licenses==4.1.0
 pluggy==1.0.0
diff --git a/setup.py b/setup.py
@@ -22,7 +22,7 @@
 
 package_name = "dbt-dremio"
 
-package_version = "1.9.0"
+package_version = "1.10.0"
 
 description = """The Dremio adapter plugin for dbt"""
 
@@ -37,9 +37,9 @@
     packages=find_namespace_packages(include=["dbt", "dbt.*"]),
     include_package_data=True,
     install_requires=[
-        "dbt-core>=1.9",
-        "dbt-common>=1.11,<2.0",
-        "dbt-adapters>=1.10.1, <2.0",
+        "dbt-core>=1.10",
+        "dbt-common>=1.27,<2.0",
+        "dbt-adapters>=1.16.1, <2.0",
         "requests>=2.31.0",
     ],
     classifiers=[
diff --git a/tests/functional/adapter/basic/test_concurrency.py b/tests/functional/adapter/basic/test_concurrency.py
@@ -113,4 +113,4 @@ def test_concurrency(self, project):
         check_table_does_not_exist(project.adapter, "invalid")
         check_table_does_not_exist(project.adapter, "skip")
 
-        assert "PASS=5 WARN=0 ERROR=1 SKIP=1 TOTAL=7" in output
+        assert "PASS=5 WARN=0 ERROR=1 SKIP=1 NO-OP=0 TOTAL=7" in output
diff --git a/tests/functional/adapter/dremio_specific/test_reflections.py b/tests/functional/adapter/dremio_specific/test_reflections.py
@@ -4,7 +4,6 @@
 from dbt.adapters.dremio.api.parameters import ParametersBuilder
 from dbt.adapters.dremio.api.authentication import DremioAuthentication
 from dbt.tests.util import run_dbt
-from pydantic.experimental.pipeline import transform
 
 view1_model = """
 SELECT IncidntNum, Category, Descript, DayOfWeek, TO_DATE("SF_incidents2016.json"."Date", 'YYYY-MM-DD', 1) AS "Date", "SF_incidents2016.json"."Time" AS "Time", PdDistrict, Resolution, Address, X, Y, Location, PdId
diff --git a/tests/functional/adapter/sample_mode/test_sample_mode.py b/tests/functional/adapter/sample_mode/test_sample_mode.py
@@ -0,0 +1,79 @@
+import datetime
+import os
+from unittest import mock
+import pytest
+
+from dbt.tests.adapter.sample_mode.test_sample_mode import (
+    BaseSampleModeTest,
+)
+from dbt.tests.util import run_dbt
+from tests.utils.util import BUCKET, relation_from_name
+from pprint import pformat
+
+# Use UTC time to match dbt-core's sample mode window calculation
+now = datetime.datetime.now(datetime.timezone.utc).replace(tzinfo=None)
+twelve_hours_ago = now - datetime.timedelta(hours=12)
+two_days_ago = now - datetime.timedelta(days=2)
+
+_input_model_sql = f"""
+{{{{ config(materialized='table', event_time='event_time') }}}}
+select 1 as id, cast('{two_days_ago.strftime('%Y-%m-%d %H:%M:%S')}' as timestamp) as event_time
+UNION ALL
+select 2 as id, cast('{twelve_hours_ago.strftime('%Y-%m-%d %H:%M:%S')}' as timestamp) as event_time
+UNION ALL
+select 3 as id, cast('{now.strftime('%Y-%m-%d %H:%M:%S')}' as timestamp) as event_time
+"""
+
+class TestDremioSampleMode(BaseSampleModeTest):
+    @pytest.fixture(scope="class")
+    def input_model_sql(self) -> str:
+        """
+        This is the SQL that defines the input model to be sampled, including any {{ config(..) }}.
+        event_time is a required configuration of this input
+        """
+        return _input_model_sql
+    
+    # Override this fixture to set the proper root_path
+    @pytest.fixture(scope="class")
+    def dbt_profile_data(
+        self, unique_schema, dbt_profile_target, profiles_config_update
+    ):
+        profile = {
+            "test": {
+                "outputs": {
+                    "default": {},
+                },
+                "target": "default",
+            },
+        }
+        target = dbt_profile_target
+        target["schema"] = unique_schema
+        target["root_path"] = f"{BUCKET}.{unique_schema}"
+        profile["test"]["outputs"]["default"] = target
+
+        if profiles_config_update:
+            profile.update(profiles_config_update)
+        return profile
+
+    # Pasting the original function here so it uses our relation_from_name
+    def assert_row_count(self, project, relation_name: str, expected_row_count: int):
+        relation = relation_from_name(project.adapter, relation_name)
+        result = project.run_sql(f"select * from {relation}", fetch="all")
+
+        assert len(result) == expected_row_count, f"{relation_name}:{pformat(result)}"
+
+    @mock.patch.dict(os.environ, {"DBT_EXPERIMENTAL_SAMPLE_MODE": "True"})
+    def test_sample_mode(self, project) -> None:
+        _ = run_dbt(["run"])
+        self.assert_row_count(
+            project=project,
+            relation_name="model_that_samples_input_sql",
+            expected_row_count=3,
+        )
+
+        _ = run_dbt(["run", "--sample=1 day"])
+        self.assert_row_count(
+            project=project,
+            relation_name="model_that_samples_input_sql",
+            expected_row_count=2,
+        )
diff --git a/tests/functional/adapter/unit_testing/test_unit_testing.py b/tests/functional/adapter/unit_testing/test_unit_testing.py
@@ -56,7 +56,7 @@ def data_types(self):
             ["array[timestamp '2019-01-01']", "['2019-01-01 00:00:00.000']"],
 
             # Binary
-            ["cast('abc' as binary)", "YWJj"],
+            ["cast('abc' as varbinary)", "YWJj"],
 
             # Intervals
             ["interval '2' year", "\"'2' year\""],
