diff --git a/.github/scripts/compare_test_failures.sh b/.github/scripts/compare_test_failures.sh index 159ccf02..afb82f9a 100644 --- a/.github/scripts/compare_test_failures.sh +++ b/.github/scripts/compare_test_failures.sh @@ -21,7 +21,7 @@ echo "Comparing actual test failures with expected failures from: $expected_fail shopt -s globstar # Extract actual failures from test reports -actual_failures=$(grep -E "(FAILED tests|ERROR tests)" reports/**/*.txt | awk '{print $2}' | sort) +actual_failures=$(grep -E "(FAILED tests|ERROR tests)" reports/**/*.txt | awk '{print $2}' | sort -u) # Read expected failures expected_failures=$(sort "$expected_failures_file") @@ -49,7 +49,7 @@ if [ -n "$unexpected_failures" ]; then fi if [ -n "$missing_failures" ]; then - echo "::warning::Expected test failures that did not occur (they passed):" + echo "⚠️ Expected test failures that did not occur (they passed):" echo "$missing_failures" exit_code=1 fi diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a362290f..c68d1206 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -112,7 +112,7 @@ jobs: run: | mkdir reports/ report_file="reports/${{ env.sanitized_test_dir }}.txt" - pytest ${{ matrix.test_dir }} | tee $report_file + pytest -s ${{ matrix.test_dir }} | tee $report_file exit ${PIPESTATUS[0]} - name: Upload test report as artifact diff --git a/CHANGELOG.md b/CHANGELOG.md index 6c029ff1..d3583cc9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,12 +1,21 @@ -# dbt-dremio v1.9.1 +# dbt-dremio v1.10.0 ## Changes -- [#299](https://github.com/dremio/dbt-dremio/pull/299) Enhance persist_docs macro to wrap model and column metadata (including descriptions, tags and tests) into a Markdown wiki for Dremio. +- Updated dbt-dremio to match dbt-core v1.10 with sample mode +- Enhanced persist_docs macro to wrap model and column metadata (including descriptions, tags and tests) into a Markdown wiki for Dremio. - Refactored CI - Fixed tests for hooks and grants - Added Dremio Enterprise Catalog tests +## Features + +- [#310](https://github.com/dremio/dbt-dremio/pull/310) Sample mode + +## Dependency + +- Upgraded dbt-core to 1.10.0 and dbt-tests-adapter to 1.16.0 + # dbt-dremio v1.9.0 ## Changes diff --git a/README.md b/README.md index 2c6d3c71..578ecd3a 100644 --- a/README.md +++ b/README.md @@ -12,16 +12,16 @@ The `dbt-dremio` package contains all of the code enabling dbt to work with [Dre The dbt-dremio package supports both Dremio Cloud and Dremio Software (versions 22.0 and later). -## dbt-dremio version 1.9.0 +## dbt-dremio version 1.10.0 -Version 1.9.0 of the dbt-dremio adapter is compatible with dbt-core versions 1.9.*. +Version 1.10.0 of the dbt-dremio adapter is compatible with dbt-core versions 1.10.*. > Prior to version 1.1.0b, dbt-dremio was created and maintained by [Fabrice Etanchaud](https://github.com/fabrice-etanchaud) on [their GitHub repo](https://github.com/fabrice-etanchaud/dbt-dremio). Code for using Dremio REST APIs was originally authored by [Ryan Murray](https://github.com/rymurr). Contributors in this repo are credited for laying the groundwork and maintaining the adapter till version 1.0.6.5. The dbt-dremio adapter is maintained and distributed by Dremio starting with version 1.1.0b. ## Getting started - [Install dbt-dremio](https://docs.getdbt.com/reference/warehouse-setups/dremio-setup) - - Version 1.9.0 of dbt-dremio requires dbt-core >= 1.9.*. + - Version 1.10.0 of dbt-dremio requires dbt-core >= 1.10.*. - Read the [introduction](https://docs.getdbt.com/docs/introduction/) and [viewpoint](https://docs.getdbt.com/docs/about/viewpoint/) ## Join the dbt Community diff --git a/dbt/adapters/dremio/__version__.py b/dbt/adapters/dremio/__version__.py index b9b62836..51caacf1 100644 --- a/dbt/adapters/dremio/__version__.py +++ b/dbt/adapters/dremio/__version__.py @@ -9,4 +9,4 @@ # See the License for the specific language governing permissions and # limitations under the License. -version = "1.9.0" +version = "1.10.0" diff --git a/dbt/adapters/dremio/relation.py b/dbt/adapters/dremio/relation.py index 22b4a77c..6ab98733 100644 --- a/dbt/adapters/dremio/relation.py +++ b/dbt/adapters/dremio/relation.py @@ -18,6 +18,7 @@ BaseRelation, Policy, ComponentName, + EventTimeFilter, ) from typing import Optional, Tuple, Iterator @@ -45,6 +46,7 @@ class DremioRelation(BaseRelation): no_schema = "no_schema" format: Optional[str] = None format_clause: Optional[str] = None + event_time_filter: Optional[EventTimeFilter] = None def quoted_by_component(self, identifier, componentName): dot_char = "." @@ -85,3 +87,42 @@ def _render_iterator( ): # or key == ComponentName.Schema): path_part = self.quoted_by_component(path_part, key) yield key, path_part + + def _format_timestamp(self, timestamp_str: str) -> str: + """ + Convert timestamp with timezone info to Dremio-compatible format. + Removes timezone information and limits microseconds precision since Dremio + doesn't support timezone info in timestamp literals and has limited microsecond precision. + + Example: '2025-10-02 11:45:01.142708+00:00' -> '2025-10-02 11:45:01.142' + """ + if timestamp_str is None: + return timestamp_str + + # Remove timezone information (e.g., +00:00, -05:00, Z) + # This regex matches timezone patterns at the end of the string + timestamp_str = str(timestamp_str) + timestamp_str = re.sub(r'[+-]\d{2}:\d{2}$|Z$', '', timestamp_str) + + # Limit microseconds to 3 digits (milliseconds) as Dremio does not support full 6-digit microseconds + # Pattern: YYYY-MM-DD HH:MM:SS.ssssss -> YYYY-MM-DD HH:MM:SS.sss + timestamp_str = re.sub(r'(\.\d{3})\d{3}$', r'\1', timestamp_str) + + return timestamp_str + + # Override in order to apply _format_timestamp + def _render_event_time_filtered(self, event_time_filter: EventTimeFilter) -> str: + """ + Returns "" if start and end are both None + """ + filter = "" + start_ts = self._format_timestamp(event_time_filter.start) + end_ts = self._format_timestamp(event_time_filter.end) + if event_time_filter.start and event_time_filter.end: + filter = f"{event_time_filter.field_name} >= '{start_ts}' and {event_time_filter.field_name} < '{end_ts}'" + elif event_time_filter.start: + filter = f"{event_time_filter.field_name} >= '{start_ts}'" + elif event_time_filter.end: + filter = f"{event_time_filter.field_name} < '{end_ts}'" + + return filter diff --git a/dbt/include/dremio/macros/builtins/builtins.sql b/dbt/include/dremio/macros/builtins/builtins.sql index 64394c82..a965fcd8 100644 --- a/dbt/include/dremio/macros/builtins/builtins.sql +++ b/dbt/include/dremio/macros/builtins/builtins.sql @@ -24,7 +24,7 @@ limitations under the License.*/ and model.config.format is defined else none -%} {%- set format_clause = format_clause_from_node(model.config) if format is not none else none -%} - {%- set relation2 = api.Relation.create(database=relation.database, schema=relation.schema, identifier=relation.identifier, format=format, format_clause=format_clause, limit=relation.limit) -%} + {%- set relation2 = api.Relation.create(database=relation.database, schema=relation.schema, identifier=relation.identifier, format=format, format_clause=format_clause, limit=relation.limit, event_time_filter=relation.event_time_filter) -%} {{ return (relation2) }} {%- else -%} {{ return (relation) }} @@ -40,7 +40,7 @@ limitations under the License.*/ and source.external.format is defined else none -%} {%- set format_clause = format_clause_from_node(source.external) if format is not none else none -%} - {%- set relation2 = api.Relation.create(database=relation.database, schema=relation.schema, identifier=relation.identifier, format=format, format_clause=format_clause, limit=relation.limit) -%} + {%- set relation2 = api.Relation.create(database=relation.database, schema=relation.schema, identifier=relation.identifier, format=format, format_clause=format_clause, limit=relation.limit, event_time_filter=relation.event_time_filter) -%} {{ return (relation2) }} {%- else -%} {{ return (relation) }} diff --git a/dev_requirements.txt b/dev_requirements.txt index 03d891f3..4221ded6 100644 --- a/dev_requirements.txt +++ b/dev_requirements.txt @@ -3,8 +3,8 @@ Babel==2.12.1 betterproto==1.2.5 certifi==2023.7.22 charset-normalizer==3.1.0 -dbt-core==1.9.0 -dbt-tests-adapter==1.11.0 +dbt-core==1.10.0 +dbt-tests-adapter==1.16.0 python-dotenv==1.0.1 exceptiongroup==1.1.1 future==0.18.3 @@ -13,11 +13,9 @@ h2==4.1.0 hpack==4.0.0 hyperframe==6.0.1 iniconfig==2.0.0 -jsonschema==4.17.3 leather==0.3.4 MarkupSafe==2.1.2 msgpack==1.0.5 -multidict==6.0.4 parsedatetime==2.4 pip-licenses==4.1.0 pluggy==1.0.0 diff --git a/setup.py b/setup.py index 186ede64..2860805d 100644 --- a/setup.py +++ b/setup.py @@ -22,7 +22,7 @@ package_name = "dbt-dremio" -package_version = "1.9.0" +package_version = "1.10.0" description = """The Dremio adapter plugin for dbt""" @@ -37,9 +37,9 @@ packages=find_namespace_packages(include=["dbt", "dbt.*"]), include_package_data=True, install_requires=[ - "dbt-core>=1.9", - "dbt-common>=1.11,<2.0", - "dbt-adapters>=1.10.1, <2.0", + "dbt-core>=1.10", + "dbt-common>=1.27,<2.0", + "dbt-adapters>=1.16.1, <2.0", "requests>=2.31.0", ], classifiers=[ diff --git a/tests/functional/adapter/basic/test_concurrency.py b/tests/functional/adapter/basic/test_concurrency.py index 5f7ff9e0..a2d3f4b2 100644 --- a/tests/functional/adapter/basic/test_concurrency.py +++ b/tests/functional/adapter/basic/test_concurrency.py @@ -113,4 +113,4 @@ def test_concurrency(self, project): check_table_does_not_exist(project.adapter, "invalid") check_table_does_not_exist(project.adapter, "skip") - assert "PASS=5 WARN=0 ERROR=1 SKIP=1 TOTAL=7" in output + assert "PASS=5 WARN=0 ERROR=1 SKIP=1 NO-OP=0 TOTAL=7" in output diff --git a/tests/functional/adapter/dremio_specific/test_reflections.py b/tests/functional/adapter/dremio_specific/test_reflections.py index 1356cf97..c3ee6893 100644 --- a/tests/functional/adapter/dremio_specific/test_reflections.py +++ b/tests/functional/adapter/dremio_specific/test_reflections.py @@ -4,7 +4,6 @@ from dbt.adapters.dremio.api.parameters import ParametersBuilder from dbt.adapters.dremio.api.authentication import DremioAuthentication from dbt.tests.util import run_dbt -from pydantic.experimental.pipeline import transform view1_model = """ SELECT IncidntNum, Category, Descript, DayOfWeek, TO_DATE("SF_incidents2016.json"."Date", 'YYYY-MM-DD', 1) AS "Date", "SF_incidents2016.json"."Time" AS "Time", PdDistrict, Resolution, Address, X, Y, Location, PdId diff --git a/tests/functional/adapter/sample_mode/test_sample_mode.py b/tests/functional/adapter/sample_mode/test_sample_mode.py new file mode 100644 index 00000000..df5658ea --- /dev/null +++ b/tests/functional/adapter/sample_mode/test_sample_mode.py @@ -0,0 +1,79 @@ +import datetime +import os +from unittest import mock +import pytest + +from dbt.tests.adapter.sample_mode.test_sample_mode import ( + BaseSampleModeTest, +) +from dbt.tests.util import run_dbt +from tests.utils.util import BUCKET, relation_from_name +from pprint import pformat + +# Use UTC time to match dbt-core's sample mode window calculation +now = datetime.datetime.now(datetime.timezone.utc).replace(tzinfo=None) +twelve_hours_ago = now - datetime.timedelta(hours=12) +two_days_ago = now - datetime.timedelta(days=2) + +_input_model_sql = f""" +{{{{ config(materialized='table', event_time='event_time') }}}} +select 1 as id, cast('{two_days_ago.strftime('%Y-%m-%d %H:%M:%S')}' as timestamp) as event_time +UNION ALL +select 2 as id, cast('{twelve_hours_ago.strftime('%Y-%m-%d %H:%M:%S')}' as timestamp) as event_time +UNION ALL +select 3 as id, cast('{now.strftime('%Y-%m-%d %H:%M:%S')}' as timestamp) as event_time +""" + +class TestDremioSampleMode(BaseSampleModeTest): + @pytest.fixture(scope="class") + def input_model_sql(self) -> str: + """ + This is the SQL that defines the input model to be sampled, including any {{ config(..) }}. + event_time is a required configuration of this input + """ + return _input_model_sql + + # Override this fixture to set the proper root_path + @pytest.fixture(scope="class") + def dbt_profile_data( + self, unique_schema, dbt_profile_target, profiles_config_update + ): + profile = { + "test": { + "outputs": { + "default": {}, + }, + "target": "default", + }, + } + target = dbt_profile_target + target["schema"] = unique_schema + target["root_path"] = f"{BUCKET}.{unique_schema}" + profile["test"]["outputs"]["default"] = target + + if profiles_config_update: + profile.update(profiles_config_update) + return profile + + # Pasting the original function here so it uses our relation_from_name + def assert_row_count(self, project, relation_name: str, expected_row_count: int): + relation = relation_from_name(project.adapter, relation_name) + result = project.run_sql(f"select * from {relation}", fetch="all") + + assert len(result) == expected_row_count, f"{relation_name}:{pformat(result)}" + + @mock.patch.dict(os.environ, {"DBT_EXPERIMENTAL_SAMPLE_MODE": "True"}) + def test_sample_mode(self, project) -> None: + _ = run_dbt(["run"]) + self.assert_row_count( + project=project, + relation_name="model_that_samples_input_sql", + expected_row_count=3, + ) + + _ = run_dbt(["run", "--sample=1 day"]) + self.assert_row_count( + project=project, + relation_name="model_that_samples_input_sql", + expected_row_count=2, + ) diff --git a/tests/functional/adapter/unit_testing/test_unit_testing.py b/tests/functional/adapter/unit_testing/test_unit_testing.py index bb4704f0..7f4d76c0 100644 --- a/tests/functional/adapter/unit_testing/test_unit_testing.py +++ b/tests/functional/adapter/unit_testing/test_unit_testing.py @@ -56,7 +56,7 @@ def data_types(self): ["array[timestamp '2019-01-01']", "['2019-01-01 00:00:00.000']"], # Binary - ["cast('abc' as binary)", "YWJj"], + ["cast('abc' as varbinary)", "YWJj"], # Intervals ["interval '2' year", "\"'2' year\""],