Merge pull request #929 from InfuseAI/feature/sc-32598/piperider-support-dbt-1-7

[Feature] Support dbt 1.7

Author: wcchang1115

.github/workflows/dbt-compatible-tests.yaml

@@ -16,7 +16,7 @@ jobs:
       fail-fast: false
       matrix:
         version: [ '3.8', '3.9', '3.10', '3.11' ]
-        dbt: [ ">=1.3,<1.4", ">=1.4,<1.5", ">=1.5,<1.6", ">=1.6,<1.7" ]
+        dbt: [ ">=1.3,<1.4", ">=1.4,<1.5", ">=1.5,<1.6", ">=1.6,<1.7", ">=1.7,<1.8" ]
 
     steps:
       - uses: actions/checkout@v2

piperider_cli/dbt/list_task.py

@@ -34,7 +34,7 @@ def load_full_manifest(target_path: str, project_dir: str = None):
     register_adapter(runtime_config)
 
     v = dbt_version
-    if v == '1.5' or v == '1.6':
+    if v == '1.5' or v == '1.6' or v == '1.7':
         return ManifestLoader.get_full_manifest(
             runtime_config, write_perf_info=False
         )
@@ -56,7 +56,7 @@ def load_manifest(manifest: Dict):
     if v == '1.5':
         return _load_manifest_version_15(manifest)
 
-    if v == '1.6':
+    if v == '1.6' or v == '1.7':
         return _load_manifest_version_16(manifest)
 
     raise NotImplementedError(f'dbt-core version: {v} is not supported')
@@ -114,12 +114,7 @@ def patched_get_manifest_schema_version(dct: dict) -> int:
             return int(match.group(1))
         raise ValueError("Manifest doesn't have schema version")
 
-    import dbt.contracts.graph.manifest
-    origin_function = dbt.contracts.graph.manifest.get_manifest_schema_version
-    dbt.contracts.graph.manifest.get_manifest_schema_version = patched_get_manifest_schema_version
-
     result = WritableManifest.upgrade_schema_version(data)
-    dbt.contracts.graph.manifest.get_manifest_schema_version = origin_function
     return result
 
 
@@ -265,7 +260,7 @@ def _get_v15_runtime_config(flags):
     setattr(flags, "target", None)
 
     v = dbt_version
-    if v == '1.5' or v == '1.6':
+    if v == '1.5' or v == '1.6' or v == '1.7':
         return _get_v15_runtime_config(flags)
     elif v == '1.4':
         return _get_v14_runtime_config(flags)
@@ -362,6 +357,13 @@ def has_field(field_name):
         if has_field('packages_specified_path'):
             data['packages_specified_path'] = "packages.yml"
 
+        # dbt 1.7
+        if has_field('semantic_models'):
+            data['semantic_models'] = {}
+
+        if has_field('saved_queries'):
+            data['saved_queries'] = {}
+
         super().__init__(args=None, **data)
 
     def validate(self):

piperider_cli/dbtutil.py

@@ -468,20 +468,39 @@ def _create_metric(name, filter=None, alias=None, root_name=None):
             primary_entity = None
             metric_filter = []
             if metric.get('filter') is not None:
-                f = get_metric_filter(root_name, metric.get('filter'))
-                if f is not None:
-                    metric_filter.append(f)
+                # in dbt 1.7, there's an additional layer of 'where_filters'
+                if 'where_filters' in metric.get('filter'):
+                    for where_filter in metric.get('filter').get('where_filters', []):
+                        f = get_metric_filter(root_name, where_filter)
+                        if f is not None:
+                            metric_filter.append(f)
+                        else:
+                            statistics.add_field_one('nosupport')
+                            return None
                 else:
-                    statistics.add_field_one('nosupport')
-                    return None
+                    f = get_metric_filter(root_name, metric.get('filter'))
+                    if f is not None:
+                        metric_filter.append(f)
+                    else:
+                        statistics.add_field_one('nosupport')
+                        return None
 
             if filter is not None:
-                f = get_metric_filter(root_name, filter)
-                if f is not None:
-                    metric_filter.append(f)
+                if 'where_filters' in filter:
+                    for where_filter in filter.get('where_filters', []):
+                        f = get_metric_filter(root_name, where_filter)
+                        if f is not None:
+                            metric_filter.append(f)
+                        else:
+                            statistics.add_field_one('nosupport')
+                            return None
                 else:
-                    statistics.add_field_one('nosupport')
-                    return None
+                    f = get_metric_filter(root_name, filter)
+                    if f is not None:
+                        metric_filter.append(f)
+                    else:
+                        statistics.add_field_one('nosupport')
+                        return None
 
             nodes = metric.get('depends_on').get('nodes', [])
             depends_on = nodes[0]

setup.py

@@ -45,7 +45,7 @@ def _get_version():
           'requests>=2.28.1',
           'requests_toolbelt>=0.9.1',
           'deepmerge',
-          'dbt-core>=1.3,<1.7'
+          'dbt-core>=1.3'
       ],
       tests_require=['pytest'],
       extras_require={

tests/mock_dbt_data/compatible/dbt-duckdb-1.7.0-manifest.json

@@ -61,6 +61,13 @@ def base_run_1_6(self):
     def target_run_1_6(self):
         return self.run_object("jaffle_shop_target_1_6.json")
 
+    def base_run_1_7(self):
+        return self.run_object("jaffle_shop_base_1_7.json")
+
+    def target_run_1_7(self):
+        return self.run_object("jaffle_shop_target_1_7.json")
+
+
     def base_31587(self):
         return self.run_object("sc-31587-base.json")
 
@@ -187,6 +194,60 @@ def test_list_all_resources_16(self):
 
         self.assertDbtResources(expected, all_results)
 
+    @pytest.mark.skipif(dbt_version < 'v1.7', reason='skip manifest test before dbt-core 1.7')
+    def test_list_all_resources_17(self):
+        expected = [
+            "metric.jaffle_shop.average_order_amount",
+            "metric.jaffle_shop.expenses",
+            "metric.jaffle_shop.profit",
+            "metric.jaffle_shop.revenue",
+            "model.jaffle_shop.int_customer_order_history_joined",
+            "model.jaffle_shop.int_order_payments_pivoted",
+            "model.jaffle_shop.metricflow_time_spine",
+            "model.jaffle_shop.orders",
+            "model.jaffle_shop.stg_customers",
+            "model.jaffle_shop.stg_orders",
+            "model.jaffle_shop.stg_payments",
+            "seed.jaffle_shop.raw_customers",
+            "seed.jaffle_shop.raw_orders",
+            "seed.jaffle_shop.raw_payments",
+            "semantic_model.jaffle_shop.orders",
+            "test.jaffle_shop.accepted_values_int_order_payments_pivoted_status__placed__shipped__completed__return_pending__returned.0ccdff53e8",
+            "test.jaffle_shop.accepted_values_orders_status__placed__shipped__completed__return_pending__returned.be6b5b5ec3",
+            "test.jaffle_shop.accepted_values_stg_orders_status__placed__shipped__completed__return_pending__returned.080fb20aad",
+            "test.jaffle_shop.accepted_values_stg_payments_payment_method__credit_card__coupon__bank_transfer__gift_card.3c3820f278",
+            "test.jaffle_shop.not_null_int_customer_order_history_joined_customer_id.5eeb8cdf92",
+            "test.jaffle_shop.not_null_int_order_payments_pivoted_amount.b7598e0e3b",
+            "test.jaffle_shop.not_null_int_order_payments_pivoted_bank_transfer_amount.1a9e62933b",
+            "test.jaffle_shop.not_null_int_order_payments_pivoted_coupon_amount.2532b538c2",
+            "test.jaffle_shop.not_null_int_order_payments_pivoted_credit_card_amount.ae9c42d967",
+            "test.jaffle_shop.not_null_int_order_payments_pivoted_customer_id.3db59c6de4",
+            "test.jaffle_shop.not_null_int_order_payments_pivoted_gift_card_amount.710d789cc0",
+            "test.jaffle_shop.not_null_int_order_payments_pivoted_order_id.787ba994a8",
+            "test.jaffle_shop.not_null_orders_amount.106140f9fd",
+            "test.jaffle_shop.not_null_orders_bank_transfer_amount.7743500c49",
+            "test.jaffle_shop.not_null_orders_coupon_amount.ab90c90625",
+            "test.jaffle_shop.not_null_orders_credit_card_amount.d3ca593b59",
+            "test.jaffle_shop.not_null_orders_customer_id.c5f02694af",
+            "test.jaffle_shop.not_null_orders_gift_card_amount.413a0d2d7a",
+            "test.jaffle_shop.not_null_orders_order_id.cf6c17daed",
+            "test.jaffle_shop.not_null_stg_customers_customer_id.e2cfb1f9aa",
+            "test.jaffle_shop.not_null_stg_orders_order_id.81cfe2fe64",
+            "test.jaffle_shop.not_null_stg_payments_payment_id.c19cc50075",
+            "test.jaffle_shop.relationships_int_order_payments_pivoted_customer_id__customer_id__ref_int_customer_order_history_joined_.654a1aa35d",
+            "test.jaffle_shop.unique_int_customer_order_history_joined_customer_id.995635f7d9",
+            "test.jaffle_shop.unique_int_order_payments_pivoted_order_id.34a0f3307d",
+            "test.jaffle_shop.unique_orders_order_id.fed79b3a6e",
+            "test.jaffle_shop.unique_stg_customers_customer_id.c7614daada",
+            "test.jaffle_shop.unique_stg_orders_order_id.e3b841c71a",
+            "test.jaffle_shop.unique_stg_payments_payment_id.3744510712"
+        ]
+
+        manifest = load_manifest(_load_manifest('dbt-duckdb-1.7.0-manifest.json'))
+        all_results = list_resources_unique_id_from_manifest(manifest)
+
+        self.assertDbtResources(expected, all_results)
+
     def test_compare_with_manifests(self):
         without_downstream = compare_models_between_manifests(
             self.base_manifest(), self.target_manifest()
@@ -209,9 +270,15 @@ def test_list_explicit_changes(self):
             expected = ["model.jaffle_shop.customers", "model.jaffle_shop.orders"]
             changes = c.list_explicit_changes()
             self.assertDbtResources(changes, expected)
-        else:
+        elif dbt_version < '1.7':
             c = GraphDataChangeSet(self.base_run_1_6(), self.target_run_1_6())
 
+            expected = ["model.jaffle_shop.orders"]
+            changes = c.list_explicit_changes()
+            self.assertDbtResources(changes, expected)
+        elif dbt_version < '1.8':
+            c = GraphDataChangeSet(self.base_run_1_7(), self.target_run_1_7())
+
             expected = ["model.jaffle_shop.orders"]
             changes = c.list_explicit_changes()
             self.assertDbtResources(changes, expected)
@@ -266,7 +333,7 @@ def test_list_changes_metrics_case(self):
                 changes,
                 ["metric.jaffle_shop.average_order_amount", "model.jaffle_shop.orders"],
             )
-        else:
+        elif dbt_version < '1.7':
             c = GraphDataChangeSet(self.base_run_1_6(), self.target_run_1_6())
             expected = ["model.jaffle_shop.orders"]
             changes = c.list_explicit_changes()
@@ -277,6 +344,12 @@ def test_list_changes_metrics_case(self):
                 changes,
                 ["metric.jaffle_shop.average_order_amount", "model.jaffle_shop.orders"],
             )
+        elif dbt_version < '1.8':
+            c = GraphDataChangeSet(self.base_run_1_7(), self.target_run_1_7())
+
+            expected = ["model.jaffle_shop.orders"]
+            changes = c.list_explicit_changes()
+            self.assertDbtResources(changes, expected)
 
     @unittest.skipIf(
         dbt_version < version.parse("1.4"),

tests/mock_dbt_data/jaffle_shop_base_1_7.json

@@ -34,6 +34,11 @@ def manifest_from_1_6():
     return _load_manifest('dbt-duckdb-1.6.0-manifest.json')
 
 
+@pytest.fixture()
+def manifest_from_1_7():
+    return _load_manifest('dbt-duckdb-1.7.0-manifest.json')
+
+
 def check_manifest_type(m):
     from dbt.contracts.graph.manifest import Manifest, WritableManifest
     assert isinstance(m, Manifest) or isinstance(m, WritableManifest)
@@ -62,6 +67,12 @@ def test_load_manifest_10(manifest_from_1_6):
     check_manifest_type(m)
 
 
+@pytest.mark.skipif(dbt_version < version.parse('v1.7'), reason='skip manifest test before dbt-core 1.7')
+def test_load_manifest_11(manifest_from_1_7):
+    m = load_manifest(manifest_from_1_7)
+    check_manifest_type(m)
+
+
 def test_log_functions():
     with disable_dbt_compile_stats():
         # make sure this not breaking in all dbt versions

tests/mock_dbt_data/jaffle_shop_target_1_7.json

@@ -252,6 +252,30 @@ def test_get_dbt_state_metrics_16(self, _get_state_manifest):
         metrics = dbtutil.get_dbt_state_metrics_16(self.dbt_state_dir, dbt_tag=None, dbt_resources=None)
         self.assertEqual(len(metrics), 0)
 
+    @pytest.mark.skipif(dbt_version < '1.7', reason="only for dbt 1.7")
+    @mock.patch('piperider_cli.dbtutil._get_state_manifest')
+    def test_get_dbt_state_metrics_17(self, _get_state_manifest):
+        _get_state_manifest.return_value = _load_manifest('dbt-duckdb-1.7.0-manifest.json')
+        metrics = dbtutil.get_dbt_state_metrics_16(self.dbt_state_dir, dbt_tag=None, dbt_resources=None)
+
+        self.assertEqual(len(metrics), 4)
+
+        # average_order_amount
+        self.assertEqual(metrics[0].model.table, 'orders')
+        self.assertEqual(metrics[0].model.timestamp, 'order_date')
+        self.assertEqual(metrics[0].model.expression, 'amount')
+
+        # profit
+        self.assertEqual(metrics[1].model, None)
+        self.assertEqual(metrics[1].calculation_method, 'derived')
+        self.assertEqual(metrics[1].expression, 'revenue - expenses')
+
+        # expenses
+        self.assertEqual(metrics[2].name, 'expenses')
+
+        # revenue
+        self.assertEqual(metrics[3].calculation_method, 'sum')
+
     @mock.patch('pathlib.Path.cwd',
                 return_value=Path(os.path.join(os.path.dirname(__file__), 'mock_dbt_project', 'dir_1', 'dir_2')))
     def test_search_dbt_project_path(self, *args):
@@ -284,7 +308,9 @@ def test_is_ready(self):
     def test_load_dbt_resources(self, get_dbt_manifest):
         v = dbt_version
         target_path = os.path.join(os.path.dirname(__file__), 'mock_dbt_data')
-        if v == '1.6':
+        if v == '1.7':
+            get_dbt_manifest.return_value = _load_manifest('dbt-duckdb-1.7.0-manifest.json')
+        elif v == '1.6':
             get_dbt_manifest.return_value = _load_manifest('dbt-duckdb-1.6.0-manifest.json')
         elif v == '1.5':
             get_dbt_manifest.return_value = _load_manifest('dbt-duckdb-1.5.1-manifest.json')