Add tests

Signed-off-by: Wei-Chun, Chang <[email protected]>

Author: wcchang1115

.github/workflows/dbt-compatible-tests.yaml

@@ -16,7 +16,7 @@ jobs:
       fail-fast: false
       matrix:
         version: [ '3.8', '3.9', '3.10', '3.11' ]
-        dbt: [ ">=1.3,<1.4", ">=1.4,<1.5", ">=1.5,<1.6", ">=1.6,<1.7" ]
+        dbt: [ ">=1.3,<1.4", ">=1.4,<1.5", ">=1.5,<1.6", ">=1.6,<1.7", ">=1.7,<1.8" ]
 
     steps:
       - uses: actions/checkout@v2

tests/mock_dbt_data/compatible/dbt-duckdb-1.7.0-manifest.json

@@ -61,6 +61,13 @@ def base_run_1_6(self):
     def target_run_1_6(self):
         return self.run_object("jaffle_shop_target_1_6.json")
 
+    def base_run_1_7(self):
+        return self.run_object("jaffle_shop_base_1_7.json")
+
+    def target_run_1_7(self):
+        return self.run_object("jaffle_shop_target_1_7.json")
+
+
     def base_31587(self):
         return self.run_object("sc-31587-base.json")
 
@@ -187,6 +194,60 @@ def test_list_all_resources_16(self):
 
         self.assertDbtResources(expected, all_results)
 
+    @pytest.mark.skipif(dbt_version < 'v1.7', reason='skip manifest test before dbt-core 1.7')
+    def test_list_all_resources_17(self):
+        expected = [
+            "metric.jaffle_shop.average_order_amount",
+            "metric.jaffle_shop.expenses",
+            "metric.jaffle_shop.profit",
+            "metric.jaffle_shop.revenue",
+            "model.jaffle_shop.int_customer_order_history_joined",
+            "model.jaffle_shop.int_order_payments_pivoted",
+            "model.jaffle_shop.metricflow_time_spine",
+            "model.jaffle_shop.orders",
+            "model.jaffle_shop.stg_customers",
+            "model.jaffle_shop.stg_orders",
+            "model.jaffle_shop.stg_payments",
+            "seed.jaffle_shop.raw_customers",
+            "seed.jaffle_shop.raw_orders",
+            "seed.jaffle_shop.raw_payments",
+            "semantic_model.jaffle_shop.orders",
+            "test.jaffle_shop.accepted_values_int_order_payments_pivoted_status__placed__shipped__completed__return_pending__returned.0ccdff53e8",
+            "test.jaffle_shop.accepted_values_orders_status__placed__shipped__completed__return_pending__returned.be6b5b5ec3",
+            "test.jaffle_shop.accepted_values_stg_orders_status__placed__shipped__completed__return_pending__returned.080fb20aad",
+            "test.jaffle_shop.accepted_values_stg_payments_payment_method__credit_card__coupon__bank_transfer__gift_card.3c3820f278",
+            "test.jaffle_shop.not_null_int_customer_order_history_joined_customer_id.5eeb8cdf92",
+            "test.jaffle_shop.not_null_int_order_payments_pivoted_amount.b7598e0e3b",
+            "test.jaffle_shop.not_null_int_order_payments_pivoted_bank_transfer_amount.1a9e62933b",
+            "test.jaffle_shop.not_null_int_order_payments_pivoted_coupon_amount.2532b538c2",
+            "test.jaffle_shop.not_null_int_order_payments_pivoted_credit_card_amount.ae9c42d967",
+            "test.jaffle_shop.not_null_int_order_payments_pivoted_customer_id.3db59c6de4",
+            "test.jaffle_shop.not_null_int_order_payments_pivoted_gift_card_amount.710d789cc0",
+            "test.jaffle_shop.not_null_int_order_payments_pivoted_order_id.787ba994a8",
+            "test.jaffle_shop.not_null_orders_amount.106140f9fd",
+            "test.jaffle_shop.not_null_orders_bank_transfer_amount.7743500c49",
+            "test.jaffle_shop.not_null_orders_coupon_amount.ab90c90625",
+            "test.jaffle_shop.not_null_orders_credit_card_amount.d3ca593b59",
+            "test.jaffle_shop.not_null_orders_customer_id.c5f02694af",
+            "test.jaffle_shop.not_null_orders_gift_card_amount.413a0d2d7a",
+            "test.jaffle_shop.not_null_orders_order_id.cf6c17daed",
+            "test.jaffle_shop.not_null_stg_customers_customer_id.e2cfb1f9aa",
+            "test.jaffle_shop.not_null_stg_orders_order_id.81cfe2fe64",
+            "test.jaffle_shop.not_null_stg_payments_payment_id.c19cc50075",
+            "test.jaffle_shop.relationships_int_order_payments_pivoted_customer_id__customer_id__ref_int_customer_order_history_joined_.654a1aa35d",
+            "test.jaffle_shop.unique_int_customer_order_history_joined_customer_id.995635f7d9",
+            "test.jaffle_shop.unique_int_order_payments_pivoted_order_id.34a0f3307d",
+            "test.jaffle_shop.unique_orders_order_id.fed79b3a6e",
+            "test.jaffle_shop.unique_stg_customers_customer_id.c7614daada",
+            "test.jaffle_shop.unique_stg_orders_order_id.e3b841c71a",
+            "test.jaffle_shop.unique_stg_payments_payment_id.3744510712"
+        ]
+
+        manifest = load_manifest(_load_manifest('dbt-duckdb-1.7.0-manifest.json'))
+        all_results = list_resources_unique_id_from_manifest(manifest)
+
+        self.assertDbtResources(expected, all_results)
+
     def test_compare_with_manifests(self):
         without_downstream = compare_models_between_manifests(
             self.base_manifest(), self.target_manifest()
@@ -209,9 +270,15 @@ def test_list_explicit_changes(self):
             expected = ["model.jaffle_shop.customers", "model.jaffle_shop.orders"]
             changes = c.list_explicit_changes()
             self.assertDbtResources(changes, expected)
-        else:
+        elif dbt_version < '1.7':
             c = GraphDataChangeSet(self.base_run_1_6(), self.target_run_1_6())
 
+            expected = ["model.jaffle_shop.orders"]
+            changes = c.list_explicit_changes()
+            self.assertDbtResources(changes, expected)
+        elif dbt_version < '1.8':
+            c = GraphDataChangeSet(self.base_run_1_7(), self.target_run_1_7())
+
             expected = ["model.jaffle_shop.orders"]
             changes = c.list_explicit_changes()
             self.assertDbtResources(changes, expected)
@@ -266,7 +333,7 @@ def test_list_changes_metrics_case(self):
                 changes,
                 ["metric.jaffle_shop.average_order_amount", "model.jaffle_shop.orders"],
             )
-        else:
+        elif dbt_version < '1.7':
             c = GraphDataChangeSet(self.base_run_1_6(), self.target_run_1_6())
             expected = ["model.jaffle_shop.orders"]
             changes = c.list_explicit_changes()
@@ -277,6 +344,12 @@ def test_list_changes_metrics_case(self):
                 changes,
                 ["metric.jaffle_shop.average_order_amount", "model.jaffle_shop.orders"],
             )
+        elif dbt_version < '1.8':
+            c = GraphDataChangeSet(self.base_run_1_7(), self.target_run_1_7())
+
+            expected = ["model.jaffle_shop.orders"]
+            changes = c.list_explicit_changes()
+            self.assertDbtResources(changes, expected)
 
     @unittest.skipIf(
         dbt_version < version.parse("1.4"),

tests/mock_dbt_data/jaffle_shop_base_1_7.json

@@ -34,6 +34,11 @@ def manifest_from_1_6():
     return _load_manifest('dbt-duckdb-1.6.0-manifest.json')
 
 
+@pytest.fixture()
+def manifest_from_1_7():
+    return _load_manifest('dbt-duckdb-1.7.0-manifest.json')
+
+
 def check_manifest_type(m):
     from dbt.contracts.graph.manifest import Manifest, WritableManifest
     assert isinstance(m, Manifest) or isinstance(m, WritableManifest)
@@ -62,6 +67,12 @@ def test_load_manifest_10(manifest_from_1_6):
     check_manifest_type(m)
 
 
+@pytest.mark.skipif(dbt_version < version.parse('v1.7'), reason='skip manifest test before dbt-core 1.7')
+def test_load_manifest_11(manifest_from_1_7):
+    m = load_manifest(manifest_from_1_7)
+    check_manifest_type(m)
+
+
 def test_log_functions():
     with disable_dbt_compile_stats():
         # make sure this not breaking in all dbt versions

tests/mock_dbt_data/jaffle_shop_target_1_7.json

@@ -252,6 +252,30 @@ def test_get_dbt_state_metrics_16(self, _get_state_manifest):
         metrics = dbtutil.get_dbt_state_metrics_16(self.dbt_state_dir, dbt_tag=None, dbt_resources=None)
         self.assertEqual(len(metrics), 0)
 
+    @pytest.mark.skipif(dbt_version < '1.7', reason="only for dbt 1.7")
+    @mock.patch('piperider_cli.dbtutil._get_state_manifest')
+    def test_get_dbt_state_metrics_17(self, _get_state_manifest):
+        _get_state_manifest.return_value = _load_manifest('dbt-duckdb-1.7.0-manifest.json')
+        metrics = dbtutil.get_dbt_state_metrics_16(self.dbt_state_dir, dbt_tag=None, dbt_resources=None)
+
+        self.assertEqual(len(metrics), 4)
+
+        # average_order_amount
+        self.assertEqual(metrics[0].model.table, 'orders')
+        self.assertEqual(metrics[0].model.timestamp, 'order_date')
+        self.assertEqual(metrics[0].model.expression, 'amount')
+
+        # profit
+        self.assertEqual(metrics[1].model, None)
+        self.assertEqual(metrics[1].calculation_method, 'derived')
+        self.assertEqual(metrics[1].expression, 'revenue - expenses')
+
+        # expenses
+        self.assertEqual(metrics[2].name, 'expenses')
+
+        # revenue
+        self.assertEqual(metrics[3].calculation_method, 'sum')
+
     @mock.patch('pathlib.Path.cwd',
                 return_value=Path(os.path.join(os.path.dirname(__file__), 'mock_dbt_project', 'dir_1', 'dir_2')))
     def test_search_dbt_project_path(self, *args):
@@ -284,7 +308,9 @@ def test_is_ready(self):
     def test_load_dbt_resources(self, get_dbt_manifest):
         v = dbt_version
         target_path = os.path.join(os.path.dirname(__file__), 'mock_dbt_data')
-        if v == '1.6':
+        if v == '1.7':
+            get_dbt_manifest.return_value = _load_manifest('dbt-duckdb-1.7.0-manifest.json')
+        elif v == '1.6':
             get_dbt_manifest.return_value = _load_manifest('dbt-duckdb-1.6.0-manifest.json')
         elif v == '1.5':
             get_dbt_manifest.return_value = _load_manifest('dbt-duckdb-1.5.1-manifest.json')

tests/test_dbt_integeration.py

@@ -10,8 +10,8 @@ wheel_build_env = .pkg
 
 deps =
     pytest>=6
-    dbt-core>=1.6,<1.7
-    dbt-duckdb>=1.6,<1.7
+    dbt-core
+    dbt-duckdb
     ruamel-0.17.21: ruamel.yaml<=0.17.21
     ruamel-0.17.22: ruamel.yaml>0.17.21,<0.18
 

tests/test_dbt_manifest_compatible.py

@@ -10,8 +10,8 @@ wheel_build_env = .pkg
 
 deps =
     pytest>=6
-    dbt-core>=1.6,<1.7
-    dbt-duckdb>=1.6,<1.7
+    dbt-core
+    dbt-duckdb
     sqlalchemy-14: sqlalchemy>=1.4,<2.0
     sqlalchemy-20: sqlalchemy>=2.0