fix: #33 fix source test results saving (#34)

il-dat · web-flow · commit a024f21fd8a5 · 2024-10-08T10:58:21.000+07:00
diff --git a/README.md b/README.md
@@ -24,11 +24,16 @@ The purpose of the dq tool is to make simple storing test results and visualisat
       version: [">=1.4.0", "<1.5.0"]
   ```
 
-- Configure schema in `dbt_project.yml` file:
+- (Optional) Configure schema in `dbt_project.yml` file:
 
   ```yml
-  vars:
-    # (optional) to create db table in the schema named as AUDIT, default to `target.schema` if not specified
+  models:
+    dq_tools:
+      # +database: DQ
+      +schema: AUDIT
+      
+  vars: # (TO BE DEPRECATED)
+    # dbt_dq_tool_database: DQ
     dbt_dq_tool_schema: AUDIT
   ```
 
@@ -75,7 +80,7 @@ models:
   - name: my_model
     columns:
       - name: my_column
-        tests:
+        data_tests:
           - dq_tools.unique_where_db:
               kpi_category: MyKPI # not recommended
 ```
@@ -88,22 +93,22 @@ models:
   - create metrics views
   - set up looker dashboard
 
-- STEP 2 - define dbt tests:
+- STEP 2 - define dbt data_tests:
   Define tests following the description in the package documentation.
 
   ```yaml
   models:
   - name: dim_customers
     description: This table has basic information about a customer, as well as some derived facts based on a customer's orders
-    tests:
+    data_tests:
       - dq_tools.equal_rowcount_where_db:
           compare_model: ref('stg_customers')
           where: customer_id > 50
           compare_model_where: customer_id > 50
     columns:
       - name: customer_id
         description: This is a unique identifier for a customer
-        tests:
+        data_tests:
           - dq_tools.unique_where_db
           - dq_tools.not_null_where_db
   ```
@@ -321,7 +326,7 @@ models:
   - name: my_model
     columns:
       - name: id
-        tests:
+        data_tests:
           - dq_tools.not_null_where_db:
               where: "_deleted = false"
               severity_level: error
@@ -347,7 +352,7 @@ models:
   - name: model_name
     columns:
       - name: id
-        tests:
+        data_tests:
           - dq_tools.relationships_where_db:
               to: ref('other_model_name')
               field: client_id
@@ -376,7 +381,7 @@ models:
   - name: my_model
     columns:
       - name: id
-        tests:
+        data_tests:
           - dq_tools.unique_where_db:
               where: "_deleted = false"
               severity_level: error
@@ -400,7 +405,7 @@ version: 2
 
 models:
   - name: model_name
-    tests:
+    data_tests:
       - dq_tools.recency_db:
           datepart: day
           field: created_at
@@ -426,7 +431,7 @@ version: 2
 
 models:
   - name: model_name
-    tests:
+    data_tests:
       - dq_tools.expression_is_true_db:
           expression: "col_a + col_b = total"
           kpi_category: Validity
@@ -450,7 +455,7 @@ version: 2
 
 models:
   - name: model_name
-    tests:
+    data_tests:
       - dq_tools.accepted_values_where_db:
           values: [value1, value2]
           severity_level: warn
@@ -474,7 +479,7 @@ version: 2
 
 models:
   - name: model_name
-    tests:
+    data_tests:
       - dq_tools.equal_rowcount_where_db:
           compare_model: some_other_model
           where: "_deleted = false"
@@ -500,7 +505,7 @@ version: 2
 
 models:
   - name: model_name
-    tests:
+    data_tests:
       - dq_tools.equality_where_db:
           compare_model: some_other_model
           compare_columns:
diff --git a/integration_tests/dbt_project.yml b/integration_tests/dbt_project.yml
@@ -50,5 +50,6 @@ seeds:
 
 models:
   dq_tools:
+    +schema: dq_raw
     03_mart:
       +schema: dq_tools_mart
diff --git a/integration_tests/models/artifacts/graph/graph.yml b/integration_tests/models/artifacts/graph/graph.yml
@@ -2,6 +2,6 @@ version: 2
 
 models:
   - name: test__get_tables_from_graph
-    tests:
+    data_tests:
       - dq_tools.expression_is_true_db:
           expression: actual = expected
diff --git a/integration_tests/models/artifacts/test/utilities/_general/_general.yml b/integration_tests/models/artifacts/test/utilities/_general/_general.yml
@@ -8,18 +8,18 @@ sources:
 
 models:
   - name: test__get_test_model
-    tests:
+    data_tests:
       - dq_tools.expression_is_true_db:
           expression: actual = expected
   - name: test__get_test_name
-    tests:
+    data_tests:
       - dq_tools.expression_is_true_db:
           expression: actual = expected
   - name: test__get_test_type
-    tests:
+    data_tests:
       - dq_tools.expression_is_true_db:
           expression: actual = expected
   - name: test__get_where_subquery
-    tests:
+    data_tests:
       - dq_tools.expression_is_true_db:
           expression: actual = expected
diff --git a/integration_tests/models/artifacts/test/utilities/column/column.yml b/integration_tests/models/artifacts/test/utilities/column/column.yml
@@ -2,10 +2,10 @@ version: 2
 
 models:
   - name: test__get_column_name
-    tests:
+    data_tests:
       - dq_tools.expression_is_true_db:
           expression: actual = expected
   - name: test__get_to_column_name
-    tests:
+    data_tests:
       - dq_tools.expression_is_true_db:
           expression: actual = expected
diff --git a/integration_tests/models/artifacts/test/utilities/dq/dq.yml b/integration_tests/models/artifacts/test/utilities/dq/dq.yml
@@ -2,10 +2,10 @@ version: 2
 
 models:
   - name: test__get_dq_issue_type
-    tests:
+    data_tests:
       - dq_tools.expression_is_true_db:
           expression: actual = expected
   - name: test__get_kpi_categorize
-    tests:
+    data_tests:
       - dq_tools.expression_is_true_db:
           expression: actual = expected
diff --git a/integration_tests/models/artifacts/test/utilities/relation/relation.yml b/integration_tests/models/artifacts/test/utilities/relation/relation.yml
@@ -8,18 +8,18 @@ sources:
 
 models:
   - name: test__get_relation_ref
-    tests:
+    data_tests:
       - dq_tools.expression_is_true_db:
           expression: actual = expected
   - name: test__get_relation_source
-    tests:
+    data_tests:
       - dq_tools.expression_is_true_db:
           expression: actual = expected
   - name: test__get_relation
-    tests:
+    data_tests:
       - dq_tools.expression_is_true_db:
           expression: actual = expected
   - name: test__get_to_relation
-    tests:
+    data_tests:
       - dq_tools.expression_is_true_db:
           expression: actual = expected
diff --git a/integration_tests/models/artifacts/test/utilities/utilites.yml b/integration_tests/models/artifacts/test/utilities/utilites.yml
@@ -2,10 +2,10 @@ version: 2
 
 models:
   - name: test__get_test_coverage_exclusion
-    tests:
+    data_tests:
       - dq_tools.expression_is_true_db:
           expression: actual = expected
   - name: test__get_test_coverage_exclusion_sql
-    tests:
+    data_tests:
       - dq_tools.expression_is_true_db:
           expression: actual = expected
diff --git a/integration_tests/models/general/test_failures.yml b/integration_tests/models/general/test_failures.yml
@@ -2,12 +2,12 @@ version: 2
 
 models:
   - name: test_failures
-    tests:
+    data_tests:
       - dq_tools.expression_is_true_db:
           expression: 1 / 0 # intentional error - this shouldn't be captured into result log
     columns:
       - name: col_non_zero
-        tests:
+        data_tests:
           - dq_tools.expression_is_true_db:
               expression: col_non_zero = 0 # intentional failure to check test result log
               severity: error
diff --git a/integration_tests/models/generic_tests/data_test_accepted_values.yml b/integration_tests/models/generic_tests/data_test_accepted_values.yml
@@ -4,7 +4,7 @@ seeds:
   - name: data_test_accepted_values
     columns:
       - name: yesno
-        tests:
+        data_tests:
           - dq_tools.accepted_values_where_db:
               values: ['Y','N','N/A']
           - dq_tools.accepted_values_where_db:
diff --git a/integration_tests/models/generic_tests/data_test_equal_rowcount.yml b/integration_tests/models/generic_tests/data_test_equal_rowcount.yml
@@ -2,7 +2,7 @@ version: 2
 
 seeds:
   - name: data_test_equal_rowcount_a
-    tests:
+    data_tests:
       - dq_tools.equal_rowcount_where_db:
           compare_model: ref('data_test_equal_rowcount_b')
       - dq_tools.equal_rowcount_where_db:
@@ -14,7 +14,7 @@ seeds:
           compare_model: (select 1 as id)
 
   - name: data_test_equal_rowcount_b
-    tests:
+    data_tests:
       - dq_tools.equal_rowcount_where_db:
           where: id = -1
           compare_model: (select 1 as id)
diff --git a/integration_tests/models/generic_tests/data_test_equality.yml b/integration_tests/models/generic_tests/data_test_equality.yml
@@ -2,7 +2,7 @@ version: 2
 
 seeds:
   - name: data_test_equality_a
-    tests:
+    data_tests:
       - dq_tools.equality_where_db:
           compare_model: ref('data_test_equality_b')
           compare_columns: ['id']
diff --git a/integration_tests/models/generic_tests/data_test_expression_is_true.yml b/integration_tests/models/generic_tests/data_test_expression_is_true.yml
@@ -2,21 +2,21 @@ version: 2
 
 seeds:
   - name: data_test_expression_is_true
-    tests:
+    data_tests:
       - dq_tools.expression_is_true_db:
           where: id != -1
           expression: (age is not null and name is not null)
     columns:
       - name: id
-        tests:
+        data_tests:
           - dq_tools.expression_is_true_db:
               expression: id is not null
       - name: name
-        tests:
+        data_tests:
           - dq_tools.expression_is_true_db:
               where: id != -1
               expression: length(name) = 1
       - name: age
-        tests:
+        data_tests:
           - dq_tools.expression_is_true_db:
               expression: age > 0
diff --git a/integration_tests/models/generic_tests/data_test_not_null.yml b/integration_tests/models/generic_tests/data_test_not_null.yml
@@ -4,9 +4,9 @@ seeds:
   - name: data_test_not_null
     columns:
       - name: id
-        tests:
+        data_tests:
           - dq_tools.not_null_where_db
       - name: name
-        tests:
+        data_tests:
           - dq_tools.not_null_where_db:
               where: id not in (-1, 2)
diff --git a/integration_tests/models/generic_tests/data_test_relationships.yml b/integration_tests/models/generic_tests/data_test_relationships.yml
@@ -4,14 +4,14 @@ seeds:
   - name: data_test_relationships_b
     columns:
       - name: b_id
-        tests:
+        data_tests:
           - dq_tools.relationships_where_db:
               to: ref('data_test_relationships_a')
               field: a_id
   - name: data_test_relationships_a
     columns:
       - name: a_id
-        tests:
+        data_tests:
           - dq_tools.relationships_where_db:
               from_condition: a_id < 0
               to: ref('data_test_relationships_b')
diff --git a/integration_tests/models/generic_tests/data_test_unique.yml b/integration_tests/models/generic_tests/data_test_unique.yml
@@ -2,10 +2,10 @@ version: 2
 
 seeds:
   - name: data_test_unique
-    tests:
+    data_tests:
       - dq_tools.unique_where_db:
           column_name: concat(customer_id, location)
     columns:
       - name: id
-        tests:
+        data_tests:
           - dq_tools.unique_where_db
diff --git a/integration_tests/models/generic_tests/test_recency.yml b/integration_tests/models/generic_tests/test_recency.yml
@@ -2,7 +2,7 @@ version: 2
 
 models:
   - name: test_recency
-    tests:
+    data_tests:
       - dq_tools.recency_db:
           field: today
           datepart: day
@@ -13,7 +13,7 @@ models:
             kpi_category: Validity
     columns:
       - name: today
-        tests:
+        data_tests:
           - dq_tools.recency_db:
               datepart: hour
               interval: 24
diff --git a/integration_tests/models/source_dq_tools_test.yml b/integration_tests/models/source_dq_tools_test.yml
diff --git a/macros/artifacts/test/utilities/relation/__get_relation_source.sql b/macros/artifacts/test/utilities/relation/__get_relation_source.sql
diff --git a/models/03_mart/data-quality-score/bi_column_analysis.yml b/models/03_mart/data-quality-score/bi_column_analysis.yml
diff --git a/models/03_mart/data-quality-score/bi_dq_metrics.yml b/models/03_mart/data-quality-score/bi_dq_metrics.yml
diff --git a/models/03_mart/test-coverage/test_coverage.yml b/models/03_mart/test-coverage/test_coverage.yml
diff --git a/poetry.lock b/poetry.lock
diff --git a/pyproject.toml b/pyproject.toml
