Merge branch 'master' of github.com:elementary-data/dbt-data-reliability into ele-3925-add-quality-dimensions

Author: ofek1weiss

.github/workflows/test-warehouse.yml

@@ -84,7 +84,7 @@ jobs:
       - name: Setup Python
         uses: actions/setup-python@v4
         with:
-          python-version: "3.8"
+          python-version: "3.9"
           cache: "pip"
 
       - name: Install Spark requirements

README.md

@@ -34,7 +34,7 @@ Available as self-hosted or Cloud service with premium features.
 ```yml packages.yml
 packages:
   - package: elementary-data/elementary
-    version: 0.16.2
+    version: 0.16.3
     ## Docs: https://docs.elementary-data.com
 ```
 

dbt_project.yml

@@ -1,5 +1,5 @@
 name: "elementary"
-version: "0.16.2"
+version: "0.16.3"
 
 require-dbt-version: [">=1.0.0", "<2.0.0"]
 

integration_tests/tests/dbt_project.py

@@ -18,7 +18,7 @@
 _DEFAULT_VARS = {
     "disable_dbt_invocation_autoupload": True,
     "disable_dbt_artifacts_autoupload": True,
-    "disable_dbt_columns_autoupload": True,
+    "columns_upload_strategy": "none",
     "disable_run_results": True,
     "disable_freshness_results": True,
     "debug_logs": True,

integration_tests/tests/test_dbt_artifacts/test_columns.py

@@ -28,17 +28,17 @@
 @Parametrization.autodetect_parameters()
 @Parametrization.case(
     name="default",
-    only_with_description=None,
+    columns_upload_strategy=None,
     expected_columns=["with_description"],
 )
 @Parametrization.case(
     name="only_with_description",
-    only_with_description=True,
+    columns_upload_strategy="enriched_only",
     expected_columns=["with_description"],
 )
 @Parametrization.case(
     name="all",
-    only_with_description=False,
+    columns_upload_strategy="all",
     expected_columns=[
         "with_description",
         "without_description",
@@ -48,13 +48,11 @@
 )
 def test_flatten_table_columns(
     dbt_project: DbtProject,
-    only_with_description: Optional[bool],
+    columns_upload_strategy: Optional[str],
     expected_columns: List[str],
 ) -> None:
-    if only_with_description is not None:
-        dbt_project.dbt_runner.vars[
-            "upload_only_columns_with_descriptions"
-        ] = only_with_description
+    if columns_upload_strategy is not None:
+        dbt_project.dbt_runner.vars["columns_upload_strategy"] = columns_upload_strategy
     flattened_columns = json.loads(
         dbt_project.dbt_runner.run_operation(
             "elementary.flatten_table_columns", macro_args={"table_node": TABLE_NODE}

macros/edr/dbt_artifacts/upload_dbt_artifacts.sql

@@ -11,7 +11,7 @@
       }
     %}
 
-    {% if not elementary.get_config_var('disable_dbt_columns_autoupload') %}
+    {% if elementary.get_config_var('columns_upload_strategy') != 'none' %}
       {% do model_upload_func_map.update({"dbt_columns": elementary.upload_dbt_columns}) %}
     {% endif %}
 

macros/edr/dbt_artifacts/upload_dbt_columns.sql

@@ -35,7 +35,17 @@
 
     {% set flattened_columns = [] %}
     {% for column_node in column_nodes.values() %}
-        {% if not elementary.get_config_var('upload_only_columns_with_descriptions') or column_node.get('description') %}
+        {% set config_dict = elementary.safe_get_with_default(column_node, 'config', {}) %}
+        {% set config_meta_dict = elementary.safe_get_with_default(config_dict, 'meta') %}
+        {% set meta_dict = elementary.safe_get_with_default(column_node, 'meta', {}) %}
+        {% set has_meta = config_meta_dict or meta_dict | length > 0 %}
+
+        {% set config_tags = elementary.safe_get_with_default(config_dict, 'tags') %}
+        {% set global_tags = elementary.safe_get_with_default(column_node, 'tags') %}
+        {% set meta_tags = elementary.safe_get_with_default(meta_dict, 'tags') %}
+        {% set has_tags = config_tags or global_tags or meta_tags %}
+
+        {% if elementary.get_config_var('columns_upload_strategy') == 'all' or column_node.get('description') or has_meta or has_tags %}
             {% set flat_column = elementary.flatten_column(table_node, column_node) %}
             {% do flattened_columns.append(flat_column) %}
         {% endif %}

macros/edr/dbt_artifacts/upload_dbt_tests.sql

@@ -55,13 +55,18 @@
     {% set default_description = elementary.get_default_description(test_original_name, test_namespace) %}
 
     {% set config_meta_dict = elementary.safe_get_with_default(config_dict, 'meta', {}) %}
-    {% set meta_dict = {} %}
-    {% if default_description %}
-        {% set meta_dict =  {'description': default_description} %} 
-    {% endif %}
-    {% do meta_dict.update(elementary.safe_get_with_default(node_dict, 'meta', {})) %}
+    {% set meta_dict = elementary.safe_get_with_default(node_dict, 'meta', {}) %}
     {% do meta_dict.update(config_meta_dict) %}
 
+    {% set description = none %}
+    {% if dbt_version >= '1.9.0' and node_dict.get('description') %}
+        {% set description = node_dict.get('description') %}
+    {% elif meta_dict.get('description') %}
+        {% set description = meta_dict.pop('description') %}
+    {% elif default_description %}
+        {% set description = default_description %}
+    {% endif %}
+
     {% set config_tags = elementary.safe_get_with_default(config_dict, 'tags', []) %}
     {% set global_tags = elementary.safe_get_with_default(node_dict, 'tags', []) %}
     {% set meta_tags = elementary.safe_get_with_default(meta_dict, 'tags', []) %}
@@ -160,7 +165,7 @@
         'depends_on_macros': elementary.filter_none_and_sort(depends_on_dict.get('macros', [])),
         'depends_on_nodes': elementary.filter_none_and_sort(depends_on_dict.get('nodes', [])),
         'parent_model_unique_id': primary_test_model_id.data,
-        'description': meta_dict.get('description'),
+        'description': description,
         'name': node_dict.get('name'),
         'package_name': node_dict.get('package_name'),
         'type': elementary.get_test_sub_type(original_file_path, test_namespace),

macros/edr/system/system_utils/get_config_var.sql

@@ -32,8 +32,7 @@
     'disable_freshness_results': false,
     'disable_tests_results': false,
     'disable_dbt_artifacts_autoupload': false,
-    'disable_dbt_columns_autoupload': false,
-    'upload_only_columns_with_descriptions': true,
+    'columns_upload_strategy': 'enriched_only',
     'disable_dbt_invocation_autoupload': false,
     'disable_skipped_model_alerts': true,
     'disable_skipped_test_alerts': true,