Merge pull request #1870 from elementary-data/athena-integrations

Author: GuyEshdat

.github/workflows/test-warehouse.yml

@@ -14,6 +14,7 @@ on:
           - databricks
           - databricks_catalog
           - spark
+          - athena
       elementary-ref:
         type: string
         required: false

elementary/monitor/dbt_project/macros/create_temp_table.sql

@@ -11,7 +11,7 @@
                 *,
                 {{ elementary.edr_datediff(elementary.edr_cast_as_timestamp('detected_at'), elementary.edr_current_timestamp(), 'day') }} as days_diff,
                 {# When we split test into multiple test results, we want to have the same invocation order for the test results from the same run so we use rank. #}
-                rank() over (partition by elementary_unique_id order by detected_at desc) as invocations_rank_index
+                rank() over (partition by elementary_unique_id order by {{elementary.edr_cast_as_timestamp('detected_at')}} desc) as invocations_rank_index
             from test_results
         )
 
@@ -22,7 +22,7 @@
             test_results.model_unique_id,
             test_results.test_unique_id,
             test_results.elementary_unique_id,
-            test_results.detected_at,
+            {{elementary.edr_cast_as_timestamp('test_results.detected_at')}} as detected_at,
             test_results.database_name,
             test_results.schema_name,
             test_results.table_name,
@@ -59,7 +59,7 @@
     {% set test_results = [] %}
 
     {% set elementary_database, elementary_schema = elementary.get_package_database_and_schema() %}
-    {% set ordered_test_results_relation = elementary_cli.create_temp_table(elementary_database, elementary_schema, 'ordered_test_results', select_test_results) %}
+    {% set ordered_test_results_relation = elementary.create_temp_table(elementary_database, elementary_schema, 'ordered_test_results', select_test_results) %}
 
     {% set test_results_agate_sql %}
         select * from {{ ordered_test_results_relation }}
@@ -73,6 +73,9 @@
 
     {% set test_results_agate = elementary.run_query(test_results_agate_sql) %}
     {% set test_result_rows_agate = elementary_cli.get_result_rows_agate(days_back, valid_ids_query) %}
+    {% if not elementary.has_temp_table_support() %}
+        {% do elementary.fully_drop_relation(ordered_test_results_relation) %}
+    {% endif %}
     {% set tests = elementary.agate_to_dicts(test_results_agate) %}
 
     {% set filtered_tests = [] %}

elementary/monitor/dbt_project/macros/get_test_results.sql

@@ -1,6 +1,6 @@
 packages:
   - package: dbt-labs/dbt_utils
     version: 0.8.6
-  - package: elementary-data/elementary
-    version: 0.18.0
-sha1_hash: 24666e9f3179ea742d9fb6adce8005b9af3fc466
+  - git: https://github.com/elementary-data/dbt-data-reliability.git
+    revision: cc226bf865f922d6c4c5c6d65b2a142540e222f8
+sha1_hash: a838112140dab39e15226f415d0b0bcbfbe311ff

elementary/monitor/dbt_project/package-lock.yml

@@ -1,10 +1,10 @@
 packages:
   - package: dbt-labs/dbt_utils
     version: [">=0.8.0", "<0.9.0"]
-  - package: elementary-data/elementary
-    version: 0.18.0
+  # - package: elementary-data/elementary
+  #   version: 0.18.0
 
   #  NOTE - for unreleased CLI versions we often need to update the package version to a commit hash (please leave this
   #  commented, so it will be easy to access)
-  # - git: https://github.com/elementary-data/dbt-data-reliability.git
-  #   revision: 0f82a3f6c76706237172184b8bde3058040f5a8e
+  - git: https://github.com/elementary-data/dbt-data-reliability.git
+    revision: cc226bf865f922d6c4c5c6d65b2a142540e222f8

elementary/monitor/dbt_project/packages.yml

@@ -29,10 +29,7 @@ def test_run(self, custom_dbt_runner: BaseDbtRunner):
         result = custom_dbt_runner.run(select="one", vars={"test_marker": test_marker})
         assert result is True
 
-        invocations = self._run_query(
-            custom_dbt_runner,
-            f"""select * from {{{{ ref("dbt_invocations") }}}} where vars ilike '%{test_marker}%'""",
-        )
+        invocations = self._get_invocations_by_marker(custom_dbt_runner, test_marker)
         assert len(invocations) == 1
 
         invocation = invocations[0]
@@ -47,16 +44,21 @@ def test_test(self, custom_dbt_runner: BaseDbtRunner):
         result = custom_dbt_runner.test(select="one", vars={"test_marker": test_marker})
         assert result is True
 
-        invocations = self._run_query(
-            custom_dbt_runner,
-            f"""select * from {{{{ ref("dbt_invocations") }}}} where vars ilike '%{test_marker}%'""",
-        )
+        invocations = self._get_invocations_by_marker(custom_dbt_runner, test_marker)
         assert len(invocations) == 1
 
         invocation = invocations[0]
         assert invocation["command"] == "test"
         assert invocation["selected"] == json.dumps(["one"])
 
+    def _get_invocations_by_marker(
+        self, custom_dbt_runner: BaseDbtRunner, test_marker: str
+    ):
+        return self._run_query(
+            custom_dbt_runner,
+            f"""select * from {{{{ ref("dbt_invocations") }}}} where LOWER(vars) LIKE LOWER('%{test_marker}%')""",
+        )
+
     @staticmethod
     def _run_query(dbt_runner: SubprocessDbtRunner, query: str):
         return json.loads(