Merge branch 'master' of github.com:elementary-data/elementary into athena-integrations

GuyEshdat · GuyEshdat · commit d81710aabdbc · 2025-03-26T10:39:55.000+02:00
diff --git a/.github/workflows/close_pylon_issue.yml b/.github/workflows/close_pylon_issue.yml
@@ -3,29 +3,26 @@ name: Close Pylon Ticket on Issue or Pull Request Closure
 on:
   issues:
     types: [closed]
-  pull_request:
+  pull_request_target:
     types: [closed]
 
 jobs:
   close_pylon_ticket:
     runs-on: ubuntu-latest
     steps:
-      - name: Check out the repository
-        uses: actions/checkout@v2
-
       - name: Extract Pylon Ticket ID
         id: extract_ticket_id
         run: |
           if [[ "${{ github.event_name }}" == 'issues' ]]; then
             ISSUE_BODY=$(curl -s -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" \
                          "https://api.github.com/repos/${{ github.repository }}/issues/${{ github.event.issue.number }}" | jq -r '.body')
             pylon_ticket_id=$(echo "$ISSUE_BODY" | grep -oP '(?<=<!-- pylon-ticket-id: )\b[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}\b(?= -->)')
-          elif [[ "${{ github.event_name }}" == 'pull_request' ]]; then
+          elif [[ "${{ github.event_name }}" =~ ^pull_request ]]; then
             PR_BODY=$(curl -s -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" \
                      "https://api.github.com/repos/${{ github.repository }}/pulls/${{ github.event.pull_request.number }}" | jq -r '.body')
             pylon_ticket_id=$(echo "$PR_BODY" | grep -oP '(?<=<!-- pylon-ticket-id: )\b[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}\b(?= -->)')
           fi
-          echo "::set-output name=pylon_ticket_id::$pylon_ticket_id"
+          echo "pylon_ticket_id=$pylon_ticket_id" >> "$GITHUB_OUTPUT"
 
       - name: Close Pylon Ticket
         if: steps.extract_ticket_id.outputs.pylon_ticket_id != ''
diff --git a/.github/workflows/create_pylon_issue.yml b/.github/workflows/create_pylon_issue.yml
@@ -3,16 +3,13 @@ name: Create Pylon Issue
 on:
   issues:
     types: [opened]
-  pull_request:
+  pull_request_target:
     types: [opened]
 
 jobs:
   create_pylon_issue:
     runs-on: ubuntu-latest
     steps:
-      - name: Check out the repository
-        uses: actions/checkout@v2
-
       - name: Install jq
         run: sudo apt-get install -y jq
 
@@ -46,7 +43,7 @@ jobs:
             "https://api.github.com/repos/${{ github.repository }}/issues/${{ github.event.issue.number }}"
 
       - name: Create Pylon Issue for Pull Request
-        if: github.event_name == 'pull_request'
+        if: contains(github.event_name, 'pull_request')
         run: |
           response=$(curl --request POST \
             --url https://api.usepylon.com/issues \
diff --git a/.github/workflows/test-main-warehouse.yml b/.github/workflows/test-main-warehouse.yml
@@ -1,6 +1,6 @@
 name: Test main warehouse platform
 on:
-  pull_request:
+  pull_request_target:
     branches: ["master"]
     paths:
       - elementary/**
@@ -13,4 +13,5 @@ jobs:
     uses: ./.github/workflows/test-warehouse.yml
     with:
       warehouse-type: postgres
+      elementary-ref: ${{ (github.event_name == 'pull_request_target' && github.event.pull_request.head.sha) || '' }}
     secrets: inherit
diff --git a/.github/workflows/test-warehouse.yml b/.github/workflows/test-warehouse.yml
@@ -77,6 +77,7 @@ env:
 jobs:
   test:
     runs-on: ubuntu-latest
+    environment: elementary_test_env # This is a github environment (not to be confused with env vars)
     defaults:
       run:
         working-directory: elementary
diff --git a/docs/_snippets/quickstart-package-install.mdx b/docs/_snippets/quickstart-package-install.mdx
@@ -39,7 +39,7 @@ Some packages we recommend you check out: [dbt_utils](https://github.com/dbt-lab
     ```yml packages.yml
     packages:
       - package: elementary-data/elementary
-        version: 0.17.0
+        version: 0.18.0
         ## Docs: https://docs.elementary-data.com
     ```
   </Step>
diff --git a/elementary/monitor/data_monitoring/report/index.html b/elementary/monitor/data_monitoring/report/index.html
diff --git a/elementary/monitor/data_monitoring/schema.py b/elementary/monitor/data_monitoring/schema.py
@@ -1,7 +1,7 @@
 import re
 from datetime import datetime
 from enum import Enum
-from typing import Any, Generic, List, Optional, Pattern, Tuple, TypeVar
+from typing import Any, Generic, Iterable, List, Optional, Pattern, Set, Tuple, TypeVar
 
 from elementary.utils.log import get_logger
 from elementary.utils.pydantic_shim import BaseModel, Field, validator
@@ -97,6 +97,20 @@ def apply_filter_on_values(self, values: List[ValueT]) -> bool:
             return all(self.apply_filter_on_value(value) for value in values)
         raise ValueError(f"Unsupported filter type: {self.type}")
 
+    def get_matching_values(self, values: Iterable[ValueT]) -> Set[ValueT]:
+        values_list = set(values)
+        matching_values = set(
+            value for value in values_list if self.apply_filter_on_value(value)
+        )
+        if self.type in ANY_OPERATORS:
+            return matching_values
+        elif self.type in ALL_OPERATORS:
+            if len(matching_values) != len(values_list):
+                return set()
+            return matching_values
+
+        raise ValueError(f"Unsupported filter type: {self.type}")
+
 
 class StatusFilterSchema(FilterSchema[Status]):
     values: List[Status]
diff --git a/elementary/monitor/dbt_project/package-lock.yml b/elementary/monitor/dbt_project/package-lock.yml
@@ -2,5 +2,5 @@ packages:
   - package: dbt-labs/dbt_utils
     version: 0.8.6
   - package: elementary-data/elementary
-    version: 0.17.0
-sha1_hash: b63f4a2be6fec35c17f4e41107c8ad47a79b5c65
+    version: 0.18.0
+sha1_hash: 24666e9f3179ea742d9fb6adce8005b9af3fc466
diff --git a/elementary/monitor/dbt_project/packages.yml b/elementary/monitor/dbt_project/packages.yml
@@ -2,7 +2,7 @@ packages:
   - package: dbt-labs/dbt_utils
     version: [">=0.8.0", "<0.9.0"]
   - package: elementary-data/elementary
-    version: 0.17.0
+    version: 0.18.0
 
   #  NOTE - for unreleased CLI versions we often need to update the package version to a commit hash (please leave this
   #  commented, so it will be easy to access)
diff --git a/pyproject.toml b/pyproject.toml
@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "elementary-data"
-version = "0.17.0"
+version = "0.18.0"
 description = "Data monitoring and lineage"
 authors = ["Elementary"]
 keywords = ["data", "lineage", "data lineage", "data warehouse", "DWH", "observability", "data monitoring", "data observability", "Snowflake", "BigQuery", "Redshift", "data reliability", "analytics engineering"]
@@ -34,7 +34,7 @@ azure-storage-blob = ">=12.11.0"
 pymsteams = ">=0.2.2,<1.0.0"
 numpy = "<2.0.0"
 tabulate = ">= 0.9.0"
-pytz = "^2025.1"
+pytz = ">= 2025.1"
 
 dbt-snowflake = {version = ">=0.20,<2.0.0", optional = true}
 dbt-bigquery = {version = ">=0.20,<2.0.0", optional = true}
diff --git a/tests/unit/monitor/data_monitoring/test_filter_schema.py b/tests/unit/monitor/data_monitoring/test_filter_schema.py
@@ -71,3 +71,26 @@ def test_filter_schema_apply_filter_on_values_contains_operator():
 
     # Should not match when no values contain any filter values
     assert filter_schema.apply_filter_on_values(["abc", "xyz"]) is False
+
+
+def test_get_matching_values() -> None:
+    filter_schema = FilterSchema(values=["test1", "test2"], type=FilterType.IS)
+    values = ["test1", "test3", "test4"]
+    assert filter_schema.get_matching_values(values) == {"test1"}
+
+    filter_schema = FilterSchema(values=["test"], type=FilterType.CONTAINS)
+    values = ["test1", "testing", "other"]
+    assert filter_schema.get_matching_values(values) == {"test1", "testing"}
+
+    filter_schema = FilterSchema(values=["test1"], type=FilterType.IS_NOT)
+    values = ["test2", "test3"]
+    assert filter_schema.get_matching_values(values) == {"test2", "test3"}
+
+    filter_schema = FilterSchema(values=["test1"], type=FilterType.IS_NOT)
+    values = ["test1", "test2", "test3"]
+    assert filter_schema.get_matching_values(values) == set()
+
+    filter_schema = FilterSchema(values=["test"], type=FilterType.IS)
+    filter_schema.type = "unsupported"  # type: ignore
+    with pytest.raises(ValueError, match="Unsupported filter type: unsupported"):
+        filter_schema.get_matching_values(values)
