Merge pull request #1945 from elementary-data/ele-4677-add-exclude-option-to-edr-monitor

added excludes option to edr monitor

Author: ofek1weiss

elementary/monitor/cli.py

@@ -269,6 +269,15 @@ def get_cli_properties() -> dict:
     help="Filter the alerts by tags:<tags separated by commas> / owners:<owners separated by commas> / models:<models separated by commas> / "
     "statuses:<warn/fail/error/skipped> / resource_types:<model/test>.",
 )
+@click.option(
+    "--excludes",
+    "-ex",
+    type=str,
+    default=None,
+    multiple=True,
+    help="Exclude alerts by tags:<tags separated by commas> / owners:<owners separated by commas> / models:<models separated by commas> / "
+    "statuses:<warn/fail/error/skipped> / resource_types:<model/test>.",
+)
 @click.option(
     "--teams-webhook",
     "-tw",
@@ -305,6 +314,7 @@ def monitor(
     override_dbt_project_config,
     report_url,
     filters,
+    excludes,
     teams_webhook,
 ):
     """
@@ -344,8 +354,8 @@ def monitor(
         config.validate_monitor()
 
         alert_filters = FiltersSchema()
-        if bool(filters):
-            alert_filters = FiltersSchema.from_cli_params(filters)
+        if bool(filters) or bool(excludes):
+            alert_filters = FiltersSchema.from_cli_params(filters, excludes)
         elif select is not None:
             click.secho(
                 '\n"--select" is deprecated and won\'t be supported in the near future.\n'

elementary/monitor/data_monitoring/schema.py

@@ -178,8 +178,16 @@ def validate_report_selector(self) -> None:
             )
 
     @staticmethod
-    def from_cli_params(cli_filters: Tuple[str]) -> "FiltersSchema":
-        if not cli_filters:
+    def from_cli_params(
+        cli_filters: Tuple[str], cli_excludes: Tuple[str]
+    ) -> "FiltersSchema":
+        all_filters: list[tuple[str, FilterType]] = []
+        for cli_filter in cli_filters:
+            all_filters.append((cli_filter, FilterType.IS))
+        for cli_exclude in cli_excludes:
+            all_filters.append((cli_exclude, FilterType.IS_NOT))
+
+        if not all_filters:
             return FiltersSchema()
 
         tags = []
@@ -188,26 +196,26 @@ def from_cli_params(cli_filters: Tuple[str]) -> "FiltersSchema":
         statuses = []
         resource_types = []
 
-        for cli_filter in cli_filters:
+        for cli_filter, filter_type in all_filters:
             tags_match = FiltersSchema._match_filter_regex(
                 filter_string=cli_filter, regex=re.compile(r"tags:(.*)")
             )
             if tags_match:
-                tags.append(FilterSchema(values=tags_match))
+                tags.append(FilterSchema(values=tags_match, type=filter_type))
                 continue
 
             owners_match = FiltersSchema._match_filter_regex(
                 filter_string=cli_filter, regex=re.compile(r"owners:(.*)")
             )
             if owners_match:
-                owners.append(FilterSchema(values=owners_match))
+                owners.append(FilterSchema(values=owners_match, type=filter_type))
                 continue
 
             models_match = FiltersSchema._match_filter_regex(
                 filter_string=cli_filter, regex=re.compile(r"models:(.*)")
             )
             if models_match:
-                models.append(FilterSchema(values=models_match))
+                models.append(FilterSchema(values=models_match, type=filter_type))
                 continue
 
             statuses_match = FiltersSchema._match_filter_regex(
@@ -216,7 +224,8 @@ def from_cli_params(cli_filters: Tuple[str]) -> "FiltersSchema":
             if statuses_match:
                 statuses.append(
                     StatusFilterSchema(
-                        values=[Status(status) for status in statuses_match]
+                        values=[Status(status) for status in statuses_match],
+                        type=filter_type,
                     )
                 )
                 continue
@@ -230,7 +239,8 @@ def from_cli_params(cli_filters: Tuple[str]) -> "FiltersSchema":
                         values=[
                             ResourceType(resource_type)
                             for resource_type in resource_types_match
-                        ]
+                        ],
+                        type=filter_type,
                     )
                 )
                 continue
@@ -239,11 +249,14 @@ def from_cli_params(cli_filters: Tuple[str]) -> "FiltersSchema":
                 f'Filter "{cli_filter.split(":")[0]}" is not supported - Skipping this filter ("{cli_filter}").'
             )
 
+        if not any(status_filter.type == FilterType.IS for status_filter in statuses):
+            statuses.extend(_get_default_statuses_filter())
+
         return FiltersSchema(
             tags=tags,
             owners=owners,
             models=models,
-            statuses=statuses if statuses else _get_default_statuses_filter(),
+            statuses=statuses,
             resource_types=resource_types,
         )
 

tests/unit/monitor/data_monitoring/test_filters_schema.py

@@ -1,11 +1,12 @@
 import pytest
 
-from elementary.monitor.data_monitoring.schema import FiltersSchema
+from elementary.monitor.data_monitoring.schema import FiltersSchema, FilterType
 
 
 def test_empty_from_cli_params():
     cli_filter = ()
-    filter_schema = FiltersSchema.from_cli_params(cli_filter)
+    cli_excludes = ()
+    filter_schema = FiltersSchema.from_cli_params(cli_filter, cli_excludes)
     assert len(filter_schema.tags) == 0
     assert len(filter_schema.owners) == 0
     assert len(filter_schema.models) == 0
@@ -18,7 +19,8 @@ def test_empty_from_cli_params():
 
 def test_tags_key_from_cli_params():
     cli_filter = ("tags:tag1",)
-    filter_schema = FiltersSchema.from_cli_params(cli_filter)
+    cli_excludes = ()
+    filter_schema = FiltersSchema.from_cli_params(cli_filter, cli_excludes)
     assert len(filter_schema.tags) == 1
     assert filter_schema.tags[0].values == ["tag1"]
     assert len(filter_schema.owners) == 0
@@ -30,7 +32,8 @@ def test_tags_key_from_cli_params():
     assert len(filter_schema.resource_types) == 0
 
     cli_filter = ("tags:tag1,tag2",)
-    filter_schema = FiltersSchema.from_cli_params(cli_filter)
+    cli_excludes = ()
+    filter_schema = FiltersSchema.from_cli_params(cli_filter, cli_excludes)
     assert len(filter_schema.tags) == 1
     assert sorted(filter_schema.tags[0].values) == sorted(["tag1", "tag2"])
     assert len(filter_schema.owners) == 0
@@ -42,7 +45,8 @@ def test_tags_key_from_cli_params():
     assert len(filter_schema.resource_types) == 0
 
     cli_filter = ("tags:tag1", "tags:tag2")
-    filter_schema = FiltersSchema.from_cli_params(cli_filter)
+    cli_excludes = ()
+    filter_schema = FiltersSchema.from_cli_params(cli_filter, cli_excludes)
     assert len(filter_schema.tags) == 2
     assert filter_schema.tags[0].values == ["tag1"]
     assert filter_schema.tags[1].values == ["tag2"]
@@ -57,7 +61,8 @@ def test_tags_key_from_cli_params():
 
 def test_owners_key_from_cli_params():
     cli_filter = ("owners:freddy",)
-    filter_schema = FiltersSchema.from_cli_params(cli_filter)
+    cli_excludes = ()
+    filter_schema = FiltersSchema.from_cli_params(cli_filter, cli_excludes)
     assert len(filter_schema.tags) == 0
     assert len(filter_schema.owners) == 1
     assert filter_schema.owners[0].values == ["freddy"]
@@ -69,7 +74,8 @@ def test_owners_key_from_cli_params():
     assert len(filter_schema.resource_types) == 0
 
     cli_filter = ("owners:freddy,dredd",)
-    filter_schema = FiltersSchema.from_cli_params(cli_filter)
+    cli_excludes = ()
+    filter_schema = FiltersSchema.from_cli_params(cli_filter, cli_excludes)
     assert len(filter_schema.tags) == 0
     assert len(filter_schema.owners) == 1
     assert sorted(filter_schema.owners[0].values) == sorted(["freddy", "dredd"])
@@ -81,7 +87,8 @@ def test_owners_key_from_cli_params():
     assert len(filter_schema.resource_types) == 0
 
     cli_filter = ("owners:freddy", "owners:dredd")
-    filter_schema = FiltersSchema.from_cli_params(cli_filter)
+    cli_excludes = ()
+    filter_schema = FiltersSchema.from_cli_params(cli_filter, cli_excludes)
     assert len(filter_schema.tags) == 0
     assert len(filter_schema.owners) == 2
     assert filter_schema.owners[0].values == ["freddy"]
@@ -96,7 +103,8 @@ def test_owners_key_from_cli_params():
 
 def test_models_key_from_cli_params():
     cli_filter = ("models:freddy",)
-    filter_schema = FiltersSchema.from_cli_params(cli_filter)
+    cli_excludes = ()
+    filter_schema = FiltersSchema.from_cli_params(cli_filter, cli_excludes)
     assert len(filter_schema.tags) == 0
     assert len(filter_schema.models) == 1
     assert filter_schema.models[0].values == ["freddy"]
@@ -108,7 +116,8 @@ def test_models_key_from_cli_params():
     assert len(filter_schema.resource_types) == 0
 
     cli_filter = ("models:freddy,dredd",)
-    filter_schema = FiltersSchema.from_cli_params(cli_filter)
+    cli_excludes = ()
+    filter_schema = FiltersSchema.from_cli_params(cli_filter, cli_excludes)
     assert len(filter_schema.tags) == 0
     assert len(filter_schema.models) == 1
     assert sorted(filter_schema.models[0].values) == sorted(["freddy", "dredd"])
@@ -120,7 +129,8 @@ def test_models_key_from_cli_params():
     assert len(filter_schema.resource_types) == 0
 
     cli_filter = ("models:freddy", "models:dredd")
-    filter_schema = FiltersSchema.from_cli_params(cli_filter)
+    cli_excludes = ()
+    filter_schema = FiltersSchema.from_cli_params(cli_filter, cli_excludes)
     assert len(filter_schema.tags) == 0
     assert len(filter_schema.models) == 2
     assert filter_schema.models[0].values == ["freddy"]
@@ -136,18 +146,22 @@ def test_models_key_from_cli_params():
 def test_statuses_key_from_cli_params():
     with pytest.raises(ValueError):
         cli_filter = ("statuses:freddy",)
-        FiltersSchema.from_cli_params(cli_filter)
+        cli_excludes = ()
+        FiltersSchema.from_cli_params(cli_filter, cli_excludes)
 
     with pytest.raises(ValueError):
         cli_filter = ("statuses:warn,freddy",)
-        FiltersSchema.from_cli_params(cli_filter)
+        cli_excludes = ()
+        FiltersSchema.from_cli_params(cli_filter, cli_excludes)
 
     with pytest.raises(ValueError):
         cli_filter = ("statuses:warn", "statuses:freddy")
-        FiltersSchema.from_cli_params(cli_filter)
+        cli_excludes = ()
+        FiltersSchema.from_cli_params(cli_filter, cli_excludes)
 
     cli_filter = ("statuses:warn",)
-    filter_schema = FiltersSchema.from_cli_params(cli_filter)
+    cli_excludes = ()
+    filter_schema = FiltersSchema.from_cli_params(cli_filter, cli_excludes)
     assert len(filter_schema.tags) == 0
     assert len(filter_schema.models) == 0
     assert len(filter_schema.owners) == 0
@@ -156,7 +170,8 @@ def test_statuses_key_from_cli_params():
     assert len(filter_schema.resource_types) == 0
 
     cli_filter = ("statuses:warn,fail",)
-    filter_schema = FiltersSchema.from_cli_params(cli_filter)
+    cli_excludes = ()
+    filter_schema = FiltersSchema.from_cli_params(cli_filter, cli_excludes)
     assert len(filter_schema.tags) == 0
     assert len(filter_schema.models) == 0
     assert len(filter_schema.owners) == 0
@@ -165,7 +180,8 @@ def test_statuses_key_from_cli_params():
     assert len(filter_schema.resource_types) == 0
 
     cli_filter = ("statuses:warn", "statuses:fail")
-    filter_schema = FiltersSchema.from_cli_params(cli_filter)
+    cli_excludes = ()
+    filter_schema = FiltersSchema.from_cli_params(cli_filter, cli_excludes)
     assert len(filter_schema.tags) == 0
     assert len(filter_schema.models) == 0
     assert len(filter_schema.owners) == 0
@@ -178,18 +194,22 @@ def test_statuses_key_from_cli_params():
 def test_resource_types_key_from_cli_params():
     with pytest.raises(ValueError):
         cli_filter = ("resource_types:freddy",)
-        FiltersSchema.from_cli_params(cli_filter)
+        cli_excludes = ()
+        FiltersSchema.from_cli_params(cli_filter, cli_excludes)
 
     with pytest.raises(ValueError):
         cli_filter = ("resource_types:test,freddy",)
-        FiltersSchema.from_cli_params(cli_filter)
+        cli_excludes = ()
+        FiltersSchema.from_cli_params(cli_filter, cli_excludes)
 
     with pytest.raises(ValueError):
         cli_filter = ("resource_types:test", "resource_types:freddy")
-        FiltersSchema.from_cli_params(cli_filter)
+        cli_excludes = ()
+        FiltersSchema.from_cli_params(cli_filter, cli_excludes)
 
     cli_filter = ("resource_types:test",)
-    filter_schema = FiltersSchema.from_cli_params(cli_filter)
+    cli_excludes = ()
+    filter_schema = FiltersSchema.from_cli_params(cli_filter, cli_excludes)
     assert len(filter_schema.tags) == 0
     assert len(filter_schema.models) == 0
     assert len(filter_schema.owners) == 0
@@ -201,7 +221,8 @@ def test_resource_types_key_from_cli_params():
     assert filter_schema.resource_types[0].values == ["test"]
 
     cli_filter = ("resource_types:test,model",)
-    filter_schema = FiltersSchema.from_cli_params(cli_filter)
+    cli_excludes = ()
+    filter_schema = FiltersSchema.from_cli_params(cli_filter, cli_excludes)
     assert len(filter_schema.tags) == 0
     assert len(filter_schema.models) == 0
     assert len(filter_schema.owners) == 0
@@ -213,7 +234,8 @@ def test_resource_types_key_from_cli_params():
     assert sorted(filter_schema.resource_types[0].values) == sorted(["test", "model"])
 
     cli_filter = ("resource_types:test", "resource_types:model")
-    filter_schema = FiltersSchema.from_cli_params(cli_filter)
+    cli_excludes = ()
+    filter_schema = FiltersSchema.from_cli_params(cli_filter, cli_excludes)
     assert len(filter_schema.tags) == 0
     assert len(filter_schema.models) == 0
     assert len(filter_schema.owners) == 0
@@ -228,7 +250,8 @@ def test_resource_types_key_from_cli_params():
 
 def test_unsupported_key_from_cli_params():
     cli_filter = ("fake",)
-    filter_schema = FiltersSchema.from_cli_params(cli_filter)
+    cli_excludes = ()
+    filter_schema = FiltersSchema.from_cli_params(cli_filter, cli_excludes)
     assert len(filter_schema.tags) == 0
     assert len(filter_schema.owners) == 0
     assert len(filter_schema.models) == 0
@@ -241,7 +264,8 @@ def test_unsupported_key_from_cli_params():
 
 def test_multiple_keys_from_cli_params():
     cli_filter = ("tags:tag1", "owners:freddy,dredd")
-    filter_schema = FiltersSchema.from_cli_params(cli_filter)
+    cli_excludes = ()
+    filter_schema = FiltersSchema.from_cli_params(cli_filter, cli_excludes)
     assert len(filter_schema.tags) == 1
     assert filter_schema.tags[0].values == ["tag1"]
     assert len(filter_schema.owners) == 1
@@ -252,3 +276,37 @@ def test_multiple_keys_from_cli_params():
         ["fail", "error", "runtime error", "warn"]
     )
     assert len(filter_schema.resource_types) == 0
+
+
+def test_exclude_filters():
+    cli_filter = ("tags:tag1",)
+    cli_excludes = ("tags:tag2",)
+    filter_schema = FiltersSchema.from_cli_params(cli_filter, cli_excludes)
+    assert len(filter_schema.tags) == 2
+    assert filter_schema.tags[0].values == ["tag1"]
+    assert filter_schema.tags[0].type == FilterType.IS
+    assert filter_schema.tags[1].values == ["tag2"]
+    assert filter_schema.tags[1].type == FilterType.IS_NOT
+    assert len(filter_schema.owners) == 0
+    assert len(filter_schema.models) == 0
+    assert len(filter_schema.statuses) == 1
+    assert sorted(filter_schema.statuses[0].values) == sorted(
+        ["fail", "error", "runtime error", "warn"]
+    )
+    assert len(filter_schema.resource_types) == 0
+
+
+def test_exclude_statuses_filters():
+    cli_filters = ()
+    cli_excludes = ("statuses:fail",)
+    filter_schema = FiltersSchema.from_cli_params(cli_filters, cli_excludes)
+    assert len(filter_schema.tags) == 0
+    assert len(filter_schema.models) == 0
+    assert len(filter_schema.owners) == 0
+    assert len(filter_schema.statuses) == 2
+    assert filter_schema.statuses[0].values == ["fail"]
+    assert filter_schema.statuses[0].type == FilterType.IS_NOT
+    assert sorted(filter_schema.statuses[1].values) == sorted(
+        ["fail", "error", "runtime error", "warn"]
+    )
+    assert filter_schema.statuses[1].type == FilterType.IS