Merge pull request #375 from splunk/no_more_missing_datasources

Convert warning for missing datasource to an error

Author: pyth0n1c

contentctl/input/director.py

@@ -1,30 +1,29 @@
 import os
 import sys
-from pathlib import Path
 from dataclasses import dataclass, field
-from pydantic import ValidationError
+from pathlib import Path
 from uuid import UUID
-from contentctl.input.yml_reader import YmlReader
 
-from contentctl.objects.detection import Detection
-from contentctl.objects.story import Story
+from pydantic import ValidationError
 
-from contentctl.objects.baseline import Baseline
-from contentctl.objects.investigation import Investigation
-from contentctl.objects.playbook import Playbook
-from contentctl.objects.deployment import Deployment
-from contentctl.objects.macro import Macro
-from contentctl.objects.lookup import LookupAdapter, Lookup
-from contentctl.objects.atomic import AtomicEnrichment
-from contentctl.objects.security_content_object import SecurityContentObject
-from contentctl.objects.data_source import DataSource
-from contentctl.objects.dashboard import Dashboard
 from contentctl.enrichments.attack_enrichment import AttackEnrichment
 from contentctl.enrichments.cve_enrichment import CveEnrichment
-
+from contentctl.helper.utils import Utils
+from contentctl.input.yml_reader import YmlReader
+from contentctl.objects.atomic import AtomicEnrichment
+from contentctl.objects.baseline import Baseline
 from contentctl.objects.config import validate
+from contentctl.objects.dashboard import Dashboard
+from contentctl.objects.data_source import DataSource
+from contentctl.objects.deployment import Deployment
+from contentctl.objects.detection import Detection
 from contentctl.objects.enums import SecurityContentType
-from contentctl.helper.utils import Utils
+from contentctl.objects.investigation import Investigation
+from contentctl.objects.lookup import Lookup, LookupAdapter
+from contentctl.objects.macro import Macro
+from contentctl.objects.playbook import Playbook
+from contentctl.objects.security_content_object import SecurityContentObject
+from contentctl.objects.story import Story
 
 
 @dataclass
@@ -113,20 +112,6 @@ def execute(self, input_dto: validate) -> None:
         self.createSecurityContent(SecurityContentType.detections)
         self.createSecurityContent(SecurityContentType.dashboards)
 
-        from contentctl.objects.abstract_security_content_objects.detection_abstract import (
-            MISSING_SOURCES,
-        )
-
-        if len(MISSING_SOURCES) > 0:
-            missing_sources_string = "\n 🟡 ".join(sorted(list(MISSING_SOURCES)))
-            print(
-                "WARNING: The following data_sources have been used in detections, but are not yet defined.\n"
-                "This is not yet an error since not all data_sources have been defined, but will be convered to an error soon:\n 🟡 "
-                f"{missing_sources_string}"
-            )
-        else:
-            print("No missing data_sources!")
-
     def createSecurityContent(self, contentType: SecurityContentType) -> None:
         if contentType in [
             SecurityContentType.deployments,

contentctl/objects/abstract_security_content_objects/detection_abstract.py

@@ -51,8 +51,6 @@
 from contentctl.objects.test_group import TestGroup
 from contentctl.objects.unit_test import UnitTest
 
-MISSING_SOURCES: set[str] = set()
-
 # Those AnalyticsTypes that we do not test via contentctl
 SKIPPED_ANALYTICS_TYPES: set[str] = {AnalyticsType.Correlation}
 
@@ -514,7 +512,7 @@ def model_post_init(self, __context: Any) -> None:
             baseline.tags.detections = new_detections
 
         # Data source may be defined 1 on each line, OR they may be defined as
-        # SOUCE_1 AND ANOTHERSOURCE AND A_THIRD_SOURCE
+        # SOURCE_1 AND ANOTHERSOURCE AND A_THIRD_SOURCE
         # if more than 1 data source is required for a detection (for example, because it includes a join)
         # Parse and update the list to resolve individual names and remove potential duplicates
         updated_data_source_names: set[str] = set()
@@ -524,27 +522,9 @@ def model_post_init(self, __context: Any) -> None:
             updated_data_source_names.update(split_data_sources)
 
         sources = sorted(list(updated_data_source_names))
-
-        matched_data_sources: list[DataSource] = []
-        missing_sources: list[str] = []
-        for source in sources:
-            try:
-                matched_data_sources += DataSource.mapNamesToSecurityContentObjects(
-                    [source], director
-                )
-            except Exception:
-                # We gobble this up and add it to a global set so that we
-                # can print it ONCE at the end of the build of datasources.
-                # This will be removed later as per the note below
-                MISSING_SOURCES.add(source)
-
-        if len(missing_sources) > 0:
-            # This will be changed to ValueError when we have a complete list of data sources
-            print(
-                "WARNING: The following exception occurred when mapping the data_source field to "
-                f"DataSource objects:{missing_sources}"
-            )
-
+        matched_data_sources = DataSource.mapNamesToSecurityContentObjects(
+            sources, director
+        )
         self.data_source_objects = matched_data_sources
 
         for story in self.tags.analytic_story: