Merge branch 'obs_to_rba' into integration_testing_rba_migration

Author: pyth0n1c

contentctl/objects/baseline.py

@@ -1,6 +1,9 @@
 
 from __future__ import annotations
-from typing import Annotated, List,Any
+from typing import Annotated, List,Any, TYPE_CHECKING
+if TYPE_CHECKING:
+    from contentctl.input.director import DirectorOutputDto
+
 from pydantic import field_validator, ValidationInfo, Field, model_serializer, computed_field
 from contentctl.objects.deployment import Deployment
 from contentctl.objects.security_content_object import SecurityContentObject
@@ -9,7 +12,7 @@
 
 from contentctl.objects.config import CustomApp
 
-
+from contentctl.objects.lookup import Lookup
 from contentctl.objects.constants import CONTENTCTL_MAX_SEARCH_NAME_LENGTH,CONTENTCTL_BASELINE_STANZA_NAME_FORMAT_TEMPLATE
 
 class Baseline(SecurityContentObject):
@@ -19,10 +22,24 @@ class Baseline(SecurityContentObject):
     how_to_implement: str = Field(..., min_length=4)
     known_false_positives: str = Field(..., min_length=4)
     tags: BaselineTags = Field(...)
-
+    lookups: list[Lookup] = Field([], validate_default=True)
     # enrichment
     deployment: Deployment = Field({})
-    
+
+
+    @field_validator('lookups', mode="before")
+    @classmethod
+    def getBaselineLookups(cls, v:list[str], info:ValidationInfo) -> list[Lookup]:
+        '''
+        This function has been copied and renamed from the Detection_Abstract class
+        '''
+        director:DirectorOutputDto = info.context.get("output_dto",None)
+        search: str | None = info.data.get("search",None)
+        if search is None:
+            raise ValueError("Search was None - is this file missing the search field?")
+        
+        lookups = Lookup.get_lookups(search, director)
+        return lookups
 
     def get_conf_stanza_name(self, app:CustomApp)->str:
         stanza_name = CONTENTCTL_BASELINE_STANZA_NAME_FORMAT_TEMPLATE.format(app_label=app.label, detection_name=self.name)

contentctl/objects/lookup.py

@@ -18,6 +18,9 @@
 LOOKUPS_TO_IGNORE.add("ut_shannon_lookup") #In the URL toolbox app which is recommended for ESCU
 LOOKUPS_TO_IGNORE.add("identity_lookup_expanded") #Shipped with the Asset and Identity Framework
 LOOKUPS_TO_IGNORE.add("cim_corporate_web_domain_lookup") #Shipped with the Asset and Identity Framework
+LOOKUPS_TO_IGNORE.add("cim_corporate_email_domain_lookup") #Shipped with the Enterprise Security
+LOOKUPS_TO_IGNORE.add("cim_cloud_domain_lookup") #Shipped with the Enterprise Security
+
 LOOKUPS_TO_IGNORE.add("alexa_lookup_by_str") #Shipped with the Asset and Identity Framework
 LOOKUPS_TO_IGNORE.add("interesting_ports_lookup") #Shipped with the Asset and Identity Framework
 LOOKUPS_TO_IGNORE.add("asset_lookup_by_str") #Shipped with the Asset and Identity Framework
@@ -89,18 +92,18 @@ def match_type_to_conf_format(self)->str:
     @staticmethod
     def get_lookups(text_field: str, director:DirectorOutputDto, ignore_lookups:set[str]=LOOKUPS_TO_IGNORE)->list[Lookup]:
         # Comprehensively match all kinds of lookups, including inputlookup and outputlookup
-        inputLookupsToGet = set(re.findall(r'inputlookup(?:\s*(?:(?:append|strict|start|max)\s*=\s*(?:true|t|false|f))){0,4}\s+([^\s\|]+)', text_field))
-        outputLookupsToGet = set(re.findall(r'outputlookup(?:\s*(?:(?:append|create_empty|override_if_empty|max|key_field|allow_updates|createinapp|create_context|output_format)\s*=\s*[^\s]*))*\s+([^\s\|]+)',text_field))
-        lookupsToGet = set(re.findall(r'(?:(?<!output)(?<!input))lookup(?:\s*(?:(?:local|update)\s*=\s*(?:true|t|false|f))){0,2}\s+([^\s\|]+)', text_field))
+        inputLookupsToGet = set(re.findall(r'[^\w]inputlookup(?:\s*(?:(?:append|strict|start|max)\s*=\s*(?:true|t|false|f))){0,4}\s+([^\s\|]+)', text_field, re.IGNORECASE))
+        outputLookupsToGet = set(re.findall(r'[^\w]outputlookup(?:\s*(?:(?:append|create_empty|override_if_empty|max|key_field|allow_updates|createinapp|create_context|output_format)\s*=\s*[^\s]*))*\s+([^\s\|]+)',text_field,re.IGNORECASE))
+        lookupsToGet = set(re.findall(r'[^\w](?:(?<!output)(?<!input))lookup(?:\s*(?:(?:local|update)\s*=\s*(?:true|t|false|f))){0,2}\s+([^\s\|]+)', text_field, re.IGNORECASE))
 
 
         input_lookups = Lookup.mapNamesToSecurityContentObjects(list(inputLookupsToGet-LOOKUPS_TO_IGNORE), director)
         output_lookups = Lookup.mapNamesToSecurityContentObjects(list(outputLookupsToGet-LOOKUPS_TO_IGNORE), director)
         lookups = Lookup.mapNamesToSecurityContentObjects(list(lookupsToGet-LOOKUPS_TO_IGNORE), director)
 
+        all_lookups = set(input_lookups + output_lookups + lookups)
 
-            
-        return lookups + input_lookups + output_lookups
+        return list(all_lookups)