[client] report discarded objects during splitting as processed in work (opencti #11359)

Author: JeremyCloarec

pycti/connector/opencti_connector_helper.py

@@ -1918,14 +1918,13 @@ def send_stix2_bundle(self, bundle: str, **kwargs) -> list:
             os.rename(write_file, final_write_file)
 
         stix2_splitter = OpenCTIStix2Splitter()
-        (
-            expectations_number,
-            bundles,
-        ) = stix2_splitter.split_bundle_with_expectations(
-            bundle=bundle,
-            use_json=True,
-            event_version=event_version,
-            cleanup_inconsistent_bundle=cleanup_inconsistent_bundle,
+        (expectations_number, _, bundles) = (
+            stix2_splitter.split_bundle_with_expectations(
+                bundle=bundle,
+                use_json=True,
+                event_version=event_version,
+                cleanup_inconsistent_bundle=cleanup_inconsistent_bundle,
+            )
         )
 
         if len(bundles) == 0:

pycti/utils/opencti_stix2.py

@@ -2748,9 +2748,25 @@ def import_bundle(
         )
 
         stix2_splitter = OpenCTIStix2Splitter()
-        _, bundles = stix2_splitter.split_bundle_with_expectations(
-            stix_bundle, False, event_version
+        _, incompatible_elements, bundles = (
+            stix2_splitter.split_bundle_with_expectations(
+                stix_bundle, False, event_version
+            )
         )
+
+        # Report every element ignored during bundle splitting
+        if work_id is not None:
+            for incompatible_element in incompatible_elements:
+                self.opencti.work.report_expectation(
+                    work_id,
+                    {
+                        "error": "Incompatible element in bundle",
+                        "source": "Element "
+                        + incompatible_element["id"]
+                        + " is incompatible and couldn't be processed",
+                    },
+                )
+
         # Import every element in a specific order
         imported_elements = []
         for bundle in bundles:

pycti/utils/opencti_stix2_splitter.py

@@ -35,6 +35,7 @@ def __init__(self):
         self.cache_index = {}
         self.cache_refs = {}
         self.elements = []
+        self.incompatible_items = []
 
     def get_internal_ids_in_extension(self, item):
         ids = []
@@ -189,6 +190,8 @@ def enlist_element(
                 is_compatible = is_id_supported(item_id)
             if is_compatible:
                 self.elements.append(item)
+            else:
+                self.incompatible_items.append(item)
             self.cache_index[item_id] = item
             for internal_id in self.get_internal_ids_in_extension(item):
                 self.cache_index[internal_id] = item
@@ -201,7 +204,7 @@ def split_bundle_with_expectations(
         use_json=True,
         event_version=None,
         cleanup_inconsistent_bundle=False,
-    ) -> Tuple[int, list]:
+    ) -> Tuple[int, list, list]:
         """splits a valid stix2 bundle into a list of bundles"""
         if use_json:
             try:
@@ -251,11 +254,11 @@ def by_dep_size(elem):
                 )
             )
 
-        return number_expectations, bundles
+        return number_expectations, self.incompatible_items, bundles
 
     @deprecated("Use split_bundle_with_expectations instead")
     def split_bundle(self, bundle, use_json=True, event_version=None) -> list:
-        expectations, bundles = self.split_bundle_with_expectations(
+        _, _, bundles = self.split_bundle_with_expectations(
             bundle, use_json, event_version
         )
         return bundles

tests/01-unit/utils/test_opencti_stix2_splitter.py

@@ -10,15 +10,15 @@ def test_split_bundle():
     stix_splitter = OpenCTIStix2Splitter()
     with open("./tests/data/enterprise-attack.json") as file:
         content = file.read()
-    expectations, bundles = stix_splitter.split_bundle_with_expectations(content)
+    expectations, _, bundles = stix_splitter.split_bundle_with_expectations(content)
     assert expectations == 7016
 
 
 def test_split_test_bundle():
     stix_splitter = OpenCTIStix2Splitter()
     with open("./tests/data/DATA-TEST-STIX2_v2.json") as file:
         content = file.read()
-    expectations, bundles = stix_splitter.split_bundle_with_expectations(content)
+    expectations, _, bundles = stix_splitter.split_bundle_with_expectations(content)
     assert expectations == 59
     base_bundles = json.loads(content)["objects"]
     for base in base_bundles:
@@ -40,13 +40,13 @@ def test_split_mono_entity_bundle():
     stix_splitter = OpenCTIStix2Splitter()
     with open("./tests/data/mono-bundle-entity.json") as file:
         content = file.read()
-    expectations, bundles = stix_splitter.split_bundle_with_expectations(content)
+    expectations, _, bundles = stix_splitter.split_bundle_with_expectations(content)
     assert expectations == 1
     json_bundle = json.loads(bundles[0])["objects"][0]
     assert json_bundle["created_by_ref"] == "fa42a846-8d90-4e51-bc29-71d5b4802168"
     # Split with cleanup_inconsistent_bundle
     stix_splitter = OpenCTIStix2Splitter()
-    expectations, bundles = stix_splitter.split_bundle_with_expectations(
+    expectations, _, bundles = stix_splitter.split_bundle_with_expectations(
         bundle=content, cleanup_inconsistent_bundle=True
     )
     assert expectations == 1
@@ -58,11 +58,11 @@ def test_split_mono_relationship_bundle():
     stix_splitter = OpenCTIStix2Splitter()
     with open("./tests/data/mono-bundle-relationship.json") as file:
         content = file.read()
-    expectations, bundles = stix_splitter.split_bundle_with_expectations(content)
+    expectations, _, bundles = stix_splitter.split_bundle_with_expectations(content)
     assert expectations == 1
     # Split with cleanup_inconsistent_bundle
     stix_splitter = OpenCTIStix2Splitter()
-    expectations, bundles = stix_splitter.split_bundle_with_expectations(
+    expectations, _, bundles = stix_splitter.split_bundle_with_expectations(
         bundle=content, cleanup_inconsistent_bundle=True
     )
     assert expectations == 0
@@ -72,19 +72,19 @@ def test_split_capec_bundle():
     stix_splitter = OpenCTIStix2Splitter()
     with open("./tests/data/mitre_att_capec.json") as file:
         content = file.read()
-    expectations, bundles = stix_splitter.split_bundle_with_expectations(content)
+    expectations, _, bundles = stix_splitter.split_bundle_with_expectations(content)
     assert expectations == 2610
 
 
 def test_split_internal_ids_bundle():
     stix_splitter = OpenCTIStix2Splitter()
     with open("./tests/data/bundle_with_internal_ids.json") as file:
         content = file.read()
-    expectations, bundles = stix_splitter.split_bundle_with_expectations(content)
+    expectations, _, bundles = stix_splitter.split_bundle_with_expectations(content)
     assert expectations == 4
     # Split with cleanup_inconsistent_bundle
     stix_splitter = OpenCTIStix2Splitter()
-    expectations, bundles = stix_splitter.split_bundle_with_expectations(
+    expectations, _, bundles = stix_splitter.split_bundle_with_expectations(
         bundle=content, cleanup_inconsistent_bundle=True
     )
     assert expectations == 4
@@ -101,11 +101,11 @@ def test_split_missing_refs_bundle():
     stix_splitter = OpenCTIStix2Splitter()
     with open("./tests/data/missing_refs.json") as file:
         content = file.read()
-    expectations, bundles = stix_splitter.split_bundle_with_expectations(content)
+    expectations, _, bundles = stix_splitter.split_bundle_with_expectations(content)
     assert expectations == 4
     # Split with cleanup_inconsistent_bundle
     stix_splitter = OpenCTIStix2Splitter()
-    expectations, bundles = stix_splitter.split_bundle_with_expectations(
+    expectations, _, bundles = stix_splitter.split_bundle_with_expectations(
         bundle=content, cleanup_inconsistent_bundle=True
     )
     assert expectations == 3
@@ -115,7 +115,7 @@ def test_split_cyclic_bundle():
     stix_splitter = OpenCTIStix2Splitter()
     with open("./tests/data/cyclic-bundle.json") as file:
         content = file.read()
-    expectations, bundles = stix_splitter.split_bundle_with_expectations(content)
+    expectations, _, bundles = stix_splitter.split_bundle_with_expectations(content)
     assert expectations == 6
     for bundle in bundles:
         json_bundle = json.loads(bundle)