[client] Allow internal id in the split (#741)

Author: richard-julien

pycti/utils/opencti_stix2_splitter.py

@@ -13,6 +13,8 @@
     SUPPORTED_STIX_ENTITY_OBJECTS,
 )
 
+OPENCTI_EXTENSION = "extension-definition--ea279b3e-5c71-4632-ac08-831c66a786ba"
+
 supported_types = (
     SUPPORTED_STIX_ENTITY_OBJECTS  # entities
     + list(STIX_CYBER_OBSERVABLE_MAPPING.keys())  # observables
@@ -21,8 +23,11 @@
 
 
 def is_id_supported(key):
-    id_type = key.split("--")[0]
-    return id_type in supported_types
+    if "--" in key:
+        id_type = key.split("--")[0]
+        return id_type in supported_types
+    # If not a stix id, don't try to filter
+    return True
 
 
 class OpenCTIStix2Splitter:
@@ -31,6 +36,18 @@ def __init__(self):
         self.cache_refs = {}
         self.elements = []
 
+    def get_internal_ids_in_extension(self, item):
+        ids = []
+        if item.get("x_opencti_id"):
+            ids.append(item["x_opencti_id"])
+        if (
+            item.get("extensions")
+            and item["extensions"].get(OPENCTI_EXTENSION)
+            and item["extensions"].get(OPENCTI_EXTENSION).get("id")
+        ):
+            ids.append(item["extensions"][OPENCTI_EXTENSION]["id"])
+        return ids
+
     def enlist_element(
         self, item_id, raw_data, cleanup_inconsistent_bundle, parent_acc
     ):
@@ -173,6 +190,8 @@ def enlist_element(
             if is_compatible:
                 self.elements.append(item)
             self.cache_index[item_id] = item
+            for internal_id in self.get_internal_ids_in_extension(item):
+                self.cache_index[internal_id] = item
 
         return nb_deps
 
@@ -202,6 +221,8 @@ def split_bundle_with_expectations(
         # Build flat list of elements
         for item in bundle_data["objects"]:
             raw_data[item["id"]] = item
+            for internal_id in self.get_internal_ids_in_extension(item):
+                raw_data[internal_id] = item
         for item in bundle_data["objects"]:
             self.enlist_element(item["id"], raw_data, cleanup_inconsistent_bundle, [])
 

tests/01-unit/utils/test_opencti_stix2_splitter.py

@@ -43,7 +43,7 @@ def test_split_mono_entity_bundle():
     expectations, bundles = stix_splitter.split_bundle_with_expectations(content)
     assert expectations == 1
     json_bundle = json.loads(bundles[0])["objects"][0]
-    assert json_bundle["created_by_ref"] == "identity--not-available"
+    assert json_bundle["created_by_ref"] == "fa42a846-8d90-4e51-bc29-71d5b4802168"
     # Split with cleanup_inconsistent_bundle
     stix_splitter = OpenCTIStix2Splitter()
     expectations, bundles = stix_splitter.split_bundle_with_expectations(
@@ -76,6 +76,27 @@ def test_split_capec_bundle():
     assert expectations == 2610
 
 
+def test_split_internal_ids_bundle():
+    stix_splitter = OpenCTIStix2Splitter()
+    with open("./tests/data/bundle_with_internal_ids.json") as file:
+        content = file.read()
+    expectations, bundles = stix_splitter.split_bundle_with_expectations(content)
+    assert expectations == 4
+    # Split with cleanup_inconsistent_bundle
+    stix_splitter = OpenCTIStix2Splitter()
+    expectations, bundles = stix_splitter.split_bundle_with_expectations(
+        bundle=content, cleanup_inconsistent_bundle=True
+    )
+    assert expectations == 4
+    for bundle in bundles:
+        json_bundle = json.loads(bundle)
+        object_json = json_bundle["objects"][0]
+        if object_json["id"] == "relationship--10e8c71d-a1b4-4e35-bca8-2e4a3785ea04":
+            assert (
+                object_json["created_by_ref"] == "ced3e53e-9663-4c96-9c60-07d2e778d931"
+            )
+
+
 def test_split_missing_refs_bundle():
     stix_splitter = OpenCTIStix2Splitter()
     with open("./tests/data/missing_refs.json") as file:

tests/data/bundle_with_internal_ids.json

@@ -0,0 +1,54 @@
+{
+  "type": "bundle",
+  "id": "bundle--8c939929-688f-4a72-badb-3dd1bd6af0fa",
+  "objects": [
+    {
+      "id": "identity--67ffc076-1fba-5164-9df5-b7523092931e",
+      "spec_version": "2.1",
+      "type": "identity",
+      "extensions": {
+        "extension-definition--ea279b3e-5c71-4632-ac08-831c66a786ba": {
+          "extension_type": "property-extension",
+          "id": "ced3e53e-9663-4c96-9c60-07d2e778d931",
+          "type": "Organization",
+          "created_at": "2024-09-26T17:36:32.107Z",
+          "updated_at": "2024-09-26T17:36:32.107Z",
+          "is_inferred": false,
+          "creator_ids": [
+            "88ec0c6a-13ce-5e39-b486-354fe4a7084f"
+          ]
+        }
+      },
+      "created": "2024-09-26T17:36:32.107Z",
+      "modified": "2024-09-26T17:36:32.107Z",
+      "revoked": false,
+      "confidence": 100,
+      "lang": "en",
+      "name": "JRI",
+      "identity_class": "organization"
+    },
+    {
+      "id": "malware--faa5b705-cf44-4e50-8472-29e5fec43c3c",
+      "type": "malware",
+      "name": "malware 01",
+      "created": "2019-09-30T16:38:26.000Z",
+      "modified": "2020-01-14T14:01:48.288Z",
+      "x_opencti_id": "82316ffd-a0ec-4519-a454-6566f8f5676c"
+    },
+    {
+      "id": "malware--faa5b705-cf44-4e50-8472-29e5fec43c3d",
+      "type": "malware",
+      "name": "malware 02",
+      "created": "2019-09-30T16:38:26.000Z",
+      "modified": "2020-01-14T14:01:48.288Z"
+    },
+    {
+      "id": "relationship--10e8c71d-a1b4-4e35-bca8-2e4a3785ea04",
+      "type": "relationship",
+      "relationship_type": "part-of",
+      "created_by_ref": "ced3e53e-9663-4c96-9c60-07d2e778d931",
+      "source_ref": "82316ffd-a0ec-4519-a454-6566f8f5676c",
+      "target_ref": "malware--faa5b705-cf44-4e50-8472-29e5fec43c3d"
+    }
+  ]
+}

tests/data/mono-bundle-entity.json

@@ -4,7 +4,7 @@
     {
       "id": "marking-definition--fa42a846-8d90-4e51-bc29-71d5b4802168",
       "created": "2017-06-01T00:00:00.000Z",
-      "created_by_ref": "identity--not-available",
+      "created_by_ref": "fa42a846-8d90-4e51-bc29-71d5b4802168",
       "definition": {
         "statement": "Copyright 2015-2024, The MITRE Corporation. MITRE ATT&CK and ATT&CK are registered trademarks of The MITRE Corporation."
       },