Merge branch 'master' into release/6.5.0

Author: JeremyCloarec

.circleci/config.yml

@@ -1,7 +1,7 @@
 ---
 version: 2.1
 orbs:
-  slack: circleci/slack@4.13.3
+  slack: circleci/slack@4.15.0
   ms-teams: cloudradar-monitoring/[email protected]
 jobs:
   ensure_formatting:

.drone.yml

@@ -63,9 +63,9 @@ steps:
 
 services:
   - name: redis
-    image: redis:7.4.1
+    image: redis:7.4.2
   - name: elastic
-    image: docker.elastic.co/elasticsearch/elasticsearch:8.15.3
+    image: docker.elastic.co/elasticsearch/elasticsearch:8.17.0
     environment:
       discovery.type: single-node
       xpack.security.enabled: false

docs/requirements.txt

@@ -1,4 +1,4 @@
 autoapi==2.0.1
-sphinx==7.4.6
-sphinx-autodoc-typehints==2.2.3
-sphinx_rtd_theme==2.0.0
+sphinx==8.1.3
+sphinx-autodoc-typehints==2.5.0
+sphinx_rtd_theme==3.0.2

pycti/__init__.py

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-__version__ = "6.4.4"
+__version__ = "6.4.8"
 
 from .api.opencti_api_client import OpenCTIApiClient
 from .api.opencti_api_connector import OpenCTIApiConnector

pycti/api/opencti_api_client.py

@@ -224,8 +224,11 @@ def set_synchronized_upsert_header(self, synchronized):
     def set_previous_standard_header(self, previous_standard):
         self.request_headers["previous-standard"] = previous_standard
 
-    def get_request_headers(self):
-        return self.request_headers
+    def get_request_headers(self, hide_token=True):
+        request_headers_copy = self.request_headers.copy()
+        if hide_token and "Authorization" in request_headers_copy:
+            request_headers_copy["Authorization"] = "*****"
+        return request_headers_copy
 
     def set_retry_number(self, retry_number):
         self.request_headers["opencti-retry-number"] = (
@@ -408,7 +411,7 @@ def health_check(self):
             self.app_logger.info("Health check (platform version)...")
             test = self.query(
                 """
-                  query {
+                  query healthCheck {
                     about {
                       version
                     }

pycti/entities/opencti_case_rfi.py

@@ -318,6 +318,8 @@ def __init__(self, opencti):
             modified
             name
             description
+            severity
+            priority
             information_types
             objects(all: true) {
                 edges {
@@ -678,6 +680,8 @@ def create(self, **kwargs):
         object_participant = kwargs.get("objectParticipant", None)
         external_references = kwargs.get("externalReferences", None)
         revoked = kwargs.get("revoked", None)
+        severity = kwargs.get("severity", None)
+        priority = kwargs.get("priority", None)
         confidence = kwargs.get("confidence", None)
         lang = kwargs.get("lang", None)
         created = kwargs.get("created", None)
@@ -716,6 +720,8 @@ def create(self, **kwargs):
                         "objects": objects,
                         "externalReferences": external_references,
                         "revoked": revoked,
+                        "severity": severity,
+                        "priority": priority,
                         "confidence": confidence,
                         "lang": lang,
                         "created": created,
@@ -881,6 +887,8 @@ def import_from_stix2(self, **kwargs):
                     else None
                 ),
                 revoked=stix_object["revoked"] if "revoked" in stix_object else None,
+                severity=stix_object["severity"] if "severity" in stix_object else None,
+                priority=stix_object["priority"] if "priority" in stix_object else None,
                 confidence=(
                     stix_object["confidence"] if "confidence" in stix_object else None
                 ),

pycti/entities/opencti_case_rft.py

@@ -318,6 +318,8 @@ def __init__(self, opencti):
                 modified
                 name
                 description
+                severity
+                priority
                 takedown_types
                 objects(all: true) {
                     edges {
@@ -677,6 +679,8 @@ def create(self, **kwargs):
         object_participant = kwargs.get("objectParticipant", None)
         external_references = kwargs.get("externalReferences", None)
         revoked = kwargs.get("revoked", None)
+        severity = kwargs.get("severity", None)
+        priority = kwargs.get("priority", None)
         confidence = kwargs.get("confidence", None)
         lang = kwargs.get("lang", None)
         created = kwargs.get("created", None)
@@ -715,6 +719,8 @@ def create(self, **kwargs):
                         "objects": objects,
                         "externalReferences": external_references,
                         "revoked": revoked,
+                        "severity": severity,
+                        "priority": priority,
                         "confidence": confidence,
                         "lang": lang,
                         "created": created,
@@ -879,6 +885,8 @@ def import_from_stix2(self, **kwargs):
                     else None
                 ),
                 revoked=stix_object["revoked"] if "revoked" in stix_object else None,
+                severity=stix_object["severity"] if "severity" in stix_object else None,
+                priority=stix_object["priority"] if "priority" in stix_object else None,
                 confidence=(
                     stix_object["confidence"] if "confidence" in stix_object else None
                 ),

pycti/entities/opencti_indicator.py

@@ -63,8 +63,6 @@ def list(self, **kwargs):
         get_all = kwargs.get("getAll", False)
         with_pagination = kwargs.get("withPagination", False)
         with_files = kwargs.get("withFiles", False)
-        if get_all:
-            first = 100
 
         self.opencti.app_logger.info(
             "Listing Indicators with filters", {"filters": json.dumps(filters)}

pycti/entities/opencti_marking_definition.py

@@ -25,16 +25,29 @@ def __init__(self, opencti):
         """
 
     @staticmethod
-    def generate_id(definition, definition_type):
-        data = {"definition": definition, "definition_type": definition_type}
+    def generate_id(definition_type, definition):
+        # Handle static IDs from OpenCTI
+        if definition_type == "TLP":
+            if definition == "TLP:CLEAR" or definition == "TLP:WHITE":
+                return "marking-definition--613f2e26-407d-48c7-9eca-b8e91df99dc9"
+            if definition == "TLP:GREEN":
+                return "marking-definition--34098fce-860f-48ae-8e50-ebd3cc5e41da"
+            if definition == "TLP:AMBER":
+                return "marking-definition--f88d31f6-486f-44da-b317-01333bde0b82"
+            if definition == "TLP:AMBER+STRICT":
+                return "marking-definition--826578e1-40ad-459f-bc73-ede076f81f37"
+            if definition == "TLP:RED":
+                return "marking-definition--5e57c739-391a-4eb3-b6be-7d15ca92d5ed"
+        # Generate IDs
+        data = {"definition_type": definition_type, "definition": definition}
         data = canonicalize(data, utf8=False)
         id = str(uuid.uuid5(uuid.UUID("00abedb4-aa42-466c-9c01-fed23315a9b7"), data))
         return "marking-definition--" + id
 
     @staticmethod
     def generate_id_from_data(data):
         return MarkingDefinition.generate_id(
-            data["definition"], data["definition_type"]
+            data["definition_type"], data["definition"]
         )
 
     """

pycti/utils/opencti_stix2.py

@@ -1378,7 +1378,7 @@ def import_sighting(
             created=stix_sighting["created"] if "created" in stix_sighting else None,
             modified=stix_sighting["modified"] if "modified" in stix_sighting else None,
             confidence=(
-                stix_sighting["confidence"] if "confidence" in stix_sighting else 15
+                stix_sighting["confidence"] if "confidence" in stix_sighting else None
             ),
             createdBy=extras["created_by_id"] if "created_by_id" in extras else None,
             objectMarking=(
@@ -2585,9 +2585,6 @@ def import_item(
             in_retry = processing_count < PROCESSING_COUNT
             # Platform is under heavy load, wait for unlock & retry indefinitely.
             if ERROR_TYPE_LOCK in error_msg:
-                worker_logger.info(
-                    "Message reprocess for lock rejection", {"count": processing_count}
-                )
                 bundles_lock_error_counter.add(1)
                 sleep_jitter = round(random.uniform(1, 3), 2)
                 time.sleep(sleep_jitter)
@@ -2596,10 +2593,6 @@ def import_item(
                 )
             # Platform detects a missing reference and have to retry
             elif ERROR_TYPE_MISSING_REFERENCE in error_msg and in_retry:
-                worker_logger.info(
-                    "Message reprocess for missing reference",
-                    {"count": processing_count},
-                )
                 bundles_missing_reference_error_counter.add(1)
                 sleep_jitter = round(random.uniform(1, 3), 2)
                 time.sleep(sleep_jitter)
@@ -2608,8 +2601,7 @@ def import_item(
                 )
             # A bad gateway error occurs
             elif ERROR_TYPE_BAD_GATEWAY in error_msg:
-                worker_logger.error("A connection error occurred")
-                worker_logger.info(
+                worker_logger.error(
                     "Message reprocess for bad gateway",
                     {"count": processing_count},
                 )
@@ -2620,8 +2612,7 @@ def import_item(
                 )
             # Request timeout error occurs
             elif ERROR_TYPE_TIMEOUT in error_msg:
-                worker_logger.error("A connection error occurred")
-                worker_logger.info(
+                worker_logger.error(
                     "Message reprocess for request timed out",
                     {"count": processing_count},
                 )
@@ -2634,10 +2625,6 @@ def import_item(
             # That also works for missing reference with too much execution
             else:
                 bundles_technical_error_counter.add(1)
-                worker_logger.error(
-                    "Error executing import",
-                    {"count": processing_count, "reason": error},
-                )
                 if work_id is not None:
                     item_str = json.dumps(item)
                     self.opencti.work.report_expectation(