Merge branch 'master' into release/6.6.0 after release 6.5.2

Author: aHenryJard

.drone.yml

@@ -65,7 +65,7 @@ services:
   - name: redis
     image: redis:7.4.2
   - name: elastic
-    image: docker.elastic.co/elasticsearch/elasticsearch:8.17.0
+    image: docker.elastic.co/elasticsearch/elasticsearch:8.17.2
     environment:
       discovery.type: single-node
       xpack.security.enabled: false

pycti/__init__.py

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-__version__ = "6.5.0"
+__version__ = "6.5.2"
 
 from .api.opencti_api_client import OpenCTIApiClient
 from .api.opencti_api_connector import OpenCTIApiConnector

pycti/api/opencti_api_client.py

@@ -3,7 +3,7 @@
 import datetime
 import io
 import json
-from typing import Union
+from typing import Dict, Tuple, Union
 
 import magic
 import requests
@@ -88,13 +88,13 @@ class OpenCTIApiClient:
     :param log_level: log level for the client
     :type log_level: str, optional
     :param ssl_verify: Requiring the requests to verify the TLS certificate at the server.
-    :type ssl_verify: bool, optional
+    :type ssl_verify: bool, str, optional
     :param proxies:
     :type proxies: dict, optional, The proxy configuration, would have `http` and `https` attributes. Defaults to {}
         ```
         proxies: {
-            "http: "http://my_proxy:8080"
-            "https: "http://my_proxy:8080"
+            "http": "http://my_proxy:8080"
+            "https": "http://my_proxy:8080"
         }
         ```
     :param json_logging: format the logs as json if set to True
@@ -107,14 +107,14 @@ class OpenCTIApiClient:
 
     def __init__(
         self,
-        url,
-        token,
+        url: str,
+        token: str,
         log_level="info",
-        ssl_verify=False,
-        proxies=None,
+        ssl_verify: Union[bool, str] = False,
+        proxies: Union[Dict[str, str], None] = None,
         json_logging=False,
         bundle_send_to_queue=True,
-        cert=None,
+        cert: Union[str, Tuple[str, str], None] = None,
         auth=None,
         perform_health_check=True,
     ):
@@ -762,12 +762,13 @@ def upload_pending_file(self, **kwargs):
         data = kwargs.get("data", None)
         mime_type = kwargs.get("mime_type", "text/plain")
         entity_id = kwargs.get("entity_id", None)
+        file_markings = kwargs.get("file_markings", [])
 
         if file_name is not None:
             self.app_logger.info("Uploading a file.")
             query = """
-                    mutation UploadPending($file: Upload!, $entityId: String) {
-                        uploadPending(file: $file, entityId: $entityId) {
+                    mutation UploadPending($file: Upload!, $entityId: String, $file_markings: [String!]) {
+                        uploadPending(file: $file, entityId: $entityId, file_markings: $file_markings) {
                             id
                             name
                         }
@@ -781,7 +782,11 @@ def upload_pending_file(self, **kwargs):
                     mime_type = magic.from_file(file_name, mime=True)
             return self.query(
                 query,
-                {"file": (File(file_name, data, mime_type)), "entityId": entity_id},
+                {
+                    "file": (File(file_name, data, mime_type)),
+                    "entityId": entity_id,
+                    "file_markings": file_markings,
+                },
             )
         else:
             self.app_logger.error("[upload] Missing parameter: file_name")

pycti/connector/opencti_connector_helper.py

@@ -769,6 +769,9 @@ class TimeUnit(Enum):
     def __init__(self, config: Dict, playbook_compatible=False) -> None:
         sys.excepthook = killProgramHook
 
+        # Cache
+        self.stream_collections = {}
+
         # Load API config
         self.config = config
         self.opencti_url = get_config_variable(
@@ -1063,6 +1066,9 @@ def get_stream_collection(self):
                     "stream_live": True,
                     "stream_public": False,
                 }
+            # Get from cache
+            elif self.connect_live_stream_id in self.stream_collections:
+                return self.stream_collections[self.connect_live_stream_id]
             else:
                 query = """
                     query StreamCollection($id: String!) {
@@ -1076,6 +1082,10 @@ def get_stream_collection(self):
                     }
                 """
                 result = self.api.query(query, {"id": self.connect_live_stream_id})
+                # Put in cache
+                self.stream_collections[self.connect_live_stream_id] = result["data"][
+                    "streamCollection"
+                ]
                 return result["data"]["streamCollection"]
         else:
             raise ValueError("This connector is not connected to any stream")
@@ -1581,6 +1591,7 @@ def send_stix2_bundle(self, bundle: str, **kwargs) -> list:
         event_version = kwargs.get("event_version", None)
         bypass_validation = kwargs.get("bypass_validation", False)
         entity_id = kwargs.get("entity_id", None)
+        file_markings = kwargs.get("file_markings", None)
         file_name = kwargs.get("file_name", None)
         bundle_send_to_queue = kwargs.get("send_to_queue", self.bundle_send_to_queue)
         cleanup_inconsistent_bundle = kwargs.get("cleanup_inconsistent_bundle", False)
@@ -1648,6 +1659,7 @@ def send_stix2_bundle(self, bundle: str, **kwargs) -> list:
                     data=bundle,
                     mime_type="application/json",
                     entity_id=entity_id,
+                    file_markings=file_markings,
                 )
                 return []
             elif validation_mode == "draft" and not draft_id:

pycti/utils/opencti_logger.py

@@ -1,5 +1,5 @@
-import datetime
 import logging
+from datetime import datetime, timezone
 
 from pythonjsonlogger import jsonlogger
 
@@ -9,8 +9,8 @@ def add_fields(self, log_record, record, message_dict):
         super(CustomJsonFormatter, self).add_fields(log_record, record, message_dict)
         if not log_record.get("timestamp"):
             # This doesn't use record.created, so it is slightly off
-            now = datetime.datetime.utcnow().strftime("%Y-%m-%dT%H:%M:%S.%fZ")
-            log_record["timestamp"] = now
+            now = datetime.now(tz=timezone.utc)
+            log_record["timestamp"] = now.strftime("%Y-%m-%dT%H:%M:%S.%fZ")
         if log_record.get("level"):
             log_record["level"] = log_record["level"].upper()
         else:

renovate.json

@@ -1,23 +1,26 @@
-{
-  "$schema": "https://docs.renovatebot.com/renovate-schema.json",
-  "extends": [
-    "config:base"
-  ],
-  "labels": [
-    "dependencies",
-    "filigran team"
-  ],
-  "prConcurrentLimit": 2,
-  "packageRules": [
-    {
-      "matchUpdateTypes": ["minor"],
-      "prPriority": 5
-    }
-  ],
-  "timezone": "Europe/Paris",
-  "schedule": [
-    "after 10pm and before 5am every weekday",
-    "every weekend"
-  ],
-  "updateNotScheduled": false
-}
+{
+  "$schema": "https://docs.renovatebot.com/renovate-schema.json",
+  "extends": [
+    "config:recommended"
+  ],
+  "labels": [
+    "dependencies",
+    "filigran team"
+  ],
+  "prConcurrentLimit": 2,
+  "packageRules": [
+    {
+      "matchUpdateTypes": [
+        "minor"
+      ],
+      "prPriority": 5
+    }
+  ],
+  "timezone": "Europe/Paris",
+  "schedule": [
+    "after 10pm every weekday",
+    "every weekend",
+    "before 5am every weekday"
+  ],
+  "updateNotScheduled": false
+}

requirements.txt

@@ -5,7 +5,7 @@ python-magic~=0.4.27; sys_platform == 'linux' or sys_platform == 'darwin'
 python-magic-bin~=0.4.14; sys_platform == 'win32'
 python_json_logger~=2.0.4
 PyYAML~=6.0
-pydantic~=2.10.4
+pydantic>=2.8.2,<2.11
 requests~=2.32.2
 setuptools~=71.1.0
 cachetools~=5.5.0

setup.cfg

@@ -37,7 +37,7 @@ install_requires =
     # Filigran
     datefinder~=0.7.3
     pika~=1.3.0
-    pydantic~=2.10.4
+    pydantic>=2.8.2,<2.11.0
     python-magic~=0.4.27; sys_platform == 'linux' or sys_platform == 'darwin'
     python-magic-bin~=0.4.14; sys_platform == 'win32'
     python_json_logger~=2.0.4
@@ -57,8 +57,8 @@ install_requires =
 dev =
     black~=24.4.0
     build~=1.2.1
-    isort~=5.13.0
-    types-pytz~=2024.2.0.20241221
+    isort~=6.0.0
+    types-pytz~=2025.1.0.20250204
     pre-commit~=3.8.0
     pytest-cases~=3.8.0
     pytest-cov~=5.0.0