From 4de71016153e0c5de080eea5df1ff7f5035cf721 Mon Sep 17 00:00:00 2001 From: Pauline Eustachy Date: Thu, 4 Dec 2025 15:25:06 +0100 Subject: [PATCH 1/6] feat: automated migration --- .../__metadata__/connector_config_schema.json | 75 ++++++++++ .../domaintools/src/connector/__init__.py | 6 +- .../src/connector/{core.py => connector.py} | 79 ++--------- .../domaintools/src/connector/settings.py | 48 +++++++ internal-enrichment/domaintools/src/main.py | 28 +++- .../domaintools/src/requirements.txt | 4 +- .../domaintools/tests/conftest.py | 4 + .../domaintools/tests/test-requirements.txt | 2 + .../domaintools/tests/test_main.py | 99 +++++++++++++ .../tests/tests_connector/test_settings.py | 132 ++++++++++++++++++ 10 files changed, 401 insertions(+), 76 deletions(-) create mode 100644 internal-enrichment/domaintools/__metadata__/connector_config_schema.json rename internal-enrichment/domaintools/src/connector/{core.py => connector.py} (79%) create mode 100644 internal-enrichment/domaintools/src/connector/settings.py create mode 100644 internal-enrichment/domaintools/tests/conftest.py create mode 100644 internal-enrichment/domaintools/tests/test-requirements.txt create mode 100644 internal-enrichment/domaintools/tests/test_main.py create mode 100644 internal-enrichment/domaintools/tests/tests_connector/test_settings.py diff --git a/internal-enrichment/domaintools/__metadata__/connector_config_schema.json b/internal-enrichment/domaintools/__metadata__/connector_config_schema.json new file mode 100644 index 00000000000..45aae567824 --- /dev/null +++ b/internal-enrichment/domaintools/__metadata__/connector_config_schema.json @@ -0,0 +1,75 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://www.filigran.io/connectors/domaintools_config.schema.json", + "type": "object", + "properties": { + "OPENCTI_URL": { + "description": "The base URL of the OpenCTI instance.", + "format": "uri", + "maxLength": 2083, + "minLength": 1, + "type": "string" + }, + "OPENCTI_TOKEN": { + "description": "The API token to connect to OpenCTI.", + "type": "string" + }, + "CONNECTOR_NAME": { + "default": "Domaintools", + "description": "The name of the connector.", + "type": "string" + }, + "CONNECTOR_SCOPE": { + "description": "The scope of the connector, e.g. 'flashpoint'.", + "items": { + "type": "string" + }, + "type": "array" + }, + "CONNECTOR_LOG_LEVEL": { + "default": "error", + "description": "The minimum level of logs to display.", + "enum": [ + "debug", + "info", + "warn", + "warning", + "error" + ], + "type": "string" + }, + "CONNECTOR_TYPE": { + "const": "INTERNAL_ENRICHMENT", + "default": "INTERNAL_ENRICHMENT", + "type": "string" + }, + "CONNECTOR_AUTO": { + "default": false, + "description": "Whether the connector should run automatically when an entity is created or updated.", + "type": "boolean" + }, + "DOMAINTOOLS_API_USERNAME": { + "default": "ChangeMe", + "description": "The username required for the authentication on DomainTools API.", + "type": "string" + }, + "DOMAINTOOLS_API_KEY": { + "default": "ChangeMe", + "description": "The password required for the authentication on DomainTools API.", + "format": "password", + "type": "string", + "writeOnly": true + }, + "DOMAINTOOLS_MAX_TLP": { + "default": "TLP:AMBER", + "description": "The maximal TLP of the observable being enriched.", + "type": "string" + } + }, + "required": [ + "OPENCTI_URL", + "OPENCTI_TOKEN", + "CONNECTOR_SCOPE" + ], + "additionalProperties": true +} \ No newline at end of file diff --git a/internal-enrichment/domaintools/src/connector/__init__.py b/internal-enrichment/domaintools/src/connector/__init__.py index ee08c0bb023..04734941f81 100644 --- a/internal-enrichment/domaintools/src/connector/__init__.py +++ b/internal-enrichment/domaintools/src/connector/__init__.py @@ -1,6 +1,6 @@ -# -*- coding: utf-8 -*- """DomainTools connector module.""" -from .core import DomainToolsConnector +from connector.connector import DomainToolsConnector +from connector.settings import ConnectorSettings -__all__ = ["DomainToolsConnector"] +__all__ = ["DomainToolsConnector", "ConnectorSettings"] diff --git a/internal-enrichment/domaintools/src/connector/core.py b/internal-enrichment/domaintools/src/connector/connector.py similarity index 79% rename from internal-enrichment/domaintools/src/connector/core.py rename to internal-enrichment/domaintools/src/connector/connector.py index 47d8737c1cd..ee40871a6d9 100644 --- a/internal-enrichment/domaintools/src/connector/core.py +++ b/internal-enrichment/domaintools/src/connector/connector.py @@ -1,15 +1,13 @@ -# -*- coding: utf-8 -*- """DomainTools enrichment module.""" from datetime import datetime -from pathlib import Path from typing import Dict import domaintools import stix2 import validators -import yaml -from pycti import Identity, OpenCTIConnectorHelper, get_config_variable +from connector.settings import ConnectorSettings +from pycti import Identity, OpenCTIConnectorHelper from .builder import DtBuilder from .constants import DEFAULT_RISK_SCORE, DOMAIN_FIELDS, EMAIL_FIELDS, EntityType @@ -21,42 +19,18 @@ class DomainToolsConnector: _DEFAULT_AUTHOR = "DomainTools" _CONNECTOR_RUN_INTERVAL_SEC = 60 * 60 - def __init__(self): - # Instantiate the connector helper from config - config_file_path = Path(__file__).parent.parent.resolve() / "config.yml" - config = ( - yaml.load(open(config_file_path, encoding="utf-8"), Loader=yaml.FullLoader) - if config_file_path.is_file() - else {} + def __init__(self, config: ConnectorSettings, helper: OpenCTIConnectorHelper): + self.config = config + self.helper = helper + self.api = domaintools.API( + self.config.domaintools.api_username, self.config.domaintools.api_key ) - self.helper = OpenCTIConnectorHelper(config, True) - - # DomainTools - api_username = get_config_variable( - "DOMAINTOOLS_API_USERNAME", - ["domaintools", "api_username"], - config, - ) - api_key = get_config_variable( - "DOMAINTOOLS_API_KEY", - ["domaintools", "api_key"], - config, - ) - self.api = domaintools.API(api_username, api_key) - - self.max_tlp = get_config_variable( - "DOMAINTOOLS_MAX_TLP", ["domaintools", "max_tlp"], config - ) - + self.max_tlp = self.config.domaintools.max_tlp self.author = stix2.Identity( id=Identity.generate_id(self._DEFAULT_AUTHOR, "organization"), name=self._DEFAULT_AUTHOR, identity_class="organization", - description=" DomainTools is a leading provider of Whois and other DNS" - " profile data for threat intelligence enrichment." - " It is a part of the Datacenter Group (DCL Group SA)." - " DomainTools data helps security analysts investigate malicious" - " activity on their networks.", + description=" DomainTools is a leading provider of Whois and other DNS profile data for threat intelligence enrichment. It is a part of the Datacenter Group (DCL Group SA). DomainTools data helps security analysts investigate malicious activity on their networks.", confidence=self.helper.connect_confidence_level, ) self.helper.metric.state("idle") @@ -102,36 +76,28 @@ def _enrich_domaintools(self, builder, opencti_entity) -> str: raise ValueError( f"Entity type of the observable: {opencti_entity['entity_type']} not supported." ) - for entry in results: self.helper.log_info(f"Starting enrichment of domain {entry['domain']}") - # Retrieve common properties for all relationships. builder.reset_score() score = entry.get("domain_risk", {}).get("risk_score", DEFAULT_RISK_SCORE) builder.set_score(score) - # Get the creation date / expiration date for the validity. creation_date = entry.get("create_date", {}).get("value", "") expiration_date = entry.get("expiration_date", {}).get("value", "") if creation_date != "" and expiration_date != "": creation_date = datetime.strptime(creation_date, "%Y-%m-%d") if expiration_date != "": expiration_date = datetime.strptime(expiration_date, "%Y-%m-%d") - if creation_date >= expiration_date: self.helper.log_warning( f"Expiration date {expiration_date} not after creation date {creation_date}, not using dates." ) creation_date = "" expiration_date = "" - - # In case of IP enrichment, create the domain as it might not exist. domain_source_id = ( builder.create_domain(entry["domain"]) if opencti_entity["entity_type"] == "IPv4-Addr" else opencti_entity["standard_id"] ) - - # Get ip for ip in entry.get("ip", ()): if "address" in ip: ip_id = builder.link_domain_resolves_to( @@ -145,21 +111,15 @@ def _enrich_domaintools(self, builder, opencti_entity) -> str: if ip_id is not None: for asn in ip.get("asn", ()): builder.link_ip_belongs_to_asn( - ip_id, - asn["value"], - creation_date, - expiration_date, + ip_id, asn["value"], creation_date, expiration_date ) - - # Get domains (name-server / mx) for category, description in DOMAIN_FIELDS.items(): for values in entry.get(category, ()): if (domain := values["domain"]["value"]) != entry["domain"]: if not validators.domain(domain): self.helper.metric.inc("error_count") self.helper.log_warning( - f"[DomainTools] domain {domain} is not correctly " - "formatted. Skipping." + f"[DomainTools] domain {domain} is not correctly formatted. Skipping." ) continue new_domain_id = builder.link_domain_resolves_to( @@ -170,7 +130,6 @@ def _enrich_domaintools(self, builder, opencti_entity) -> str: expiration_date, description, ) - # Add the related ips of the name server to the newly created domain. if new_domain_id is not None: for ip in values.get("ip", ()): builder.link_domain_resolves_to( @@ -181,8 +140,6 @@ def _enrich_domaintools(self, builder, opencti_entity) -> str: expiration_date, f"{description}-ip", ) - - # Emails for category, description in EMAIL_FIELDS.items(): emails = ( entry.get(category, ()) @@ -197,8 +154,6 @@ def _enrich_domaintools(self, builder, opencti_entity) -> str: expiration_date, description, ) - - # Domains of emails for domain in entry.get("email_domain", ()): if domain["value"] != entry["domain"]: builder.link_domain_resolves_to( @@ -209,8 +164,6 @@ def _enrich_domaintools(self, builder, opencti_entity) -> str: expiration_date, "email_domain", ) - - # Redirects (red) if (red := entry.get("redirect_domain", {}).get("value", "")) not in ( domain_source_id, "", @@ -223,7 +176,6 @@ def _enrich_domaintools(self, builder, opencti_entity) -> str: expiration_date, "redirect", ) - if len(builder.bundle) > 1: builder.send_bundle() self.helper.log_info( @@ -235,31 +187,24 @@ def _enrich_domaintools(self, builder, opencti_entity) -> str: def _process_file(self, stix_objects, opencti_entity): self.helper.metric.state("running") self.helper.metric.inc("run_count") - builder = DtBuilder(self.helper, self.author, stix_objects) - - # Enrichment using DomainTools API. result = self._enrich_domaintools(builder, opencti_entity) self.helper.metric.state("idle") return result def _process_message(self, data: Dict): opencti_entity = data["enrichment_entity"] - - # Extract TLP tlp = "TLP:CLEAR" for marking_definition in opencti_entity.get("objectMarking", []): if marking_definition["definition_type"] == "TLP": tlp = marking_definition["definition"] - if not OpenCTIConnectorHelper.check_max_tlp(tlp, self.max_tlp): raise ValueError( "Do not send any data, TLP of the observable is greater than MAX TLP" ) - stix_objects = data["stix_objects"] return self._process_file(stix_objects, opencti_entity) - def start(self): + def run(self): """Start the main loop.""" self.helper.listen(message_callback=self._process_message) diff --git a/internal-enrichment/domaintools/src/connector/settings.py b/internal-enrichment/domaintools/src/connector/settings.py new file mode 100644 index 00000000000..ffeabc805d2 --- /dev/null +++ b/internal-enrichment/domaintools/src/connector/settings.py @@ -0,0 +1,48 @@ +from connectors_sdk import ( + BaseConfigModel, + BaseConnectorSettings, + BaseInternalEnrichmentConnectorConfig, +) +from pydantic import Field, SecretStr + + +class InternalEnrichmentConnectorConfig(BaseInternalEnrichmentConnectorConfig): + """ + Override the `BaseInternalEnrichmentConnectorConfig` to add parameters and/or defaults + to the configuration for connectors of type `INTERNAL_ENRICHMENT`. + """ + + name: str = Field( + description="The name of the connector.", + default="Domaintools", + ) + + +class DomaintoolsConfig(BaseConfigModel): + """ + Define parameters and/or defaults for the configuration specific to the `DomaintoolsConnector`. + """ + + api_username: str = Field( + description="The username required for the authentication on DomainTools API.", + default="ChangeMe", + ) + api_key: SecretStr = Field( + description="The password required for the authentication on DomainTools API.", + default="ChangeMe", + ) + max_tlp: str = Field( + description="The maximal TLP of the observable being enriched.", + default="TLP:AMBER", + ) + + +class ConnectorSettings(BaseConnectorSettings): + """ + Override `BaseConnectorSettings` to include `InternalEnrichmentConnectorConfig` and `DomaintoolsConfig`. + """ + + connector: InternalEnrichmentConnectorConfig = Field( + default_factory=InternalEnrichmentConnectorConfig + ) + domaintools: DomaintoolsConfig = Field(default_factory=DomaintoolsConfig) diff --git a/internal-enrichment/domaintools/src/main.py b/internal-enrichment/domaintools/src/main.py index d51ce998625..e5d3ba045d9 100644 --- a/internal-enrichment/domaintools/src/main.py +++ b/internal-enrichment/domaintools/src/main.py @@ -1,8 +1,26 @@ -# -*- coding: utf-8 -*- -"""DomainTools connector main file.""" +import traceback -from connector import DomainToolsConnector +from connector import ConnectorSettings, DomainToolsConnector +from pycti import OpenCTIConnectorHelper if __name__ == "__main__": - connector = DomainToolsConnector() - connector.start() + """ + Entry point of the script + + - traceback.print_exc(): This function prints the traceback of the exception to the standard error (stderr). + The traceback includes information about the point in the program where the exception occurred, + which is very useful for debugging purposes. + - exit(1): effective way to terminate a Python program when an error is encountered. + It signals to the operating system and any calling processes that the program did not complete successfully. + """ + try: + settings = ConnectorSettings() + helper = OpenCTIConnectorHelper( + config=settings.to_helper_config(), playbook_compatible=True + ) + + connector = DomainToolsConnector(config=settings, helper=helper) + connector.run() + except Exception: + traceback.print_exc() + exit(1) diff --git a/internal-enrichment/domaintools/src/requirements.txt b/internal-enrichment/domaintools/src/requirements.txt index 481b9c753c5..63f0f66fb02 100644 --- a/internal-enrichment/domaintools/src/requirements.txt +++ b/internal-enrichment/domaintools/src/requirements.txt @@ -1,3 +1,5 @@ pycti==6.9.5 domaintools-api==2.7.0 -validators~=0.35.0 \ No newline at end of file +validators~=0.35.0 +pydantic >=2.8.2, <3 +connectors-sdk @ git+https://github.com/OpenCTI-Platform/connectors.git@master#subdirectory=connectors-sdk diff --git a/internal-enrichment/domaintools/tests/conftest.py b/internal-enrichment/domaintools/tests/conftest.py new file mode 100644 index 00000000000..5ee8fc0e226 --- /dev/null +++ b/internal-enrichment/domaintools/tests/conftest.py @@ -0,0 +1,4 @@ +import os +import sys + +sys.path.append(os.path.join(os.path.dirname(__file__), "..", "src")) diff --git a/internal-enrichment/domaintools/tests/test-requirements.txt b/internal-enrichment/domaintools/tests/test-requirements.txt new file mode 100644 index 00000000000..bdef682113c --- /dev/null +++ b/internal-enrichment/domaintools/tests/test-requirements.txt @@ -0,0 +1,2 @@ +-r ../src/requirements.txt +pytest==8.4.2 diff --git a/internal-enrichment/domaintools/tests/test_main.py b/internal-enrichment/domaintools/tests/test_main.py new file mode 100644 index 00000000000..6d1b5f80473 --- /dev/null +++ b/internal-enrichment/domaintools/tests/test_main.py @@ -0,0 +1,99 @@ +from typing import Any +from unittest.mock import MagicMock + +import pytest +from connector import ConnectorSettings, DomainToolsConnector +from pycti import OpenCTIConnectorHelper + + +@pytest.fixture +def mock_opencti_connector_helper(monkeypatch): + """Mock all heavy dependencies of OpenCTIConnectorHelper, typically API calls to OpenCTI.""" + + module_import_path = "pycti.connector.opencti_connector_helper" + monkeypatch.setattr(f"{module_import_path}.killProgramHook", MagicMock()) + monkeypatch.setattr(f"{module_import_path}.sched.scheduler", MagicMock()) + monkeypatch.setattr(f"{module_import_path}.ConnectorInfo", MagicMock()) + monkeypatch.setattr(f"{module_import_path}.OpenCTIApiClient", MagicMock()) + monkeypatch.setattr(f"{module_import_path}.OpenCTIConnector", MagicMock()) + monkeypatch.setattr(f"{module_import_path}.OpenCTIMetricHandler", MagicMock()) + monkeypatch.setattr(f"{module_import_path}.PingAlive", MagicMock()) + + +class StubConnectorSettings(ConnectorSettings): + """ + Subclass of `ConnectorSettings` (implementation of `BaseConnectorSettings`) for testing purpose. + It overrides `BaseConnectorSettings._load_config_dict` to return a fake but valid config dict. + """ + + @classmethod + def _load_config_dict(cls, _, handler) -> dict[str, Any]: + return handler( + { + "opencti": { + "url": "http://localhost:8080", + "token": "test-token", + }, + "connector": { + "id": "connector-id", + "name": "Test Connector", + "scope": "test, connector", + "log_level": "error", + "auto": True, + }, + "domaintools": { + "api_username": "test-username", + "api_key": "test-api-key", + "max_tlp": "TLP:AMBER", + }, + } + ) + + +def test_connector_settings_is_instantiated(): + """ + Test that the implementation of `BaseConnectorSettings` (from `connectors-sdk`) can be instantiated successfully: + - the implemented class MUST have a method `to_helper_config` (inherited from `BaseConnectorSettings`) + - the method `to_helper_config` MUST return a dict (as in base class) + """ + settings = StubConnectorSettings() + + assert isinstance(settings, ConnectorSettings) + assert isinstance(settings.to_helper_config(), dict) + + +def test_opencti_connector_helper_is_instantiated(mock_opencti_connector_helper): + """ + Test that `OpenCTIConnectorHelper` (from `pycti`) can be instantiated successfully: + - the value of `settings.to_helper_config` MUST be the expected dict for `OpenCTIConnectorHelper` + - the helper MUST be able to get its instance's attributes from the config dict + + :param mock_opencti_connector_helper: `OpenCTIConnectorHelper` is mocked during this test to avoid any external calls to OpenCTI API + """ + settings = StubConnectorSettings() + helper = OpenCTIConnectorHelper(config=settings.to_helper_config()) + + assert helper.opencti_url == "http://localhost:8080/" + assert helper.opencti_token == "test-token" + assert helper.connect_id == "connector-id" + assert helper.connect_name == "Test Connector" + assert helper.connect_scope == "test,connector" + assert helper.log_level == "ERROR" + assert helper.connect_auto == True + + +def test_connector_is_instantiated(mock_opencti_connector_helper): + """ + Test that the connector's main class can be instantiated successfully: + - the connector's main class MUST be able to access env/config vars through `self.config` + - the connector's main class MUST be able to access `pycti` API through `self.helper` + + :param mock_opencti_connector_helper: `OpenCTIConnectorHelper` is mocked during this test to avoid any external calls to OpenCTI API + """ + settings = StubConnectorSettings() + helper = OpenCTIConnectorHelper(config=settings.to_helper_config()) + + connector = DomainToolsConnector(config=settings, helper=helper) + + assert connector.config == settings + assert connector.helper == helper diff --git a/internal-enrichment/domaintools/tests/tests_connector/test_settings.py b/internal-enrichment/domaintools/tests/tests_connector/test_settings.py new file mode 100644 index 00000000000..2cb9433d633 --- /dev/null +++ b/internal-enrichment/domaintools/tests/tests_connector/test_settings.py @@ -0,0 +1,132 @@ +from typing import Any + +import pytest +from connector import ConnectorSettings +from connectors_sdk import BaseConfigModel, ConfigValidationError + + +@pytest.mark.parametrize( + "settings_dict", + [ + pytest.param( + { + "opencti": {"url": "http://localhost:8080", "token": "test-token"}, + "connector": { + "id": "connector-id", + "name": "Test Connector", + "scope": "test, connector", + "log_level": "error", + "auto": True, + }, + "domaintools": { + "api_username": "test-username", + "api_key": "test-api-key", + "max_tlp": "TLP:AMBER", + }, + }, + id="full_valid_settings_dict", + ), + pytest.param( + { + "opencti": {"url": "http://localhost:8080", "token": "test-token"}, + "connector": {"id": "connector-id", "scope": "test, connector"}, + "domaintools": { + "api_username": "test-username", + "api_key": "test-api-key", + "max_tlp": "TLP:AMBER", + }, + }, + id="minimal_valid_settings_dict", + ), + ], +) +def test_settings_should_accept_valid_input(settings_dict): + """ + Test that `ConnectorSettings` (implementation of `BaseConnectorSettings` from `connectors-sdk`) accepts valid input. + For the test purpose, `BaseConnectorSettings._load_config_dict` is overridden to return + a fake but valid dict (instead of the env/config vars parsed from `config.yml`, `.env` or env vars). + + :param settings_dict: The dict to use as `ConnectorSettings` input + """ + + class FakeConnectorSettings(ConnectorSettings): + """ + Subclass of `ConnectorSettings` (implementation of `BaseConnectorSettings`) for testing purpose. + It overrides `BaseConnectorSettings._load_config_dict` to return a fake but valid config dict. + """ + + @classmethod + def _load_config_dict(cls, _, handler) -> dict[str, Any]: + return handler(settings_dict) + + settings = FakeConnectorSettings() + assert isinstance(settings.opencti, BaseConfigModel) is True + assert isinstance(settings.connector, BaseConfigModel) is True + assert isinstance(settings.domaintools, BaseConfigModel) is True + + +@pytest.mark.parametrize( + "settings_dict, field_name", + [ + pytest.param({}, "settings", id="empty_settings_dict"), + pytest.param( + { + "opencti": {"url": "http://localhost:PORT", "token": "test-token"}, + "connector": { + "id": "connector-id", + "name": "Test Connector", + "scope": "test, connector", + "log_level": "error", + "auto": True, + }, + "domaintools": { + "api_username": "test-username", + "api_key": "test-api-key", + "max_tlp": "TLP:AMBER", + }, + }, + "opencti.url", + id="invalid_opencti_url", + ), + pytest.param( + { + "opencti": {"url": "http://localhost:8080", "token": "test-token"}, + "connector": { + "name": "Test Connector", + "scope": "test, connector", + "log_level": "error", + "auto": True, + }, + "domaintools": { + "api_username": "test-username", + "api_key": "test-api-key", + "max_tlp": "TLP:AMBER", + }, + }, + "connector.id", + id="missing_connector_id", + ), + ], +) +def test_settings_should_raise_when_invalid_input(settings_dict, field_name): + """ + Test that `ConnectorSettings` (implementation of `BaseConnectorSettings` from `connectors-sdk`) raises on invalid input. + For the test purpose, `BaseConnectorSettings._load_config_dict` is overridden to return + a fake and invalid dict (instead of the env/config vars parsed from `config.yml`, `.env` or env vars). + + :param settings_dict: The dict to use as `ConnectorSettings` input + """ + + class FakeConnectorSettings(ConnectorSettings): + """ + Subclass of `ConnectorSettings` (implementation of `BaseConnectorSettings`) for testing purpose. + It overrides `BaseConnectorSettings._load_config_dict` to return a fake but valid config dict. + """ + + @classmethod + def _load_config_dict(cls, _, handler) -> dict[str, Any]: + return handler(settings_dict) + + with pytest.raises(ConfigValidationError) as err: + FakeConnectorSettings() + assert str("Error validating configuration") in str(err) From 66be630c93fcd934340da0bc8cfe324a578eec94 Mon Sep 17 00:00:00 2001 From: Mariot Tsitoara Date: Tue, 23 Dec 2025 19:16:10 +0100 Subject: [PATCH 2/6] [domaintools]: fix connector problems --- .../__metadata__/connector_config_schema.json | 10 +++++---- .../domaintools/docker-compose.yml | 10 ++++----- .../domaintools/src/config.yml.sample | 11 +++++----- .../domaintools/src/connector/__init__.py | 4 ++-- .../domaintools/src/connector/builder.py | 10 ++++----- .../domaintools/src/connector/connector.py | 21 +++++++++---------- .../domaintools/src/connector/settings.py | 7 +++++-- internal-enrichment/domaintools/src/main.py | 3 ++- 8 files changed, 40 insertions(+), 36 deletions(-) diff --git a/internal-enrichment/domaintools/__metadata__/connector_config_schema.json b/internal-enrichment/domaintools/__metadata__/connector_config_schema.json index 45aae567824..97c6d33806a 100644 --- a/internal-enrichment/domaintools/__metadata__/connector_config_schema.json +++ b/internal-enrichment/domaintools/__metadata__/connector_config_schema.json @@ -20,7 +20,10 @@ "type": "string" }, "CONNECTOR_SCOPE": { - "description": "The scope of the connector, e.g. 'flashpoint'.", + "default": [ + "Domain-Name,Ipv4-Addr" + ], + "description": "The scope of the connector.", "items": { "type": "string" }, @@ -49,12 +52,10 @@ "type": "boolean" }, "DOMAINTOOLS_API_USERNAME": { - "default": "ChangeMe", "description": "The username required for the authentication on DomainTools API.", "type": "string" }, "DOMAINTOOLS_API_KEY": { - "default": "ChangeMe", "description": "The password required for the authentication on DomainTools API.", "format": "password", "type": "string", @@ -69,7 +70,8 @@ "required": [ "OPENCTI_URL", "OPENCTI_TOKEN", - "CONNECTOR_SCOPE" + "DOMAINTOOLS_API_USERNAME", + "DOMAINTOOLS_API_KEY" ], "additionalProperties": true } \ No newline at end of file diff --git a/internal-enrichment/domaintools/docker-compose.yml b/internal-enrichment/domaintools/docker-compose.yml index 60daed45263..daa3475da79 100644 --- a/internal-enrichment/domaintools/docker-compose.yml +++ b/internal-enrichment/domaintools/docker-compose.yml @@ -1,4 +1,3 @@ -version: '3' services: connector-domaintools: image: opencti/connector-domaintools:latest @@ -6,10 +5,11 @@ services: - OPENCTI_URL=http://localhost - OPENCTI_TOKEN=ChangeMe - CONNECTOR_ID=ChangeMe - - CONNECTOR_NAME=DomainTools - - CONNECTOR_SCOPE=Domain-Name,Ipv4-Addr - - CONNECTOR_AUTO=false # Enable/disable auto-enrichment of observables + #- CONNECTOR_LOG_LEVEL=info + #- CONNECTOR_NAME=DomainTools + #- CONNECTOR_SCOPE=Domain-Name,Ipv4-Addr + #- CONNECTOR_AUTO=false # Enable/disable auto-enrichment of observables - DOMAINTOOLS_API_USERNAME=ChangeMe - DOMAINTOOLS_API_KEY=ChangeMe - - DOMAINTOOLS_MAX_TLP=TLP:AMBER + #- DOMAINTOOLS_MAX_TLP=TLP:AMBER restart: always diff --git a/internal-enrichment/domaintools/src/config.yml.sample b/internal-enrichment/domaintools/src/config.yml.sample index a32e75ff81f..78c6624ce64 100644 --- a/internal-enrichment/domaintools/src/config.yml.sample +++ b/internal-enrichment/domaintools/src/config.yml.sample @@ -4,13 +4,12 @@ opencti: connector: id: 'ChangeMe' - name: 'DomainTools' - scope: 'Domain-Name,Ipv4-Addr' - auto: false # Enable/disable auto-enrichment of observables - confidence_level: 80 # From 0 (Unknown) to 100 (Fully trusted) - log_level: 'info' + #name: 'DomainTools' + #scope: 'Domain-Name,Ipv4-Addr' + #auto: false # Enable/disable auto-enrichment of observables + #log_level: 'info' domaintools: api_username: 'ChangeMe' api_key: 'ChangeMe' - max_tlp: 'TLP:AMBER' + #max_tlp: 'TLP:AMBER' diff --git a/internal-enrichment/domaintools/src/connector/__init__.py b/internal-enrichment/domaintools/src/connector/__init__.py index 04734941f81..a86420442ac 100644 --- a/internal-enrichment/domaintools/src/connector/__init__.py +++ b/internal-enrichment/domaintools/src/connector/__init__.py @@ -1,6 +1,6 @@ """DomainTools connector module.""" -from connector.connector import DomainToolsConnector -from connector.settings import ConnectorSettings +from .connector import DomainToolsConnector +from .settings import ConnectorSettings __all__ = ["DomainToolsConnector", "ConnectorSettings"] diff --git a/internal-enrichment/domaintools/src/connector/builder.py b/internal-enrichment/domaintools/src/connector/builder.py index 523e82ddd9b..be2b426cac5 100644 --- a/internal-enrichment/domaintools/src/connector/builder.py +++ b/internal-enrichment/domaintools/src/connector/builder.py @@ -6,6 +6,7 @@ import stix2 import validators +from connectors_sdk.models import OrganizationAuthor from pycti import STIX_EXT_OCTI_SCO, OpenCTIConnectorHelper, StixCoreRelationship from .constants import EntityType @@ -18,7 +19,7 @@ class DtBuilder: """ def __init__( - self, helper: OpenCTIConnectorHelper, author: stix2.Identity, stix_objects: [] + self, helper: OpenCTIConnectorHelper, author: OrganizationAuthor, stix_objects ): """Initialize DtBuilder.""" self.helper = helper @@ -26,8 +27,8 @@ def __init__( # Use custom properties to set the author and the confidence level of the object. self.extensions = {} - self.extensions[STIX_EXT_OCTI_SCO] = {"created_by_ref": author["id"]} - self.bundle = stix_objects + [self.author] + self.extensions[STIX_EXT_OCTI_SCO] = {"created_by_ref": author.id} + self.bundle = stix_objects + [self.author.to_stix2_object()] def reset_score(self): """Reset the score used.""" @@ -263,8 +264,7 @@ def create_relationship( Created relationship. """ kwargs = { - "created_by_ref": self.author, - "confidence": self.helper.connect_confidence_level, + "created_by_ref": self.author.id, } if description is not None: kwargs["description"] = description diff --git a/internal-enrichment/domaintools/src/connector/connector.py b/internal-enrichment/domaintools/src/connector/connector.py index ee40871a6d9..99a255d0a73 100644 --- a/internal-enrichment/domaintools/src/connector/connector.py +++ b/internal-enrichment/domaintools/src/connector/connector.py @@ -4,10 +4,9 @@ from typing import Dict import domaintools -import stix2 import validators -from connector.settings import ConnectorSettings -from pycti import Identity, OpenCTIConnectorHelper +from connectors_sdk.models import OrganizationAuthor +from pycti import OpenCTIConnectorHelper from .builder import DtBuilder from .constants import DEFAULT_RISK_SCORE, DOMAIN_FIELDS, EMAIL_FIELDS, EntityType @@ -19,19 +18,19 @@ class DomainToolsConnector: _DEFAULT_AUTHOR = "DomainTools" _CONNECTOR_RUN_INTERVAL_SEC = 60 * 60 - def __init__(self, config: ConnectorSettings, helper: OpenCTIConnectorHelper): + def __init__(self, config, helper: OpenCTIConnectorHelper): self.config = config self.helper = helper - self.api = domaintools.API( - self.config.domaintools.api_username, self.config.domaintools.api_key + self.api = domaintools.api.API( + self.config.domaintools.api_username, + self.config.domaintools.api_key.get_secret_value(), ) self.max_tlp = self.config.domaintools.max_tlp - self.author = stix2.Identity( - id=Identity.generate_id(self._DEFAULT_AUTHOR, "organization"), + self.author = OrganizationAuthor( name=self._DEFAULT_AUTHOR, - identity_class="organization", - description=" DomainTools is a leading provider of Whois and other DNS profile data for threat intelligence enrichment. It is a part of the Datacenter Group (DCL Group SA). DomainTools data helps security analysts investigate malicious activity on their networks.", - confidence=self.helper.connect_confidence_level, + description="DomainTools is a leading provider of Whois and other DNS profile data for " + "threat intelligence enrichment. It is a part of the Datacenter Group (DCL Group SA). " + "DomainTools data helps security analysts investigate malicious activity on their networks.", ) self.helper.metric.state("idle") diff --git a/internal-enrichment/domaintools/src/connector/settings.py b/internal-enrichment/domaintools/src/connector/settings.py index ffeabc805d2..0501b0f3d11 100644 --- a/internal-enrichment/domaintools/src/connector/settings.py +++ b/internal-enrichment/domaintools/src/connector/settings.py @@ -2,6 +2,7 @@ BaseConfigModel, BaseConnectorSettings, BaseInternalEnrichmentConnectorConfig, + ListFromString, ) from pydantic import Field, SecretStr @@ -16,6 +17,10 @@ class InternalEnrichmentConnectorConfig(BaseInternalEnrichmentConnectorConfig): description="The name of the connector.", default="Domaintools", ) + scope: ListFromString = Field( + description="The scope of the connector.", + default=["Domain-Name,Ipv4-Addr"], + ) class DomaintoolsConfig(BaseConfigModel): @@ -25,11 +30,9 @@ class DomaintoolsConfig(BaseConfigModel): api_username: str = Field( description="The username required for the authentication on DomainTools API.", - default="ChangeMe", ) api_key: SecretStr = Field( description="The password required for the authentication on DomainTools API.", - default="ChangeMe", ) max_tlp: str = Field( description="The maximal TLP of the observable being enriched.", diff --git a/internal-enrichment/domaintools/src/main.py b/internal-enrichment/domaintools/src/main.py index e5d3ba045d9..d742e60a492 100644 --- a/internal-enrichment/domaintools/src/main.py +++ b/internal-enrichment/domaintools/src/main.py @@ -1,6 +1,7 @@ import traceback -from connector import ConnectorSettings, DomainToolsConnector +from connector.connector import DomainToolsConnector +from connector.settings import ConnectorSettings from pycti import OpenCTIConnectorHelper if __name__ == "__main__": From a43234d91b196c59b00df0ed24aa972b9db9fbcd Mon Sep 17 00:00:00 2001 From: Hugo DUPRAS Date: Tue, 30 Dec 2025 18:21:46 +0100 Subject: [PATCH 3/6] fix: Fix scope default value as a list of string --- internal-enrichment/domaintools/src/connector/settings.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal-enrichment/domaintools/src/connector/settings.py b/internal-enrichment/domaintools/src/connector/settings.py index 0501b0f3d11..073caa74e84 100644 --- a/internal-enrichment/domaintools/src/connector/settings.py +++ b/internal-enrichment/domaintools/src/connector/settings.py @@ -19,7 +19,7 @@ class InternalEnrichmentConnectorConfig(BaseInternalEnrichmentConnectorConfig): ) scope: ListFromString = Field( description="The scope of the connector.", - default=["Domain-Name,Ipv4-Addr"], + default=["Domain-Name", "Ipv4-Addr"], ) From bf7035ca98db35cc2f9d8a42007ce49216572126 Mon Sep 17 00:00:00 2001 From: Hugo DUPRAS Date: Tue, 30 Dec 2025 18:23:04 +0100 Subject: [PATCH 4/6] chore: Update config Schema --- .../__metadata__/CONNECTOR_CONFIG_DOC.md | 18 ++++++++++++++++++ .../__metadata__/connector_config_schema.json | 3 ++- 2 files changed, 20 insertions(+), 1 deletion(-) create mode 100644 internal-enrichment/domaintools/__metadata__/CONNECTOR_CONFIG_DOC.md diff --git a/internal-enrichment/domaintools/__metadata__/CONNECTOR_CONFIG_DOC.md b/internal-enrichment/domaintools/__metadata__/CONNECTOR_CONFIG_DOC.md new file mode 100644 index 00000000000..7c31906c813 --- /dev/null +++ b/internal-enrichment/domaintools/__metadata__/CONNECTOR_CONFIG_DOC.md @@ -0,0 +1,18 @@ +# Connector Configurations + +Below is an exhaustive enumeration of all configurable parameters available, each accompanied by detailed explanations of their purposes, default behaviors, and usage guidelines to help you understand and utilize them effectively. + +### Type: `object` + +| Property | Type | Required | Possible values | Default | Description | +| -------- | ---- | -------- | --------------- | ------- | ----------- | +| OPENCTI_URL | `string` | ✅ | Format: [`uri`](https://json-schema.org/understanding-json-schema/reference/string#built-in-formats) | | The base URL of the OpenCTI instance. | +| OPENCTI_TOKEN | `string` | ✅ | string | | The API token to connect to OpenCTI. | +| DOMAINTOOLS_API_USERNAME | `string` | ✅ | string | | The username required for the authentication on DomainTools API. | +| DOMAINTOOLS_API_KEY | `string` | ✅ | Format: [`password`](https://json-schema.org/understanding-json-schema/reference/string#built-in-formats) | | The password required for the authentication on DomainTools API. | +| CONNECTOR_NAME | `string` | | string | `"Domaintools"` | The name of the connector. | +| CONNECTOR_SCOPE | `array` | | string | `["Domain-Name", "Ipv4-Addr"]` | The scope of the connector. | +| CONNECTOR_LOG_LEVEL | `string` | | `debug` `info` `warn` `warning` `error` | `"error"` | The minimum level of logs to display. | +| CONNECTOR_TYPE | `const` | | `INTERNAL_ENRICHMENT` | `"INTERNAL_ENRICHMENT"` | | +| CONNECTOR_AUTO | `boolean` | | boolean | `false` | Whether the connector should run automatically when an entity is created or updated. | +| DOMAINTOOLS_MAX_TLP | `string` | | string | `"TLP:AMBER"` | The maximal TLP of the observable being enriched. | diff --git a/internal-enrichment/domaintools/__metadata__/connector_config_schema.json b/internal-enrichment/domaintools/__metadata__/connector_config_schema.json index 97c6d33806a..a3b2e10cd19 100644 --- a/internal-enrichment/domaintools/__metadata__/connector_config_schema.json +++ b/internal-enrichment/domaintools/__metadata__/connector_config_schema.json @@ -21,7 +21,8 @@ }, "CONNECTOR_SCOPE": { "default": [ - "Domain-Name,Ipv4-Addr" + "Domain-Name", + "Ipv4-Addr" ], "description": "The scope of the connector.", "items": { From d9570d5ffca6f592e4a7cec75d9ae423b4e75a4e Mon Sep 17 00:00:00 2001 From: Hugo Dupras Date: Tue, 13 Jan 2026 09:19:52 +0100 Subject: [PATCH 5/6] Update internal-enrichment/domaintools/src/requirements.txt Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- internal-enrichment/domaintools/src/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal-enrichment/domaintools/src/requirements.txt b/internal-enrichment/domaintools/src/requirements.txt index 63f0f66fb02..4309fac130f 100644 --- a/internal-enrichment/domaintools/src/requirements.txt +++ b/internal-enrichment/domaintools/src/requirements.txt @@ -2,4 +2,4 @@ pycti==6.9.5 domaintools-api==2.7.0 validators~=0.35.0 pydantic >=2.8.2, <3 -connectors-sdk @ git+https://github.com/OpenCTI-Platform/connectors.git@master#subdirectory=connectors-sdk +connectors-sdk @ git+https://github.com/OpenCTI-Platform/connectors.git@6.9.5#subdirectory=connectors-sdk From 0bd559d5f9f78cb1202684ef0ba7330578ac6df0 Mon Sep 17 00:00:00 2001 From: Hugo DUPRAS Date: Tue, 13 Jan 2026 09:40:43 +0100 Subject: [PATCH 6/6] chore: Set `manager_supported` flag to True --- .../domaintools/__metadata__/connector_manifest.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal-enrichment/domaintools/__metadata__/connector_manifest.json b/internal-enrichment/domaintools/__metadata__/connector_manifest.json index 605f08fec0a..07d777b6c98 100644 --- a/internal-enrichment/domaintools/__metadata__/connector_manifest.json +++ b/internal-enrichment/domaintools/__metadata__/connector_manifest.json @@ -14,7 +14,7 @@ "support_version": ">=5.6.1", "subscription_link": null, "source_code": "https://github.com/OpenCTI-Platform/connectors/tree/master/internal-enrichment/domaintools", - "manager_supported": false, + "manager_supported": true, "container_version": "rolling", "container_image": "opencti/connector-domaintools", "container_type": "INTERNAL_ENRICHMENT"