diff --git a/.circleci/config.yml b/.circleci/config.yml index 4788c463..c0ae7a06 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -38,7 +38,7 @@ jobs: - slack/notify: event: fail template: basic_fail_1 - test: + test-nuclei: working_directory: ~/openaev docker: - image: cimg/python:3.13 @@ -62,6 +62,54 @@ jobs: working_directory: ~/openaev/nuclei name: Tests for nuclei injector command: python -m unittest + test-nmap: + working_directory: ~/openaev + docker: + - image: cimg/python:3.13 + steps: + - checkout + - setup_remote_docker + - run: + working_directory: ~/openaev/nmap + name: Install dependencies for nmap injector + command: pip install -r requirements.txt + - run: + working_directory: ~/openaev/nmap + name: Overwrite pyoaev with correct version from CI + command: | + if [ "${CIRCLE_BRANCH}" = "main" ]; then + pip install --force-reinstall git+https://github.com/OpenAEV-Platform/client-python.git@main + else + pip install --force-reinstall git+https://github.com/OpenAEV-Platform/client-python.git@release/current + fi + - run: + working_directory: ~/openaev/nmap + name: Tests for nmap injector + command: python -m unittest + test-http-query: + working_directory: ~/openaev + docker: + - image: cimg/python:3.13 + steps: + - checkout + - setup_remote_docker + - run: + working_directory: ~/openaev/http-query + name: Install dependencies for http-query injector + command: pip install -r src/requirements.txt + - run: + working_directory: ~/openaev/http-query + name: Overwrite pyoaev with correct version from CI + command: | + if [ "${CIRCLE_BRANCH}" = "main" ]; then + pip install --force-reinstall git+https://github.com/OpenAEV-Platform/client-python.git@main + else + pip install --force-reinstall git+https://github.com/OpenAEV-Platform/client-python.git@release/current + fi + - run: + working_directory: ~/openaev/http-query + name: Tests for http-query injector + command: python -m unittest build_1: working_directory: ~/openaev docker: @@ -245,7 +293,9 @@ workflows: jobs: - ensure_formatting - linter - - test + - test-nmap + - test-nuclei + - test-http-query - build_1: filters: tags: @@ -256,7 +306,9 @@ workflows: requires: - ensure_formatting - linter - - test + - test-nmap + - test-nuclei + - test-http-query filters: branches: only: @@ -265,7 +317,9 @@ workflows: requires: - ensure_formatting - linter - - test + - test-nmap + - test-nuclei + - test-http-query filters: branches: only: diff --git a/.github/workflows/validate-pr-title.yml b/.github/workflows/validate-pr-title.yml new file mode 100644 index 00000000..f9160265 --- /dev/null +++ b/.github/workflows/validate-pr-title.yml @@ -0,0 +1,32 @@ +name: "Validate PR Title" + +on: + pull_request: + types: [ opened, edited, reopened, ready_for_review, synchronize ] + +jobs: + validate-pr-title: + runs-on: ubuntu-latest + permissions: + contents: read + pull-requests: write + + steps: + - name: Check PR title format + shell: bash + run: | + TITLE="${{ github.event.pull_request.title }}" + echo "PR title: $TITLE" + + # Regex for: + # [category/subcategory] type(scope?): description (#123?) + PATTERN='^\[([a-z]+(/[a-z]+)*)\] (feat|fix|chore|docs|style|refactor|perf|test|build|ci|revert)(\([a-z]+\))?: [a-z].*( \(#[0-9]+\))$' + + if [[ ! "$TITLE" =~ $PATTERN ]]; then + echo "❌ Invalid PR title." + echo "Required format:" + echo "[category] type(scope?): description (#123)" + exit 1 + fi + + echo "✅ PR title is valid." \ No newline at end of file diff --git a/aws/docker-compose.yml b/aws/docker-compose.yml index 8de4ae9f..05407c76 100644 --- a/aws/docker-compose.yml +++ b/aws/docker-compose.yml @@ -1,7 +1,7 @@ version: '3' services: injector-aws: - image: openaev/injector-aws:2.0.4 + image: openaev/injector-aws:2.0.5 environment: - OPENAEV_URL=http://localhost - OPENAEV_TOKEN=ChangeMe diff --git a/aws/src/requirements.txt b/aws/src/requirements.txt index ba754a18..0b41f2f1 100644 --- a/aws/src/requirements.txt +++ b/aws/src/requirements.txt @@ -1,3 +1,3 @@ -pyoaev==2.0.4 +pyoaev==2.0.5 pacu>=1.5.0 awscli>=1.29.0 diff --git a/http-query/README.md b/http-query/README.md index 5e4e3a64..3ba2f234 100644 --- a/http-query/README.md +++ b/http-query/README.md @@ -84,3 +84,14 @@ python3 openaev_http.py ## Behavior This injector enables new inject contracts, allowing for API calls of the Get, Post, and Put types. + +### Passing headers with the request +The contracts created in OpenAEV include an optional "Headers" parameter. While this field is technically a free-form +text input, the contents passed to it are expected to be in a certain format: `key=value` and together separated with a comma `,`. + +Note the pattern supports whitespace within the keys and values. + +Example: +```plaintext +content-type=application/json,x-custom-header=value for the header +``` \ No newline at end of file diff --git a/http-query/docker-compose.yml b/http-query/docker-compose.yml index 2b97a15f..7a3f1cea 100644 --- a/http-query/docker-compose.yml +++ b/http-query/docker-compose.yml @@ -1,7 +1,7 @@ version: '3' services: injector-http-query: - image: openaev/injector-http-query:2.0.4 + image: openaev/injector-http-query:2.0.5 environment: - OPENAEV_URL=http://localhost - OPENAEV_TOKEN=ChangeMe diff --git a/http-query/src/helpers/__init__.py b/http-query/src/helpers/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/http-query/src/helpers/helpers.py b/http-query/src/helpers/helpers.py new file mode 100644 index 00000000..55bfa0f0 --- /dev/null +++ b/http-query/src/helpers/helpers.py @@ -0,0 +1,42 @@ +class HTTPHelpers: + @staticmethod + def parse_headers(headers_str): + if isinstance(headers_str, list): + return headers_str + headers_list = [] + for kv in headers_str.split(","): + if "=" in kv: + k, v = kv.split("=", 1) + headers_list.append({"key": k.strip(), "value": v.strip()}) + return headers_list + + @staticmethod + def parse_parts(parts_str): + if isinstance(parts_str, list): + return parts_str + parts_list = [] + for kv in parts_str.split("&"): + if "=" in kv: + k, v = kv.split("=", 1) + parts_list.append({"key": k.strip(), "value": v.strip()}) + return parts_list + + @staticmethod + def request_data_parts_body(request_data): + parts = HTTPHelpers.parse_parts( + request_data["injection"]["inject_content"]["parts"] + ) + keys = list(map(lambda p: p["key"], parts)) + values = list(map(lambda p: p["value"], parts)) + return dict(zip(keys, values)) + + @staticmethod + def response_parsing(response): + success = 200 <= response.status_code < 300 + success_status = "SUCCESS" if success else "ERROR" + return { + "url": response.url, + "code": response.status_code, + "status": success_status, + "message": response.text, + } diff --git a/http-query/src/openaev_http.py b/http-query/src/openaev_http.py index 483e5273..7a682506 100644 --- a/http-query/src/openaev_http.py +++ b/http-query/src/openaev_http.py @@ -11,6 +11,7 @@ HTTP_RAW_PUT_CONTRACT, HttpContracts, ) +from helpers.helpers import HTTPHelpers from pyoaev.helpers import OpenAEVConfigHelper, OpenAEVInjectorHelper @@ -43,24 +44,6 @@ def __init__(self): self.config, open("img/icon-http.png", "rb") ) - @staticmethod - def _request_data_parts_body(request_data): - parts = request_data["injection"]["inject_content"]["parts"] - keys = list(map(lambda p: p["key"], parts)) - values = list(map(lambda p: p["value"], parts)) - return dict(zip(keys, values)) - - @staticmethod - def _response_parsing(response): - success = 200 <= response.status_code < 300 - success_status = "SUCCESS" if success else "ERROR" - return { - "url": response.url, - "code": response.status_code, - "status": success_status, - "message": response.text, - } - def attachments_to_files(self, request_data): documents = request_data["injection"].get("inject_documents", []) attachments = list(filter(lambda d: d["document_attached"] is True, documents)) @@ -73,7 +56,9 @@ def attachments_to_files(self, request_data): def http_execution(self, data: Dict): # Build headers - inject_headers = data["injection"]["inject_content"].get("headers", []) + inject_headers = HTTPHelpers.parse_headers( + data["injection"]["inject_content"].get("headers", []) + ) headers = {} for header_definition in inject_headers: headers[header_definition["key"]] = header_definition["value"] @@ -93,35 +78,35 @@ def http_execution(self, data: Dict): # Get if inject_contract == HTTP_GET_CONTRACT: response = session.get(url=url, headers=headers) - return self._response_parsing(response) + return HTTPHelpers.response_parsing(response) # Post if inject_contract == HTTP_RAW_POST_CONTRACT: body = data["injection"]["inject_content"]["body"] response = session.post( url=url, headers=headers, data=body, files=http_files ) - return self._response_parsing(response) + return HTTPHelpers.response_parsing(response) # Put if inject_contract == HTTP_RAW_PUT_CONTRACT: body = data["injection"]["inject_content"]["body"] response = session.put( url=url, headers=headers, data=body, files=http_files ) - return self._response_parsing(response) + return HTTPHelpers.response_parsing(response) # Form Post if inject_contract == HTTP_FORM_POST_CONTRACT: - body = self._request_data_parts_body(data) + body = HTTPHelpers.request_data_parts_body(data) response = session.post( url=url, headers=headers, data=body, files=http_files ) - return self._response_parsing(response) + return HTTPHelpers.response_parsing(response) # Form Put if inject_contract == HTTP_FORM_PUT_CONTRACT: - body = self._request_data_parts_body(data) + body = HTTPHelpers.request_data_parts_body(data) response = session.put( url=url, headers=headers, data=body, files=http_files ) - return self._response_parsing(response) + return HTTPHelpers.response_parsing(response) # Nothing supported return { "code": 400, diff --git a/http-query/src/requirements.txt b/http-query/src/requirements.txt index 916a962c..b9ce06c1 100644 --- a/http-query/src/requirements.txt +++ b/http-query/src/requirements.txt @@ -1 +1 @@ -pyoaev==2.0.4 \ No newline at end of file +pyoaev==2.0.5 \ No newline at end of file diff --git a/http-query/test/__init__.py b/http-query/test/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/http-query/test/helpers/__init__.py b/http-query/test/helpers/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/http-query/test/helpers/test_helpers.py b/http-query/test/helpers/test_helpers.py new file mode 100644 index 00000000..d243532c --- /dev/null +++ b/http-query/test/helpers/test_helpers.py @@ -0,0 +1,33 @@ +from unittest import TestCase + +from src.helpers.helpers import HTTPHelpers + + +class HTTPHelpersTest(TestCase): + def test_parse_headers_with_string(self): + input_str = ( + "Content-Type=application/x-www-form-urlencoded,Accept=application/json" + ) + expected = [ + {"key": "Content-Type", "value": "application/x-www-form-urlencoded"}, + {"key": "Accept", "value": "application/json"}, + ] + result = HTTPHelpers.parse_headers(input_str) + self.assertEqual(result, expected) + + def test_parse_parts_with_string(self): + input_str = "msg=test&user=alice" + expected = [ + {"key": "msg", "value": "test"}, + {"key": "user", "value": "alice"}, + ] + result = HTTPHelpers.parse_parts(input_str) + self.assertEqual(result, expected) + + def test_parse_headers_empty_string(self): + result = HTTPHelpers.parse_headers("") + self.assertEqual(result, []) + + def test_parse_parts_empty_string(self): + result = HTTPHelpers.parse_parts("") + self.assertEqual(result, []) diff --git a/injector_common/injector_common/targets.py b/injector_common/injector_common/targets.py index 303c0b48..c328ef24 100644 --- a/injector_common/injector_common/targets.py +++ b/injector_common/injector_common/targets.py @@ -64,7 +64,11 @@ def extract_targets( ) elif selector_key == "manual": - targets = [t.strip() for t in content[TARGETS_KEY].split(",") if t.strip()] + targets = list( + dict.fromkeys( + [t.strip() for t in content[TARGETS_KEY].split(",") if t.strip()] + ) + ) else: raise ValueError("No targets provided for this injection") diff --git a/injector_common/pyproject.toml b/injector_common/pyproject.toml index 61d7021f..b106f4e8 100644 --- a/injector_common/pyproject.toml +++ b/injector_common/pyproject.toml @@ -3,7 +3,7 @@ name="injector_common" version="1.0.0" dependencies = [ - "pyoaev==2.0.4", + "pyoaev==2.0.5", ] [project.optional-dependencies] dev = [ diff --git a/nmap/docker-compose.yml b/nmap/docker-compose.yml index 2a74b525..4a87de90 100644 --- a/nmap/docker-compose.yml +++ b/nmap/docker-compose.yml @@ -1,7 +1,7 @@ version: '3' services: injector-nmap: - image: openaev/injector-nmap:2.0.4 + image: openaev/injector-nmap:2.0.5 environment: - OPENAEV_URL=http://localhost - OPENAEV_TOKEN=ChangeMe diff --git a/nmap/requirements.txt b/nmap/requirements.txt index 8b651933..68a745cd 100644 --- a/nmap/requirements.txt +++ b/nmap/requirements.txt @@ -1,2 +1,2 @@ -pyoaev==2.0.4 +pyoaev==2.0.5 ../injector_common diff --git a/nmap/src/helpers/nmap_output_parser.py b/nmap/src/helpers/nmap_output_parser.py index b755a3a9..57936cc3 100644 --- a/nmap/src/helpers/nmap_output_parser.py +++ b/nmap/src/helpers/nmap_output_parser.py @@ -1,9 +1,15 @@ from typing import Dict +from injector_common.targets import TargetExtractionResult + class NmapOutputParser: - def parse(data: Dict, result: str, asset_list: []) -> Dict: + @staticmethod + def parse(data: Dict, result: str, target_results: TargetExtractionResult) -> Dict: """Parse nmap results and extract open ports.""" + asset_list = list(target_results.ip_to_asset_id_map.values()) + targets = target_results.targets or [] + run = result["nmaprun"] if not isinstance(run["host"], list): run["host"] = [run["host"]] @@ -27,7 +33,10 @@ def parse(data: Dict, result: str, asset_list: []) -> Dict: port_result["host"] = host["address"]["@addr"] else: port_result["asset_id"] = None - port_result["host"] = asset_list[idx] + if idx < len(targets): + port_result["host"] = targets[idx] + else: + port_result["host"] = None ports_scans_results.append(port_result) return { diff --git a/nmap/src/openaev_nmap.py b/nmap/src/openaev_nmap.py index 386f393f..1a56b357 100644 --- a/nmap/src/openaev_nmap.py +++ b/nmap/src/openaev_nmap.py @@ -53,16 +53,15 @@ def nmap_execution(self, start: float, data: Dict) -> Dict: target_results = Targets.extract_targets( selector_key, selector_property, data, self.helper ) - asset_list = list(target_results.ip_to_asset_id_map.values()) # Deduplicate targets - unique_targets = list(dict.fromkeys(target_results.targets)) + targets = target_results.targets # Handle empty targets as an error - if not unique_targets: + if not targets: message = f"No target identified for the property {TargetProperty[selector_property.upper()].value}" raise ValueError(message) # Build Arguments to execute - nmap_args = NmapCommandBuilder.build_args(contract_id, unique_targets) + nmap_args = NmapCommandBuilder.build_args(contract_id, targets) self.helper.injector_logger.info( "Executing nmap with command: " + " ".join(nmap_args) @@ -90,7 +89,7 @@ def nmap_execution(self, start: float, data: Dict) -> Dict: jc = NmapProcess.js_execute(["jc", "--xml", "-p"], nmap_result) result = json.loads(jc.stdout.decode("utf-8").strip()) - return NmapOutputParser.parse(data, result, asset_list) + return NmapOutputParser.parse(data, result, target_results) def process_message(self, data: Dict) -> None: start = time.time() diff --git a/nmap/test/__init__.py b/nmap/test/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/nmap/test/helpers/__init__.py b/nmap/test/helpers/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/nmap/test/helpers/test_nmap_output_parser.py b/nmap/test/helpers/test_nmap_output_parser.py new file mode 100644 index 00000000..e81264fb --- /dev/null +++ b/nmap/test/helpers/test_nmap_output_parser.py @@ -0,0 +1,98 @@ +from unittest import TestCase + +from src.helpers.nmap_output_parser import NmapOutputParser + +from injector_common.targets import TargetExtractionResult + + +class NmapOutputParserTest(TestCase): + def setUp(self): + self.result_single_host = { + "nmaprun": { + "host": { + "address": {"@addr": "172.16.5.10"}, + "ports": { + "port": [ + { + "@portid": "21", + "state": {"@state": "open"}, + "service": {"@name": "ftp"}, + } + ] + }, + } + } + } + + # ------------------------------- + # Tests + # ------------------------------- + + def test_parse_target_assets(self): + """Ensure target_selector='assets' uses asset_list and sets asset_id.""" + data = {"injection": {"inject_content": {"target_selector": "assets"}}} + + result = NmapOutputParser.parse( + data, + self.result_single_host, + TargetExtractionResult( + ip_to_asset_id_map={"172.16.5.10": "asset-123"}, targets=[] + ), + ) + + scan = result["outputs"]["scan_results"][0] + + self.assertEqual(scan["asset_id"], "asset-123") + self.assertEqual(scan["host"], "172.16.5.10") + self.assertEqual(scan["port"], 21) + self.assertEqual(scan["service"], "ftp") + + def test_parse_target_asset_groups(self): + """Ensure target_selector='asset-groups' also uses asset_list.""" + data = {"injection": {"inject_content": {"target_selector": "asset-groups"}}} + + result = NmapOutputParser.parse( + data, + self.result_single_host, + TargetExtractionResult( + ip_to_asset_id_map={"172.16.5.10": "group-asset-555"}, + targets=["172.16.5.10"], + ), + ) + + scan = result["outputs"]["scan_results"][0] + + self.assertEqual(scan["asset_id"], "group-asset-555") + self.assertEqual(scan["host"], "172.16.5.10") + self.assertEqual(scan["port"], 21) + self.assertEqual(scan["service"], "ftp") + + def test_parse_target_manual(self): + """Ensure target_selector='manual' sets asset_id=None.""" + data = {"injection": {"inject_content": {"target_selector": "manual"}}} + + result = NmapOutputParser.parse( + data, + self.result_single_host, + TargetExtractionResult(ip_to_asset_id_map={}, targets=["172.16.5.10"]), + ) + + scan = result["outputs"]["scan_results"][0] + + self.assertIsNone(scan["asset_id"]) + self.assertEqual(scan["host"], "172.16.5.10") + + def test_parse_target_manual_None_values(self): + """Ensure target_selector='manual' sets asset_id=None.""" + data = {"injection": {"inject_content": {"target_selector": "manual"}}} + + result = NmapOutputParser.parse( + data, + self.result_single_host, + TargetExtractionResult(ip_to_asset_id_map={}, targets=[]), + ) + + scan = result["outputs"]["scan_results"][0] + + self.assertIsNone(scan["asset_id"]) + self.assertIsNone(scan["host"]) diff --git a/nuclei/docker-compose.yml b/nuclei/docker-compose.yml index ce4d6967..a8124747 100644 --- a/nuclei/docker-compose.yml +++ b/nuclei/docker-compose.yml @@ -1,7 +1,7 @@ version: '3.8' services: injector-nuclei: - image: openaev/injector-nuclei:2.0.4 + image: openaev/injector-nuclei:2.0.5 environment: - OPENAEV_URL=http://localhost - OPENAEV_TOKEN=ChangeMe diff --git a/nuclei/nuclei/openaev_nuclei.py b/nuclei/nuclei/openaev_nuclei.py index 4cd5ac30..5a13c935 100644 --- a/nuclei/nuclei/openaev_nuclei.py +++ b/nuclei/nuclei/openaev_nuclei.py @@ -79,16 +79,14 @@ def nuclei_execution(self, start: float, data: Dict) -> Dict: selector_key, selector_property, data, self.helper ) # Deduplicate targets - unique_targets = list(dict.fromkeys(target_results.targets)) + targets = target_results.targets # Handle empty targets as an error - if not unique_targets: + if not targets: message = f"No target identified for the property {TargetProperty[selector_property.upper()].value}" raise ValueError(message) # Build Arguments to execute - nuclei_args = self.command_builder.build_args( - contract_id, content, unique_targets - ) - input_data = "\n".join(unique_targets).encode("utf-8") + nuclei_args = self.command_builder.build_args(contract_id, content, targets) + input_data = "\n".join(targets).encode("utf-8") self.helper.injector_logger.info( "Executing nuclei with: " + " ".join(nuclei_args) diff --git a/nuclei/requirements.txt b/nuclei/requirements.txt index 956e499a..ccc06dfb 100644 --- a/nuclei/requirements.txt +++ b/nuclei/requirements.txt @@ -1,2 +1,2 @@ -pyoaev==2.0.4 +pyoaev==2.0.5 ../injector_common \ No newline at end of file