diff --git a/.github/workflows/check_docker.yml b/.github/workflows/check_docker.yml new file mode 100644 index 0000000..0fca2b3 --- /dev/null +++ b/.github/workflows/check_docker.yml @@ -0,0 +1,79 @@ +# TODO: Trivy scan for Dockerfile will be enabled in the upcoming issue: https://github.com/AbsaOSS/EventGate/issues/74 +#name: Docker Check +# +#on: +# pull_request: +# types: [ opened, synchronize, reopened ] +# push: +# branches: [ master ] +# workflow_dispatch: +# +#concurrency: +# group: static-docker-check-${{ github.ref }} +# cancel-in-progress: true +# +#permissions: +# contents: read +# security-events: write +# +#jobs: +# detect: +# name: Docker Changes Detection +# runs-on: ubuntu-latest +# outputs: +# docker_changed: ${{ steps.changes.outputs.docker_changed }} +# steps: +# - name: Checkout repository +# uses: actions/checkout@v5 +# with: +# persist-credentials: false +# fetch-depth: 0 +# +# - name: Check if docker file changed +# id: changes +# shell: bash +# run: | +# if [[ "${{ github.event_name }}" == "pull_request" ]]; then +# RANGE="${{ github.event.pull_request.base.sha }}...${{ github.sha }}" +# else +# RANGE="${{ github.sha }}~1...${{ github.sha }}" +# fi +# if git diff --name-only "$RANGE" | grep -qE '^Dockerfile$'; then +# echo "docker_changed=true" >> "$GITHUB_OUTPUT" +# else +# echo "docker_changed=false" >> "$GITHUB_OUTPUT" +# fi +# +# trivy-docker: +# name: Trivy Security Scan +# needs: detect +# if: needs.detect.outputs.docker_changed == 'true' +# runs-on: ubuntu-latest +# steps: +# - name: Checkout repository +# uses: actions/checkout@v5 +# with: +# persist-credentials: false +# fetch-depth: 0 +# +# - name: Setup Trivy +# uses: aquasecurity/setup-trivy@v0.2.4 +# +# - name: Trivy security scan +# run: | +# trivy config Dockerfile \ +# --format sarif \ +# --output $GITHUB_WORKSPACE/trivy_dockerfile.sarif +# +# - name: Upload Dockerfile SARIF +# uses: github/codeql-action/upload-sarif@v4 +# with: +# sarif_file: ${{ github.workspace }}/trivy_dockerfile.sarif +# +# noop: +# name: No Operation +# needs: detect +# if: needs.detect.outputs.docker_changed != 'true' +# runs-on: ubuntu-latest +# steps: +# - run: echo "No changes in the Dockerfile — passing." diff --git a/.github/workflows/check_pr_release_notes.yml b/.github/workflows/check_pr_release_notes.yml index 89c2835..6b90c43 100644 --- a/.github/workflows/check_pr_release_notes.yml +++ b/.github/workflows/check_pr_release_notes.yml @@ -1,4 +1,4 @@ -name: Check PR Release Notes in Description +name: Check PR Release Notes on: pull_request: @@ -7,6 +7,7 @@ on: jobs: check-release-notes: + name: Check PR Release Notes runs-on: ubuntu-latest steps: diff --git a/.github/workflows/check_python.yml b/.github/workflows/check_python.yml new file mode 100644 index 0000000..e07f10d --- /dev/null +++ b/.github/workflows/check_python.yml @@ -0,0 +1,161 @@ +name: Python Check + +on: + pull_request: + types: [ opened, synchronize, reopened ] + push: + branches: [ master ] + workflow_dispatch: + +concurrency: + group: static-python-check-${{ github.ref }} + cancel-in-progress: true + +permissions: + contents: read + security-events: write + +jobs: + detect: + name: Python Changes Detection + runs-on: ubuntu-latest + outputs: + python_changed: ${{ steps.changes.outputs.python_changed }} + steps: + - name: Checkout repository + uses: actions/checkout@v5 + with: + persist-credentials: false + fetch-depth: 0 + + - name: Check if Python files changed + id: changes + shell: bash + run: | + if [[ "${{ github.event_name }}" == "pull_request" ]]; then + RANGE="${{ github.event.pull_request.base.sha }}...${{ github.sha }}" + else + RANGE="${{ github.sha }}~1...${{ github.sha }}" + fi + if git diff --name-only "$RANGE" -- '*.py' | grep -q .; then + echo "python_changed=true" >> "$GITHUB_OUTPUT" + else + echo "python_changed=false" >> "$GITHUB_OUTPUT" + fi + + pylint-analysis: + name: Pylint Static Code Analysis + needs: detect + if: needs.detect.outputs.python_changed == 'true' + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v5 + with: + persist-credentials: false + fetch-depth: 0 + + - name: Set up Python + uses: actions/setup-python@v6 + with: + python-version: '3.13' + cache: 'pip' + + - name: Install dependencies + run: pip install -r requirements.txt + + - name: Analyze code with Pylint + id: analyze-code + run: | + pylint_score=$(pylint $(git ls-files '*.py')| grep 'rated at' | awk '{print $7}' | cut -d'/' -f1) + echo "PYLINT_SCORE=$pylint_score" >> $GITHUB_ENV + + - name: Check Pylint score + run: | + if (( $(echo "$PYLINT_SCORE < 9.5" | bc -l) )); then + echo "Failure: Pylint score is below 9.5 (project score: $PYLINT_SCORE)." + exit 1 + else + echo "Success: Pylint score is above 9.5 (project score: $PYLINT_SCORE)." + fi + + black-check: + name: Black Format Check + needs: detect + if: needs.detect.outputs.python_changed == 'true' + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v5 + with: + persist-credentials: false + fetch-depth: 0 + + - name: Set up Python + uses: actions/setup-python@v6 + with: + python-version: '3.13' + cache: 'pip' + + - name: Install dependencies + run: pip install -r requirements.txt + + - name: Check code format with Black + id: check-format + run: black --check $(git ls-files '*.py') + + pytest-test: + name: Pytest Unit Tests with Coverage + needs: detect + if: needs.detect.outputs.python_changed == 'true' + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v5 + with: + persist-credentials: false + fetch-depth: 0 + + - uses: actions/setup-python@v6 + with: + python-version: '3.13' + cache: 'pip' + + - name: Install Python dependencies + run: pip install -r requirements.txt + + - name: Check code coverage with Pytest + run: pytest --cov=. -v tests/ --cov-fail-under=80 + + mypy-check: + name: Mypy Type Check + needs: detect + if: needs.detect.outputs.python_changed == 'true' + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v5 + with: + persist-credentials: false + fetch-depth: 0 + + - name: Set up Python + uses: actions/setup-python@v6 + with: + python-version: '3.13' + cache: 'pip' + + - name: Install dependencies + run: pip install -r requirements.txt + + - name: Check types with Mypy + id: check-types + run: mypy . + + noop: + name: No Operation + needs: detect + if: needs.detect.outputs.python_changed != 'true' + runs-on: ubuntu-latest + steps: + - run: echo "No changes in the *.py files — passing." diff --git a/.github/workflows/check_terraform.yml b/.github/workflows/check_terraform.yml index 6d05444..ece21ab 100644 --- a/.github/workflows/check_terraform.yml +++ b/.github/workflows/check_terraform.yml @@ -1,18 +1,14 @@ -name: Static Terraform Check +name: Terraform Check on: pull_request: types: [ opened, synchronize, reopened ] - paths: - - terraform/** push: branches: [ master ] - paths: - - terraform/** workflow_dispatch: concurrency: - group: terraform-static-check-${{ github.ref }} + group: static-terraform-check-${{ github.ref }} cancel-in-progress: true permissions: @@ -20,45 +16,43 @@ permissions: security-events: write jobs: - trivy: + detect: + name: Terraform Changes Detection runs-on: ubuntu-latest + outputs: + terraform_changed: ${{ steps.changes.outputs.terraform_changed }} steps: - - uses: actions/checkout@v4 + - name: Checkout repository + uses: actions/checkout@v5 with: + persist-credentials: false fetch-depth: 0 - - name: Setup Trivy - uses: aquasecurity/setup-trivy@v0.2.4 - - - name: Trivy config scan - id: scan - working-directory: terraform + - name: Check if terraform/ changed + id: changes + shell: bash run: | - set +e - trivy config . \ - --format sarif \ - --output $GITHUB_WORKSPACE/terraform_trivy.sarif \ - --severity HIGH,CRITICAL \ - --exit-code 1 - code=$? - echo "exit_code=$code" >> "$GITHUB_OUTPUT" - exit 0 + if [[ "${{ github.event_name }}" == "pull_request" ]]; then + RANGE="${{ github.event.pull_request.base.sha }}...${{ github.sha }}" + else + RANGE="${{ github.sha }}~1...${{ github.sha }}" + fi + if git diff --name-only "$RANGE" | grep -qE '^terraform/'; then + echo "terraform_changed=true" >> "$GITHUB_OUTPUT" + else + echo "terraform_changed=false" >> "$GITHUB_OUTPUT" + fi - - name: Upload Trivy SARIF file - uses: github/codeql-action/upload-sarif@v3 - with: - sarif_file: ${{ github.workspace }}/terraform_trivy.sarif - - - name: Enforce failure on HIGH/CRITICAL - if: steps.scan.outputs.exit_code != '0' - run: | - echo "Trivy detected HIGH/CRITICAL findings. View details in PR code scanning annotations or in Security tab > Code scanning alerts (filter for pr:pr_number)." - exit 1 tflint: + name: TFLint Static Code Analysis + needs: detect + if: needs.detect.outputs.terraform_changed == 'true' runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - name: Checkout repository + uses: actions/checkout@v5 with: + persist-credentials: false fetch-depth: 0 - name: Setup TFLint @@ -71,22 +65,45 @@ jobs: run: tflint --init - name: Run TFLint - id: lint working-directory: terraform - run: | - set +e - tflint --minimum-failure-severity=error -f sarif > "$GITHUB_WORKSPACE/terraform_tflint.sarif" - code=$? - echo "exit_code=$code" >> "$GITHUB_OUTPUT" - exit 0 + run: tflint --minimum-failure-severity=error -f sarif > "$GITHUB_WORKSPACE/tflint_terraform.sarif" - name: Upload TFLint SARIF file - uses: github/codeql-action/upload-sarif@v3 + uses: github/codeql-action/upload-sarif@v4 with: - sarif_file: ${{ github.workspace }}/terraform_tflint.sarif + sarif_file: ${{ github.workspace }}/tflint_terraform.sarif - - name: Enforce failure on TFLint findings - if: steps.lint.outputs.exit_code != '0' - run: | - echo "TFLint reported error issues. View details in PR code scanning annotations or in Security tab > Code scanning alerts (filter for pr:pr_number)." - exit 1 +# TODO: Trivy scan for changed Terraform files will be enabled in the upcoming issue: https://github.com/AbsaOSS/EventGate/issues/74 +# trivy-terraform: +# name: Trivy Security Scan +# needs: detect +# if: needs.detect.outputs.terraform_changed == 'true' +# runs-on: ubuntu-latest +# steps: +# - name: Checkout repository +# uses: actions/checkout@v5 +# with: +# persist-credentials: false +# fetch-depth: 0 +# +# - name: Setup Trivy +# uses: aquasecurity/setup-trivy@v0.2.4 +# +# - name: Trivy security scan +# run: | +# trivy fs terraform/ \ +# --format sarif \ +# --output $GITHUB_WORKSPACE/trivy_terraform.sarif +# +# - name: Upload Terraform SARIF +# uses: github/codeql-action/upload-sarif@v4 +# with: +# sarif_file: ${{ github.workspace }}/trivy_terraform.sarif + + noop: + name: No Operation + needs: detect + if: needs.detect.outputs.terraform_changed != 'true' + runs-on: ubuntu-latest + steps: + - run: echo "No changes under terraform/ — passing." diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml deleted file mode 100644 index 5cfb477..0000000 --- a/.github/workflows/test.yml +++ /dev/null @@ -1,114 +0,0 @@ -name: Build and Test -on: - pull_request: - branches: - - '**' - types: [ opened, synchronize, reopened ] - -jobs: - static-code-analysis: - runs-on: ubuntu-latest - name: Pylint Static Code Analysis - steps: - - name: Checkout repository - uses: actions/checkout@v4 - with: - persist-credentials: false - - - name: Set up Python - uses: actions/setup-python@v5 - with: - python-version: '3.13' - cache: 'pip' - - - name: Install dependencies - run: | - pip install -r requirements.txt - - - name: Analyze code with Pylint - id: analyze-code - run: | - pylint_score=$(pylint $(git ls-files '*.py')| grep 'rated at' | awk '{print $7}' | cut -d'/' -f1) - echo "PYLINT_SCORE=$pylint_score" >> $GITHUB_ENV - - - name: Check Pylint score - run: | - if (( $(echo "$PYLINT_SCORE < 9.5" | bc -l) )); then - echo "Failure: Pylint score is below 9.5 (project score: $PYLINT_SCORE)." - exit 1 - else - echo "Success: Pylint score is above 9.5 (project score: $PYLINT_SCORE)." - fi - - code-format-check: - runs-on: ubuntu-latest - name: Black Format Check - steps: - - name: Checkout repository - uses: actions/checkout@v4.1.5 - with: - persist-credentials: false - - - name: Set up Python - uses: actions/setup-python@v5.1.0 - with: - python-version: '3.13' - cache: 'pip' - - - name: Install dependencies - run: | - pip install -r requirements.txt - - - name: Check code format with Black - id: check-format - run: | - black --check $(git ls-files '*.py') - - unit-test: - name: Unit Tests - runs-on: ubuntu-latest - - defaults: - run: - shell: bash - - steps: - - uses: actions/checkout@v4 - with: - fetch-depth: 0 - persist-credentials: false - - - uses: actions/setup-python@v5 - with: - python-version: '3.13' - cache: 'pip' - - - name: Install Python dependencies - run: | - pip install -r requirements.txt - - - name: Check code coverage with Pytest - run: pytest --cov=. -v tests/ --cov-fail-under=80 - - mypy-check: - runs-on: ubuntu-latest - name: Mypy Type Check - steps: - - name: Checkout repository - uses: actions/checkout@v4.1.5 - with: - persist-credentials: false - - - name: Set up Python - uses: actions/setup-python@v5.1.0 - with: - python-version: '3.13' - cache: 'pip' - - - name: Install dependencies - run: | - pip install -r requirements.txt - - name: Check types with Mypy - id: check-types - run: | - mypy . diff --git a/.github/workflows/trivy_repository_scan.yml b/.github/workflows/trivy_repository_scan.yml new file mode 100644 index 0000000..dfe3a26 --- /dev/null +++ b/.github/workflows/trivy_repository_scan.yml @@ -0,0 +1,143 @@ +name: Trivy Full Repository Scan + +on: + workflow_dispatch: + pull_request: + types: [ opened, synchronize ] + +permissions: + contents: read + issues: write + pull-requests: write + security-events: write + +jobs: + trivy: + name: Trivy Full Repository Scan + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v5 + with: + persist-credentials: false + fetch-depth: 0 + + - name: Setup Trivy + uses: aquasecurity/setup-trivy@v0.2.4 + + - name: Run Trivy filesystem scan + run: | + trivy fs . \ + --format sarif \ + --scanners vuln,secret,misconfig,license \ + --output trivy_repository_report.sarif + + - name: Upload SARIF to GitHub Security Hub + uses: github/codeql-action/upload-sarif@v4 + with: + sarif_file: trivy_repository_report.sarif + + - name: Create scan summary table + id: scan_summary_table + run: | + python <<'PY' + import os + import json + import sys + from collections import defaultdict, Counter + + SARIF_PATH = "trivy_repository_report.sarif" + SEVERITIES = ["CRITICAL", "HIGH", "MEDIUM", "LOW"] + CATEGORIES = ["vulnerability", "secret", "misconfiguration", "license"] + + try: + # Parse results from SARIF + with open(SARIF_PATH, "r", encoding="utf-8") as f: + sarif = json.load(f) + + # Validate SARIF structure + if "runs" not in sarif or not sarif["runs"]: + raise ValueError("SARIF file contains no runs") + + run = sarif["runs"][0] + if "tool" not in run or "driver" not in run["tool"]: + raise ValueError("SARIF structure missing expected tool/driver keys") + + rules = run["tool"]["driver"].get("rules", []) + results = run.get("results", []) + category_severity_counts = defaultdict(Counter) + + except (IOError, json.JSONDecodeError, KeyError, ValueError) as e: + print(f"Error processing SARIF file: {e}", file=sys.stderr) + sys.exit(1) + + # Count results by category and severity + for result in results: + try: + rule_idx = result.get("ruleIndex") + if rule_idx is None or rule_idx >= len(rules): + continue + rule = rules[rule_idx] + tags = rule.get("properties", {}).get("tags", []) + # Find category and severity + category = next((c for c in CATEGORIES if c in tags), None) + severity = next((s for s in SEVERITIES if s in tags), None) + if category and severity: + category_severity_counts[category][severity] += 1 + except (KeyError, IndexError, TypeError) as e: + print(f"Warning: Error processing result: {e}", file=sys.stderr) + continue + + # Build Markdown summary table + headers = ["TRIVY"] + SEVERITIES + ["TOTAL"] + summary_table = "| " + " | ".join(headers) + " |\n" + summary_table += "|---|---|---|---|---|---|\n" + + # Rows with counts for each category + total_severity = Counter() + total_all = 0 + for category in CATEGORIES: + row = [category] + category_total = 0 + for severity in SEVERITIES: + count = category_severity_counts[category][severity] + row.append(str(count)) + total_severity[severity] += count + category_total += count + row.append(f"**{category_total}**") + total_all += category_total + summary_table += "| " + " | ".join(row) + " |\n" + + total_row = ["**➡️ Total**"] + [f"**{total_severity[sev]}**" for sev in SEVERITIES] + [f"**{total_all}**"] + summary_table += "| " + " | ".join(total_row) + " |" + + # Set summary table output + try: + if "GITHUB_OUTPUT" in os.environ: + with open(os.environ["GITHUB_OUTPUT"], "a", encoding="utf-8") as f: + f.write("table< For installation instructions, please refer to the [following link.](https://trivy.dev/latest/getting-started/installation/) ### Run Trivy -For running Trivy tool locally run the following command from the root file: +For running Trivy tool locally you can execute one of following commands from the root file: ```shell -trivy config terraform/ # Default table output (all severities) -trivy config --severity HIGH,CRITICAL terraform/ # Show only HIGH and CRITICAL severities +trivy fs . --scanners vuln,secret,misconfig,license > trivy_scan.txt # Scan the whole project with all available scans (all severities for the whole project) +trivy fs --severity MEDIUM,HIGH,CRITICAL terraform/ > trivy_scan.txt # Show only selected severities for terraform files +trivy config Dockerfile > trivy_scan.txt # Scan only Dockerfile ``` +You can see the scan results in the `trivy_scan.txt` file located in the root directory. + ## Running Unit Test Unit tests are written using pytest. To run the tests, use the following command: diff --git a/README.md b/README.md index 42286c1..e0cb1f4 100644 --- a/README.md +++ b/README.md @@ -124,17 +124,17 @@ Use when Kafka access needs Kerberos / SASL_SSL or custom `librdkafka` build. ## Local Development & Testing -| Purpose | Relative link | -|------------------------------------|-----------------------------------------------------------------------| -| Get started | [Get Started](./DEVELOPER.md#get-started) | -| Python environment setup | [Set Up Python Environment](./DEVELOPER.md#set-up-python-environment) | -| Static code analysis (Pylint) | [Run Pylint Tool Locally](./DEVELOPER.md#run-pylint-tool-locally) | -| Formatting (Black) | [Run Black Tool Locally](./DEVELOPER.md#run-black-tool-locally) | -| Type checking (mypy) | [Run mypy Tool Locally](./DEVELOPER.md#run-mypy-tool-locally) | -| Terraform Linter (TFLint) | [Run TFLint Tool Locally](./DEVELOPER.md#run-tflint-tool-locally) | -| Terraform Security Scanner (Trivy) | [Run Trivy Tool Locally](./DEVELOPER.md#run-trivy-tool-locally) | -| Unit tests | [Running Unit Test](./DEVELOPER.md#running-unit-test) | -| Code coverage | [Code Coverage](./DEVELOPER.md#code-coverage) | +| Purpose | Relative link | +|-------------------------------|-----------------------------------------------------------------------| +| Get started | [Get Started](./DEVELOPER.md#get-started) | +| Python environment setup | [Set Up Python Environment](./DEVELOPER.md#set-up-python-environment) | +| Static code analysis (Pylint) | [Run Pylint Tool Locally](./DEVELOPER.md#run-pylint-tool-locally) | +| Formatting (Black) | [Run Black Tool Locally](./DEVELOPER.md#run-black-tool-locally) | +| Type checking (mypy) | [Run mypy Tool Locally](./DEVELOPER.md#run-mypy-tool-locally) | +| Terraform Linter (TFLint) | [Run TFLint Tool Locally](./DEVELOPER.md#run-tflint-tool-locally) | +| Security Scanner (Trivy) | [Run Trivy Tool Locally](./DEVELOPER.md#run-trivy-tool-locally) | +| Unit tests | [Running Unit Test](./DEVELOPER.md#running-unit-test) | +| Code coverage | [Code Coverage](./DEVELOPER.md#code-coverage) | ## Security & Authorization - JWT tokens must be RS256 signed; the public key is fetched at cold start from `token_public_key_url` (DER base64 inside JSON `{ "key": "..." }`). diff --git a/terraform/lambda.tf b/terraform/lambda.tf index 5344439..984aec1 100644 --- a/terraform/lambda.tf +++ b/terraform/lambda.tf @@ -24,7 +24,6 @@ resource "aws_lambda_function" "event_gate_lambda" { timeout = 60 runtime = "python3.13" package_type = var.lambda_package_type - s3_bucket = var.lambda_package_type == "Zip" ? var.lambda_src_s3_bucket : null s3_key = var.lambda_package_type == "Zip" ? var.lambda_src_s3_key : null handler = var.lambda_package_type == "Zip" ? "event_gate_lambda.lambda_handler" : null diff --git a/trivy-secret.yaml b/trivy-secret.yaml new file mode 100644 index 0000000..7613558 --- /dev/null +++ b/trivy-secret.yaml @@ -0,0 +1,27 @@ +rules: + - id: password-plaintext-assignment + category: exposed_credential + title: Plaintext password literal + severity: HIGH + keywords: + - password + regex: (?i)(["']?\w*password\w*["']?\s*[:=]\s*.+) + + - id: secret-plaintext-assignment + category: exposed_credential + title: Plaintext secret literal + severity: HIGH + keywords: + - secret + regex: (?i)(["']?\w*secret\w*["']?\s*[:=]\s*.+) + +enable-builtin-rules: + - aws-access-key-id + - aws-account-id + - aws-secret-access-key + - github-pat + - github-oauth + - github-app-token + - github-refresh-token + - github-fine-grained-pat + - gitlab-pat