diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 3bd9dbe3..0e68d15f 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -21,6 +21,10 @@ on: - '**' types: [ opened, synchronize, reopened ] +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + jobs: static-code-analysis: runs-on: ubuntu-latest @@ -34,7 +38,7 @@ jobs: - name: Set up Python uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 with: - python-version: '3.14' + python-version: '3.13' cache: 'pip' - name: Install dependencies @@ -71,7 +75,7 @@ jobs: - name: Set up Python uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 with: - python-version: '3.14' + python-version: '3.13' cache: 'pip' - name: Install dependencies @@ -99,7 +103,7 @@ jobs: - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 with: - python-version: '3.14' + python-version: '3.13' cache: 'pip' - name: Install Python dependencies @@ -124,7 +128,7 @@ jobs: - name: Set up Python uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 with: - python-version: '3.14' + python-version: '3.13' cache: 'pip' - name: Install dependencies @@ -134,3 +138,131 @@ jobs: id: check-types run: | mypy . + + snapshot-tests: + name: Snapshot Tests (Mocked) + runs-on: ubuntu-latest + + defaults: + run: + shell: bash + + steps: + - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v4 + with: + fetch-depth: 0 + persist-credentials: false + + - uses: actions/setup-python@83679a892e2d95755f2dac6acb0bfd1e9ac5d548 # v5 + with: + python-version: '3.13' + cache: 'pip' + + - name: Install Python dependencies + run: | + pip install -r requirements.txt + + - name: Set PYTHONPATH environment variable + run: echo "PYTHONPATH=${GITHUB_WORKSPACE}" >> $GITHUB_ENV + + - name: Run all integration tests + run: pytest -v tests/integration/ + + integration-test-real-api: + name: Integration Test (Real GitHub API) + runs-on: ubuntu-latest + # Only run on PRs from same repo (not forks) + if: github.event.pull_request.head.repo.full_name == github.repository + + defaults: + run: + shell: bash + + steps: + - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v4 + with: + fetch-depth: 0 + persist-credentials: false + + - uses: actions/setup-python@83679a892e2d95755f2dac6acb0bfd1e9ac5d548 # v5 + with: + python-version: '3.13' + cache: 'pip' + + - name: Install Python dependencies + run: | + pip install -r requirements.txt + + - name: Set PYTHONPATH environment variable + run: echo "PYTHONPATH=${GITHUB_WORKSPACE}" >> $GITHUB_ENV + + - name: Run action against real repository and validate output + env: + INPUT_TAG_NAME: 'v0.2.0' + INPUT_GITHUB_REPOSITORY: 'AbsaOSS/generate-release-notes' + INPUT_GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + INPUT_CHAPTERS: | + [ + { title: Breaking Changes 💥, label: breaking-change }, + { title: New Features 🎉, label: enhancement }, + { title: New Features 🎉, label: feature }, + { title: Bugfixes 🛠, label: bug } + ] + INPUT_WARNINGS: 'true' + INPUT_PRINT_EMPTY_CHAPTERS: 'false' + INPUT_VERBOSE: 'true' + INPUT_HIERARCHY: 'false' + INPUT_DUPLICITY_SCOPE: 'both' + INPUT_PUBLISHED_AT: 'false' + INPUT_SKIP_RELEASE_NOTES_LABELS: 'skip-release-notes' + run: | + # Run the action with verbose/debug logging + python main.py > output.txt 2>&1 + exit_code=$? + + # Display output for debugging + echo "=== Action Output ===" + cat output.txt + echo "=== End of Output ===" + + problems=() + + # Check exit code + if [ $exit_code -ne 0 ]; then + problems+=("action exit code was $exit_code") + fi + + # Check for markdown chapter headers + if ! grep -qE '^### (Breaking Changes 💥|New Features 🎉|Bugfixes 🛠)' output.txt; then + problems+=("missing expected chapter headers") + fi + + # Check for issue/PR references (e.g., #123) + if ! grep -qE '#[0-9]+' output.txt; then + problems+=("missing issue/PR references") + fi + + # Check for developer mentions (e.g., @username) + if ! grep -qE '@[a-zA-Z0-9_-]+' output.txt; then + problems+=("missing developer mentions") + fi + + # Check for completion message + if ! grep -q "completed successfully" output.txt; then + problems+=("missing completion success message") + fi + + # Check for DEBUG log level (verbose mode) + if ! grep -q "DEBUG" output.txt; then + problems+=("missing DEBUG logs (verbose mode may not be working)") + fi + + # Check for generated release notes block + if ! grep -q "Generated release notes:" output.txt; then + problems+=("missing 'Generated release notes:' marker") + fi + + if [ ${#problems[@]} -ne 0 ]; then + echo "❌ Integration test validation failed: ${problems[*]}" + exit 1 + fi diff --git a/DEVELOPER.md b/DEVELOPER.md index acc73920..2a16b162 100644 --- a/DEVELOPER.md +++ b/DEVELOPER.md @@ -4,8 +4,11 @@ - [Run Static Code Analysis](#running-static-code-analysis) - [Run Black Tool Locally](#run-black-tool-locally) - [Run mypy Tool Locally](#run-mypy-tool-locally) -- [Run Unit Test](#running-unit-test) +- [Running Unit Test](#running-unit-test) +- [Running Integration Tests](#running-integration-tests) +- [Code Coverage](#code-coverage) - [Run Action Locally](#run-action-locally) +- [Branch Naming Convention (PID:H-1)](#branch-naming-convention-pidh-1) ## Get Started @@ -128,7 +131,62 @@ Unit tests are written using pytest. To run the tests, use the following command pytest tests/unit ``` -This will execute all tests located in the tests directory. +This will execute all tests located in the tests/unit directory. + +## Running Integration Tests + +Integration tests verify the complete action workflow from input to output using real GitHub API data. + +### Snapshot Tests (Mocked) + +Snapshot tests validate chapter population and markdown generation logic using mock data. They run on all PRs without requiring secrets. + +```shell +pytest tests/integration/ -v +``` + +These tests: +- Run on all PRs, including from forks +- Are deterministic and fast (<1 second) +- Test chapter population and backward compatibility +- Do not require secrets or network access + +### Integration Test (Real GitHub API) + +The integration test runs the complete action flow against a real GitHub repository to validate end-to-end functionality. + +**When it runs:** +- Automatically on same-repo PRs (NOT on forks for security) +- Uses the `GITHUB_TOKEN` secret to access the GitHub API +- Configured in `.github/workflows/test.yml` as the `integration-test-real-api` job + +**What it validates:** +1. Action exits with code 0 (success) +2. Output contains expected markdown chapter headers (e.g., `### New Features 🎉`) +3. Output contains issue/PR references (e.g., `#123`) +4. Output contains developer mentions (e.g., `@username`) +5. Logs include "completed successfully" message +6. Verbose logging is working (DEBUG level found) +7. Output contains "Generated release notes:" marker + +**To run locally** (requires `GITHUB_TOKEN` environment variable): + +```shell +export INPUT_TAG_NAME="v0.2.0" +export INPUT_GITHUB_REPOSITORY="AbsaOSS/generate-release-notes" +export INPUT_GITHUB_TOKEN="your_github_token" +export INPUT_CHAPTERS='[{"title": "Features 🎉", "label": "feature"}, {"title": "Bugfixes 🛠", "label": "bug"}]' +export INPUT_WARNINGS="true" +export INPUT_PRINT_EMPTY_CHAPTERS="false" +export INPUT_VERBOSE="true" +export INPUT_HIERARCHY="false" +export INPUT_DUPLICITY_SCOPE="both" +export INPUT_PUBLISHED_AT="false" +export INPUT_SKIP_RELEASE_NOTES_LABELS="skip-release-notes" +export PYTHONPATH="${PWD}" + +python main.py +``` ## Code Coverage diff --git a/requirements.txt b/requirements.txt index b0842bcf..7635aa78 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,7 @@ pytest==9.0.2 pytest-cov==7.0.0 pytest-mock==3.15.1 +responses==0.25.3 pylint==4.0.4 requests==2.32.5 black==25.12.0