fortress-completion-tests.yml

# ------------------------------------------------------------------------------------
#  Completion Report Test Analysis (Reusable Workflow) (GoFortress)
#
#  Purpose: Process all test-related artifacts for the completion report including
#  test results, test configuration analysis, and fuzz testing results.
#
#  This workflow handles:
#    - Test statistics processing and failure analysis
#    - Test configuration and output mode analysis
#    - Fuzz test statistics and security testing results
#    - Test failure details and error extraction
#
#  Maintainer: @mrz1836
#
# ------------------------------------------------------------------------------------

name: GoFortress (Completion Tests)

on:
  workflow_call:
    inputs:
      test-suite-result:
        description: "Result of the test suite job"
        required: true
        type: string
      env-json:
        description: "JSON string of environment variables"
        required: true
        type: string
    outputs:
      report-section:
        description: "Generated test analysis markdown section"
        value: ${{ jobs.analyze-tests.outputs.tests-markdown }}
      test-metrics:
        description: "Test performance metrics"
        value: ${{ jobs.analyze-tests.outputs.test-data }}
      failure-metrics:
        description: "Test failure analysis metrics"
        value: ${{ jobs.analyze-tests.outputs.failure-data }}

# Security: Restrict default permissions (jobs must explicitly request what they need)
permissions: {}

jobs:
  # ----------------------------------------------------------------------------------
  # Test Analysis
  # ----------------------------------------------------------------------------------
  analyze-tests:
    name: 🧪 Analyze Test Results
    runs-on: ubuntu-latest
    if: always()
    permissions:
      contents: read
      actions: read
    outputs:
      tests-markdown: ${{ steps.set-output.outputs.content }}
      test-data: ${{ steps.process-tests.outputs.test-metrics }}
      failure-data: ${{ steps.process-tests.outputs.failure-metrics }}
    steps:
      # --------------------------------------------------------------------
      # Checkout repository for local actions
      # --------------------------------------------------------------------
      - name: 📥 Checkout Repository
        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2

      # --------------------------------------------------------------------
      # Parse environment variables
      # --------------------------------------------------------------------
      - name: 🔧 Parse environment variables
        env:
          ENV_JSON: ${{ inputs.env-json }}
        run: |
          echo "📋 Setting environment variables..."
          echo "$ENV_JSON" | jq -r 'to_entries | .[] | "\(.key)=\(.value)"' | while IFS='=' read -r key value; do
            echo "$key=$value" >> $GITHUB_ENV
          done

      # --------------------------------------------------------------------
      # Download specific artifacts needed for test analysis
      # --------------------------------------------------------------------
      - name: 📥 Download benchmark statistics
        if: always()
        uses: ./.github/actions/download-artifact-resilient
        with:
          pattern: "bench-stats-*"
          path: ./artifacts/
          merge-multiple: true
          max-retries: ${{ env.ARTIFACT_DOWNLOAD_RETRIES }}
          retry-delay: ${{ env.ARTIFACT_DOWNLOAD_RETRY_DELAY }}
          timeout: ${{ env.ARTIFACT_DOWNLOAD_TIMEOUT }}
          continue-on-error: ${{ env.ARTIFACT_DOWNLOAD_CONTINUE_ON_ERROR }}

      - name: 📥 Download cache statistics
        if: always()
        uses: ./.github/actions/download-artifact-resilient
        with:
          pattern: "cache-stats-*"
          path: ./artifacts/
          merge-multiple: true
          max-retries: ${{ env.ARTIFACT_DOWNLOAD_RETRIES }}
          retry-delay: ${{ env.ARTIFACT_DOWNLOAD_RETRY_DELAY }}
          timeout: ${{ env.ARTIFACT_DOWNLOAD_TIMEOUT }}
          continue-on-error: ${{ env.ARTIFACT_DOWNLOAD_CONTINUE_ON_ERROR }}

      - name: 📥 Download fuzz test failure artifacts
        if: always() && env.ENABLE_GO_TESTS == 'true' && env.ENABLE_FUZZ_TESTING == 'true'
        uses: ./.github/actions/download-artifact-resilient
        with:
          pattern: "test-results-fuzz-*"
          path: ./test-artifacts/
          merge-multiple: true
          max-retries: ${{ env.ARTIFACT_DOWNLOAD_RETRIES }}
          retry-delay: ${{ env.ARTIFACT_DOWNLOAD_RETRY_DELAY }}
          timeout: ${{ env.ARTIFACT_DOWNLOAD_TIMEOUT }}
          continue-on-error: ${{ env.ARTIFACT_DOWNLOAD_CONTINUE_ON_ERROR }}

      - name: 📥 Download CI results (native mode)
        if: always() && env.ENABLE_GO_TESTS == 'true'
        uses: ./.github/actions/download-artifact-resilient
        with:
          pattern: "ci-results-*"
          path: ./ci-artifacts/
          merge-multiple: true
          max-retries: ${{ env.ARTIFACT_DOWNLOAD_RETRIES }}
          retry-delay: ${{ env.ARTIFACT_DOWNLOAD_RETRY_DELAY }}
          timeout: ${{ env.ARTIFACT_DOWNLOAD_TIMEOUT }}
          continue-on-error: ${{ env.ARTIFACT_DOWNLOAD_CONTINUE_ON_ERROR }}

      - name: 🗂️ Flatten artifacts
        if: always()
        run: |
          echo "🗂️ Flattening downloaded artifacts..."

          # Source shared helper functions for artifact processing
          source .github/scripts/parse-test-label.sh || { echo "❌ Failed to source parse-test-label.sh"; exit 1; }

          # Verify critical function is available
          if ! type copy_ci_artifact &>/dev/null; then
            echo "❌ Error: copy_ci_artifact function not found after sourcing"
            exit 1
          fi

          # Process stats artifacts (bench-stats, cache-stats JSON files)
          if [ -d "./artifacts/" ]; then
            find ./artifacts/ -name "*.json" -type f | while read -r file; do
              filename=$(basename "$file")
              echo "Moving $file to ./$filename"
              cp "$file" "./$filename"
            done
            echo "📋 Available stats files:"
            ls -la *-stats-*.json 2>/dev/null || echo "No stats files found"
          else
            echo "ℹ️ No artifacts directory found"
          fi

          # Process CI results from ci-artifacts (unit tests)
          if [ -d "./ci-artifacts/" ]; then
            echo "📋 Processing unit test CI results..."
            while IFS= read -r -d '' file; do
              copy_ci_artifact "$file" "ci" || true
            done < <(find ./ci-artifacts/ -name "*.jsonl" -type f -print0 2>/dev/null)
          fi

          # Process CI results from test-artifacts (fuzz tests)
          if [ -d "./test-artifacts/" ]; then
            echo "📋 Processing fuzz test CI results..."
            while IFS= read -r -d '' file; do
              copy_ci_artifact "$file" "ci" || true
            done < <(find ./test-artifacts/ -name "*.jsonl" -type f -print0 2>/dev/null)
          fi

          # Show all available JSONL files
          echo "📋 Available CI results JSONL files:"
          ls -la ci-*.jsonl 2>/dev/null || echo "No CI results JSONL files found"

      # --------------------------------------------------------------------
      # Initialize test analysis section
      # --------------------------------------------------------------------
      - name: 📝 Initialize Test Analysis Section
        run: |
          touch tests-section.md

      # --------------------------------------------------------------------
      # Process test statistics
      # --------------------------------------------------------------------
      - name: 🧪 Process Test Statistics
        id: process-tests
        run: |
          # Source shared helper function for generating test labels
          source .github/scripts/parse-test-label.sh || { echo "❌ Failed to source parse-test-label.sh"; exit 1; }

          # Enable nullglob so "for f in *.jsonl" loops safely skip when no files match
          # (prevents iterating with literal pattern string "ci-*.jsonl")
          shopt -s nullglob

          # Initialize totals for summary
          TOTAL_TESTS=0
          TOTAL_FAILURES=0
          TOTAL_PASSED=0
          TOTAL_SKIPPED=0
          SUITE_COUNT=0
          HAS_DATA=false

          # Check for native CI mode JSONL files first (preferred)
          if compgen -G "ci-*.jsonl" >/dev/null 2>&1; then
            echo "📊 Processing native CI mode JSONL files..."
            HAS_DATA=true

            {
              echo ""
              echo ""
              echo "### 🧪 Test Results Summary"
              echo "| Test Suite | Duration | Tests | Runs | Passed | Failed | Skipped | Status |"
              echo "|------------|----------|-------|------|--------|--------|---------|--------|"
            } >> tests-section.md

            # Process each JSONL file
            for jsonl_file in ci-*.jsonl; do
              if [ -f "$jsonl_file" ]; then
                # Extract artifact name from filename (ci-ARTIFACT_NAME-ci-results.jsonl)
                ARTIFACT_NAME=$(echo "$jsonl_file" | sed 's/^ci-//' | sed 's/-ci-results\.jsonl$//')
                SUITE_LABEL=$(parse_test_label "$ARTIFACT_NAME")

                # Extract summary line
                SUMMARY=$(grep '"type":"summary"' "$jsonl_file" 2>/dev/null | head -1 || echo "")

                if [[ -n "$SUMMARY" ]]; then
                  STATUS=$(echo "$SUMMARY" | jq -r '.summary.status // "unknown"')
                  PASSED=$(echo "$SUMMARY" | jq -r '.summary.passed // 0')
                  FAILED=$(echo "$SUMMARY" | jq -r '.summary.failed // 0')
                  SKIPPED=$(echo "$SUMMARY" | jq -r '.summary.skipped // 0')
                  TOTAL=$(echo "$SUMMARY" | jq -r '.summary.total // 0')
                  UNIQUE=$(echo "$SUMMARY" | jq -r '.summary.unique_total // .summary.total // 0')
                  DURATION=$(echo "$SUMMARY" | jq -r '.summary.duration // "0s"')

                  STATUS_ICON=$([[ "$STATUS" == "passed" ]] && echo "✅" || echo "❌")

                  echo "| $SUITE_LABEL | $DURATION | $UNIQUE | $TOTAL | $PASSED | $FAILED | $SKIPPED | $STATUS_ICON |" >> tests-section.md

                  # Accumulate totals (use unique for primary test count)
                  TOTAL_TESTS=$((TOTAL_TESTS + UNIQUE))
                  TOTAL_PASSED=$((TOTAL_PASSED + PASSED))
                  TOTAL_FAILURES=$((TOTAL_FAILURES + FAILED))
                  TOTAL_SKIPPED=$((TOTAL_SKIPPED + SKIPPED))
                  SUITE_COUNT=$((SUITE_COUNT + 1))
                fi
              fi
            done

            # Store totals as outputs
            echo "test-metrics={\"total_tests\":$TOTAL_TESTS,\"total_failures\":$TOTAL_FAILURES,\"suite_count\":$SUITE_COUNT}" >> $GITHUB_OUTPUT

            # Add failure analysis if any failures exist
            if [[ $TOTAL_FAILURES -gt 0 ]]; then
              {
                echo ""
                echo ""
                echo "### ❌ Test Failure Analysis"
                echo "**Total Failures**: $TOTAL_FAILURES across $SUITE_COUNT test suite(s)"
                echo ""
              } >> tests-section.md

              # Show failures by suite
              echo "#### 📊 Failures by Test Suite:" >> tests-section.md
              for jsonl_file in ci-*.jsonl; do
                if [ -f "$jsonl_file" ]; then
                  ARTIFACT_NAME=$(echo "$jsonl_file" | sed 's/^ci-//' | sed 's/-ci-results\.jsonl$//')
                  SUITE_LABEL=$(parse_test_label "$ARTIFACT_NAME")
                  SUMMARY=$(grep '"type":"summary"' "$jsonl_file" 2>/dev/null | head -1 || echo "")

                  if [[ -n "$SUMMARY" ]]; then
                    FAILED=$(echo "$SUMMARY" | jq -r '.summary.failed // 0')
                    if [[ $FAILED -gt 0 ]]; then
                      echo "- **$SUITE_LABEL**: $FAILED failures" >> tests-section.md
                    fi
                  fi
                fi
              done

              # Add collapsible section for failed tests
              {
                echo ""
                echo "<details>"
                echo "<summary>🔍 Failed Tests (click to expand)</summary>"
                echo ""
                echo "| Test Name | Package | Error |"
                echo "|-----------|---------|-------|"
              } >> tests-section.md

              # Extract failure details from all JSONL files
              FAILURE_COUNT=0
              for jsonl_file in ci-*.jsonl; do
                if [ -f "$jsonl_file" ] && [[ $FAILURE_COUNT -lt 20 ]]; then
                  while read -r line; do
                    if [[ $FAILURE_COUNT -ge 20 ]]; then
                      break
                    fi

                    TEST=$(echo "$line" | jq -r '.failure.test // "unknown"')
                    PKG=$(echo "$line" | jq -r '.failure.package // "unknown"' | sed 's|.*/||')
                    ERROR=$(echo "$line" | jq -r '.failure.error // ""' | head -c 100 | tr '\n' ' ')

                    # Truncate error message for table display (max 80 chars: 77 + "...")
                    if [[ ${#ERROR} -gt 80 ]]; then
                      ERROR="${ERROR:0:77}..."
                    fi

                    echo "| \`$TEST\` | $PKG | ${ERROR:-_no message_} |"
                    FAILURE_COUNT=$((FAILURE_COUNT + 1))
                  done < <(grep '"type":"failure"' "$jsonl_file" 2>/dev/null) >> tests-section.md || true
                fi
              done

              {
                echo ""
                echo "</details>"
              } >> tests-section.md

              # Store failure metrics
              echo "failure-metrics={\"total_failures\":$TOTAL_FAILURES,\"has_error_output\":true}" >> $GITHUB_OUTPUT
            fi
          fi

          # No test statistics available
          if [[ "$HAS_DATA" == "false" ]]; then
            {
              echo ""
              echo ""
              echo "### 🧪 Test Results Summary"
              echo ""
              echo "| Status | Details |"
              echo "|--------|---------|"
              if [[ "${{ env.ENABLE_GO_TESTS }}" == "false" ]]; then
                echo "| **Test Suite** | ❌ Disabled - Set ENABLE_GO_TESTS=true to enable |"
                echo "| **Reason** | Tests are disabled via configuration flag |"
                echo "| **Note** | Enable ENABLE_GO_TESTS in .github/env/00-core.env to run tests |"
              else
                echo "| **Test Suite** | ⚠️ Skipped - No test statistics available |"
                echo "| **Reason** | Tests may have been skipped for fork PR security restrictions |"
                echo "| **Note** | Repository maintainers can run full tests on merged code |"
                echo ""
                echo "_For security reasons, fork PRs do not have access to test execution secrets._"
              fi
            } >> tests-section.md
          fi

      # --------------------------------------------------------------------
      # Add test configuration and output analysis
      # --------------------------------------------------------------------
      - name: 🎛️ Add Test Configuration Section
        id: add-test-config
        run: |
          # Add test output configuration section
          HAS_CONFIG_DATA=false

          # Check for native CI mode JSONL files
          if compgen -G "ci-*.jsonl" >/dev/null 2>&1; then
            HAS_CONFIG_DATA=true
            {
              echo ""
              echo "<br><br>"
              echo ""
              echo "### 🎛️ Test Output Configuration"
              echo ""
              echo "**Output Mode**: Native CI Mode (JSONL)"
              echo ""
              echo "- Tests executed with magex native CI mode"
              echo "- Structured output in .mage-x/ci-results.jsonl"
              echo "- Automatic GitHub annotations for failures"
            } >> tests-section.md
          fi

          if [[ "$HAS_CONFIG_DATA" == "false" ]]; then
            # No test configuration to display - test stats not available
            echo "" >> tests-section.md
            echo "ℹ️ _Test configuration section skipped - no test data available_" >> tests-section.md
          fi

      # --------------------------------------------------------------------
      # Process fuzz test statistics
      # --------------------------------------------------------------------
      - name: 🎯 Process Fuzz Test Statistics
        id: process-fuzz
        run: |
          # Process fuzz test statistics - always show status
          {
            echo "<br><br>"
            echo ""
            echo "### 🛡️ Security Testing Results"
          } >> tests-section.md

          # Check if fuzz testing is enabled in environment
          if [[ "${{ env.ENABLE_FUZZ_TESTING }}" == "true" ]]; then
            # Look for fuzz test JSONL files (native CI mode)
            FUZZ_JSONL=$(ls ci-*-ci-results-fuzz.jsonl 2>/dev/null | head -1 || echo "")

            if [[ -n "$FUZZ_JSONL" ]] && [[ -f "$FUZZ_JSONL" ]]; then
              # Extract summary from fuzz JSONL
              SUMMARY=$(grep '"type":"summary"' "$FUZZ_JSONL" 2>/dev/null | head -1 || echo "")

              if [[ -n "$SUMMARY" ]]; then
                STATUS=$(echo "$SUMMARY" | jq -r '.summary.status // "unknown"')
                TOTAL=$(echo "$SUMMARY" | jq -r '.summary.total // 0')
                DURATION=$(echo "$SUMMARY" | jq -r '.summary.duration // "0s"')

                STATUS_ICON=$([[ "$STATUS" == "passed" ]] && echo "✅" || echo "❌")

                # Create table with fuzz test data from JSONL
                {
                  echo "| Fuzz Suite | Duration | Fuzz Tests | Status | Enabled |"
                  echo "|------------|----------|------------|--------|---------|"
                  echo "| Fuzz Tests | $DURATION | $TOTAL | $STATUS_ICON | 🎯 |"
                } >> tests-section.md
              else
                # JSONL found but no summary record
                {
                  echo "| Status | Details |"
                  echo "|--------|---------|"
                  echo "| **Fuzz Testing** | ✅ Enabled |"
                  echo "| **Execution** | ⚠️ No fuzz summary found in JSONL - check job logs |"
                  echo "| **Platform** | Linux with primary Go version |"
                } >> tests-section.md
              fi
            else
              # No fuzz JSONL found
              {
                echo "| Status | Details |"
                echo "|--------|---------|"
                echo "| **Fuzz Testing** | ✅ Enabled |"
                echo "| **Execution** | ⚠️ No fuzz results found - check job logs |"
                echo "| **Platform** | Linux with primary Go version |"
              } >> tests-section.md
            fi
          else
            # Fuzz testing is disabled
            {
              echo "| Status | Details |"
              echo "|--------|---------|"
              echo "| **Fuzz Testing** | ❌ Disabled |"
              echo "| **Configuration** | Set ENABLE_FUZZ_TESTING=true to enable |"
              echo "| **Target Platform** | Would run on Linux with primary Go version |"
            } >> tests-section.md
          fi

      # --------------------------------------------------------------------
      # Upload test analysis section
      # --------------------------------------------------------------------
      - name: 📤 Upload Test Analysis Section
        id: upload-section
        if: always()
        run: |
          if [ -f "tests-section.md" ] && [ -s "tests-section.md" ]; then
            echo "🧪 Test section found, uploading..."
            ls -la tests-section.md
            echo "📋 Content preview:"
            head -5 tests-section.md
          else
            echo "⚠️ Test section file missing or empty, creating minimal section..."
            echo "### 🧪 Test Results Section" > tests-section.md
            echo "No test data available for this run." >> tests-section.md
          fi

      - name: 📤 Upload Test Artifact
        uses: ./.github/actions/upload-statistics
        with:
          artifact-name: "tests-section"
          artifact-path: "tests-section.md"
          retention-days: "7"
          if-no-files-found: "warn"

      - name: 📋 Set Output Content
        id: set-output
        run: |
          echo "content<<EOF" >> $GITHUB_OUTPUT
          cat tests-section.md >> $GITHUB_OUTPUT
          echo "EOF" >> $GITHUB_OUTPUT