wstest.yml

name: wstest

on:
  push:
    branches: [master]
    tags:
      - 'v*'
  pull_request:
    branches: [master]
  workflow_dispatch:
    inputs:
      test_mode:
        description: "Test mode to run"
        required: false
        default: "quick"
        type: choice
        options:
          - quick
          - full

env:
  UV_CACHE_DIR: ${{ github.workspace }}/.uv-cache
  # Test mode: 'quick' for fast CI, 'full' for comprehensive testing
  TEST_MODE: ${{ github.event.inputs.test_mode || 'quick' }}

jobs:
  identifiers:
    # GitHub needs to know where .cicd/workflows/identifiers.yml lives at parse time,
    # and submodules aren't included in that context! thus the following does NOT work:
    # uses: ./.cicd/workflows/identifiers.yml
    # we MUST reference the remote repo directly:
    uses: wamp-proto/wamp-cicd/.github/workflows/identifiers.yml@main
    # IMPORTANT: we still need .cicd as a Git submodule in the using repo though!
    # because e.g. identifiers.yml wants to access scripts/sanitize.sh !

  # Client testing: Test autobahn-python clients against testsuite server
  client-conformance:
    name: WebSocket Client Testing (All Combinations)
    needs: identifiers
    runs-on: ubuntu-24.04

    env:
      BASE_REPO: ${{ needs.identifiers.outputs.base_repo }}
      BASE_BRANCH: ${{ needs.identifiers.outputs.base_branch }}
      PR_NUMBER: ${{ needs.identifiers.outputs.pr_number }}
      PR_REPO: ${{ needs.identifiers.outputs.pr_repo }}
      PR_BRANCH: ${{ needs.identifiers.outputs.pr_branch }}

    steps:
      - name: Checkout code
        uses: actions/checkout@v4
        with:
          submodules: recursive

      - name: Install toolchain (just & uv)
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        run: |
          curl --proto '=https' --tlsv1.2 -sSf https://just.systems/install.sh | bash -s -- --to ~/bin
          curl -LsSf https://astral.sh/uv/install.sh | sh
          echo "$HOME/bin" >> $GITHUB_PATH
          echo "$HOME/.cargo/bin" >> $GITHUB_PATH

      - name: Setup uv cache
        uses: actions/cache@v4
        with:
          path: ${{ env.UV_CACHE_DIR }}
          key:
            uv-cache-clients-${{ hashFiles('pyproject.toml') }}

      - name:
          Setup Python environments and install Autobahn|Python
        run: |
          just install cpy311
          just install cpy314
          just install pypy311

      - name: Run all client test combinations (with-nvx and without-nvx)
        run: |
          # Test both NVX configurations sequentially
          # NOTE: We restart fuzzingserver for each configuration to ensure clean report directories
          for nvx_config in "with-nvx" "without-nvx"; do
            if [ "$nvx_config" = "with-nvx" ]; then
              export AUTOBAHN_USE_NVX=1
              echo "======================================================================"
              echo "==> Testing WITH NVX acceleration (AUTOBAHN_USE_NVX=1)"
              echo "======================================================================"
            else
              export AUTOBAHN_USE_NVX=0
              echo "======================================================================"
              echo "==> Testing WITHOUT NVX acceleration (AUTOBAHN_USE_NVX=0)"
              echo "======================================================================"
            fi

            # Start fuzzingserver for this configuration
            echo "==> Starting fuzzingserver for $nvx_config..."
            nohup just wstest-fuzzingserver "" "" ${{ env.TEST_MODE }} > fuzzingserver-$nvx_config.log 2>&1 &
            echo $! > fuzzingserver-$nvx_config.pid

            echo "==> Waiting for fuzzingserver to start..."
            for i in {1..30}; do
              if curl -f http://127.0.0.1:9001 >/dev/null 2>&1; then
                echo "✅ Fuzzingserver is running after ${i} seconds"
                break
              fi
              if [ $i -eq 30 ]; then
                echo "❌ Fuzzingserver failed to start after 30 seconds"
                echo "==> Fuzzingserver log:"
                cat fuzzingserver-$nvx_config.log
                exit 1
              fi
              sleep 1
            done

            echo "==> Running client tests for all 6 combinations ($nvx_config)..."

            # Client 1: Twisted + CPython 3.11
            echo "==> Testing Twisted + cpy311 client ($nvx_config)..."
            just wstest-testeeclient-twisted cpy311

            # Client 2: asyncio + CPython 3.11
            echo "==> Testing asyncio + cpy311 client ($nvx_config)..."
            just wstest-testeeclient-asyncio cpy311

            # Client 3: Twisted + CPython 3.14
            echo "==> Testing Twisted + cpy314 client ($nvx_config)..."
            just wstest-testeeclient-twisted cpy314

            # Client 4: asyncio + CPython 3.14
            echo "==> Testing asyncio + cpy314 client ($nvx_config)..."
            just wstest-testeeclient-asyncio cpy314

            # Client 5: Twisted + PyPy 3.11
            echo "==> Testing Twisted + pypy311 client ($nvx_config)..."
            just wstest-testeeclient-twisted pypy311

            # Client 6: asyncio + PyPy 3.11
            echo "==> Testing asyncio + pypy311 client ($nvx_config)..."
            just wstest-testeeclient-asyncio pypy311

            echo "✅ All 6 client test combinations completed ($nvx_config)"

            # Give fuzzingserver time to write all reports to disk
            echo "==> Waiting for fuzzingserver to finish writing reports..."
            sleep 3

            # Check if Docker container is still running
            echo "==> Docker container status:"
            docker ps | grep fuzzingserver || echo "fuzzingserver container not found in ps"
            docker ps -a | grep fuzzingserver || echo "fuzzingserver container not found in ps -a"

            # Stop fuzzingserver for this configuration
            echo "==> Stopping fuzzingserver for $nvx_config..."
            if [ -f fuzzingserver-$nvx_config.pid ]; then
              kill $(cat fuzzingserver-$nvx_config.pid) || true
              rm -f fuzzingserver-$nvx_config.pid
            fi
            docker stop fuzzingserver || true

            # Wait for port to be released
            sleep 2

            # Debug: Check what the fuzzingserver created
            echo "==> Debugging: fuzzingserver output directory:"
            find . -name "*.json" -path "*/.wstest/*" -type f 2>/dev/null | head -20 || echo "No JSON files found in .wstest"
            echo ""
            echo "==> Directory structure after tests:"
            ls -laR .wstest/ 2>/dev/null | head -50 || echo "No .wstest directory"

            # Move reports to nvx-specific directory
            echo "==> Checking for client reports in .wstest/clients..."
            if [ -d .wstest/clients ]; then
              echo "✅ .wstest/clients exists"
              ls -la .wstest/clients/ | head -20
              echo "File count: $(find .wstest/clients -type f | wc -l)"

              # Change ownership from root to runner (Docker creates files as root)
              # We need write permission on the parent .wstest/ directory to move subdirectories
              echo "==> Changing ownership of .wstest/ directory to runner user..."
              sudo chown -R runner:runner .wstest/

              # Now we can move the directory
              mv .wstest/clients .wstest/clients-$nvx_config
              echo "✅ Reports moved to .wstest/clients-$nvx_config"
              ls -la .wstest/clients-$nvx_config/ | head -20
            else
              echo "⚠️  Warning: .wstest/clients directory not found for $nvx_config"
              echo "==> Checking .wstest directory structure:"
              ls -la .wstest/ || echo "No .wstest directory"
            fi
          done

          echo "======================================================================"
          echo "✅ All client tests completed for BOTH NVX configurations"
          echo "======================================================================"

          echo "==> Final directory structure:"
          ls -la .wstest/
          echo ""
          echo "==> With-NVX client reports:"
          ls -la .wstest/clients-with-nvx/ | head -10 || echo "Not found"
          echo ""
          echo "==> Without-NVX client reports:"
          ls -la .wstest/clients-without-nvx/ | head -10 || echo "Not found"

      - name: Upload client reports (with-nvx)
        uses: actions/upload-artifact@v4
        if: always()
        with:
          name: clients-all-${{ env.TEST_MODE }}-with-nvx
          path: .wstest/clients-with-nvx/
          retention-days: 30

      - name: Upload client reports (without-nvx)
        uses: actions/upload-artifact@v4
        if: always()
        with:
          name: clients-all-${{ env.TEST_MODE }}-without-nvx
          path: .wstest/clients-without-nvx/
          retention-days: 30

  # Server testing: Test all server combinations against testsuite client
  server-conformance:
    name: WebSocket Server Testing (All Combinations)
    needs: identifiers
    runs-on: ubuntu-24.04

    env:
      BASE_REPO: ${{ needs.identifiers.outputs.base_repo }}
      BASE_BRANCH: ${{ needs.identifiers.outputs.base_branch }}
      PR_NUMBER: ${{ needs.identifiers.outputs.pr_number }}
      PR_REPO: ${{ needs.identifiers.outputs.pr_repo }}
      PR_BRANCH: ${{ needs.identifiers.outputs.pr_branch }}

    steps:
      - name: Checkout code
        uses: actions/checkout@v4
        with:
          submodules: recursive

      - name: Install toolchain (just & uv)
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        run: |
          curl --proto '=https' --tlsv1.2 -sSf https://just.systems/install.sh | bash -s -- --to ~/bin
          curl -LsSf https://astral.sh/uv/install.sh | sh
          echo "$HOME/bin" >> $GITHUB_PATH
          echo "$HOME/.cargo/bin" >> $GITHUB_PATH

      - name: Setup uv cache
        uses: actions/cache@v4
        with:
          path: ${{ env.UV_CACHE_DIR }}
          key:
            uv-cache-servers-${{ hashFiles('pyproject.toml') }}

      - name:
          Setup Python environments and install Autobahn|Python
        run: |
          just install cpy311
          just install cpy314
          just install pypy311

      - name: Run server tests (with-nvx and without-nvx)
        run: |
          # Test both NVX configurations sequentially
          # NOTE: Servers must be restarted for each configuration to ensure correct NVX setting
          for nvx_config in "with-nvx" "without-nvx"; do
            if [ "$nvx_config" = "with-nvx" ]; then
              export AUTOBAHN_USE_NVX=1
              echo "======================================================================"
              echo "==> Testing WITH NVX acceleration (AUTOBAHN_USE_NVX=1)"
              echo "======================================================================"
            else
              export AUTOBAHN_USE_NVX=0
              echo "======================================================================"
              echo "==> Testing WITHOUT NVX acceleration (AUTOBAHN_USE_NVX=0)"
              echo "======================================================================"
            fi

            echo "==> Starting 6 WebSocket server combinations for $nvx_config..."

            # Server 1: Twisted + CPython 3.11 on port 9011
            echo "==> Starting Twisted + cpy311 server on port 9011 ($nvx_config)..."
            nohup .venvs/cpy311/bin/python ./wstest/testee_server_tx.py --url ws://127.0.0.1:9011 > twisted-cpy311-$nvx_config.log 2>&1 &
            echo $! > twisted-cpy311.pid
            for i in {1..30}; do
              if netstat -tuln | grep :9011 >/dev/null 2>&1; then
                echo "✅ Twisted + cpy311 server running on port 9011"
                break
              fi
              if [ $i -eq 30 ]; then
                echo "❌ Twisted + cpy311 server failed to start on port 9011"
                cat twisted-cpy311-$nvx_config.log
                exit 1
              fi
              sleep 1
            done

            # Server 2: asyncio + CPython 3.11 on port 9012
            echo "==> Starting asyncio + cpy311 server on port 9012 ($nvx_config)..."
            nohup .venvs/cpy311/bin/python ./wstest/testee_server_aio.py --url ws://127.0.0.1:9012 > asyncio-cpy311-$nvx_config.log 2>&1 &
            echo $! > asyncio-cpy311.pid
            for i in {1..30}; do
              if netstat -tuln | grep :9012 >/dev/null 2>&1; then
                echo "✅ asyncio + cpy311 server running on port 9012"
                break
              fi
              if [ $i -eq 30 ]; then
                echo "❌ asyncio + cpy311 server failed to start on port 9012"
                cat asyncio-cpy311-$nvx_config.log
                exit 1
              fi
              sleep 1
            done

            # Server 3: Twisted + CPython 3.14 on port 9013
            echo "==> Starting Twisted + cpy314 server on port 9013 ($nvx_config)..."
            nohup .venvs/cpy314/bin/python ./wstest/testee_server_tx.py --url ws://127.0.0.1:9013 > twisted-cpy314-$nvx_config.log 2>&1 &
            echo $! > twisted-cpy314.pid
            for i in {1..30}; do
              if netstat -tuln | grep :9013 >/dev/null 2>&1; then
                echo "✅ Twisted + cpy314 server running on port 9013"
                break
              fi
              if [ $i -eq 30 ]; then
                echo "❌ Twisted + cpy314 server failed to start on port 9013"
                cat twisted-cpy314-$nvx_config.log
                exit 1
              fi
              sleep 1
            done

            # Server 4: asyncio + CPython 3.14 on port 9014
            echo "==> Starting asyncio + cpy314 server on port 9014 ($nvx_config)..."
            nohup .venvs/cpy314/bin/python ./wstest/testee_server_aio.py --url ws://127.0.0.1:9014 > asyncio-cpy314-$nvx_config.log 2>&1 &
            echo $! > asyncio-cpy314.pid
            for i in {1..30}; do
              if netstat -tuln | grep :9014 >/dev/null 2>&1; then
                echo "✅ asyncio + cpy314 server running on port 9014"
                break
              fi
              if [ $i -eq 30 ]; then
                echo "❌ asyncio + cpy314 server failed to start on port 9014"
                cat asyncio-cpy314-$nvx_config.log
                exit 1
              fi
              sleep 1
            done

            # Server 5: Twisted + PyPy 3.11 on port 9015
            echo "==> Starting Twisted + pypy311 server on port 9015 ($nvx_config)..."
            nohup .venvs/pypy311/bin/python ./wstest/testee_server_tx.py --url ws://127.0.0.1:9015 > twisted-pypy311-$nvx_config.log 2>&1 &
            echo $! > twisted-pypy311.pid
            for i in {1..30}; do
              if netstat -tuln | grep :9015 >/dev/null 2>&1; then
                echo "✅ Twisted + pypy311 server running on port 9015"
                break
              fi
              if [ $i -eq 30 ]; then
                echo "❌ Twisted + pypy311 server failed to start on port 9015"
                cat twisted-pypy311-$nvx_config.log
                exit 1
              fi
              sleep 1
            done

            # Server 6: asyncio + PyPy 3.11 on port 9016
            echo "==> Starting asyncio + pypy311 server on port 9016 ($nvx_config)..."
            nohup .venvs/pypy311/bin/python ./wstest/testee_server_aio.py --url ws://127.0.0.1:9016 > asyncio-pypy311-$nvx_config.log 2>&1 &
            echo $! > asyncio-pypy311.pid
            for i in {1..30}; do
              if netstat -tuln | grep :9016 >/dev/null 2>&1; then
                echo "✅ asyncio + pypy311 server running on port 9016"
                break
              fi
              if [ $i -eq 30 ]; then
                echo "❌ asyncio + pypy311 server failed to start on port 9016"
                cat asyncio-pypy311-$nvx_config.log
                exit 1
              fi
              sleep 1
            done

            echo "✅ All 6 WebSocket servers running successfully for $nvx_config"

            echo "==> Running fuzzingclient tests against all 6 servers ($nvx_config)..."
            just wstest-fuzzingclient "" "" ${{ env.TEST_MODE }}

            echo "✅ Server tests completed ($nvx_config)"

            # Stop all servers for this configuration
            echo "==> Stopping all WebSocket servers for $nvx_config..."
            for pid_file in twisted-cpy311.pid asyncio-cpy311.pid twisted-cpy314.pid asyncio-cpy314.pid twisted-pypy311.pid asyncio-pypy311.pid; do
              if [ -f "$pid_file" ]; then
                kill $(cat "$pid_file") 2>/dev/null || true
                rm -f "$pid_file"
              fi
            done

            # Wait for ports to be released before next configuration
            echo "==> Waiting for ports to be released..."
            sleep 3

            # Move reports to nvx-specific directory
            if [ -d .wstest/servers ]; then
              mv .wstest/servers .wstest/servers-$nvx_config || true
              echo "✅ Reports moved to .wstest/servers-$nvx_config"
            else
              echo "⚠️  Warning: .wstest/servers directory not found for $nvx_config"
            fi
          done

          echo "======================================================================"
          echo "✅ All server tests completed for BOTH NVX configurations"
          echo "======================================================================"

      - name: Upload server reports (with-nvx)
        uses: actions/upload-artifact@v4
        if: always()
        with:
          name: servers-all-${{ env.TEST_MODE }}-with-nvx
          path: .wstest/servers-with-nvx/
          retention-days: 30

      - name: Upload server reports (without-nvx)
        uses: actions/upload-artifact@v4
        if: always()
        with:
          name: servers-all-${{ env.TEST_MODE }}-without-nvx
          path: .wstest/servers-without-nvx/
          retention-days: 30

  # Consolidate all reports using the proven justfile recipe
  consolidate-reports:
    name: Consolidate WebSocket Reports
    needs: [identifiers, client-conformance, server-conformance]
    runs-on: ubuntu-24.04
    if: always()

    env:
      BASE_REPO: ${{ needs.identifiers.outputs.base_repo }}
      BASE_BRANCH: ${{ needs.identifiers.outputs.base_branch }}
      PR_NUMBER: ${{ needs.identifiers.outputs.pr_number }}
      PR_REPO: ${{ needs.identifiers.outputs.pr_repo }}
      PR_BRANCH: ${{ needs.identifiers.outputs.pr_branch }}

    steps:
      - name: Checkout code
        uses: actions/checkout@v4
        with:
          submodules: recursive

      - name: Install Just
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        run: |
          curl --proto '=https' --tlsv1.2 -sSf https://just.systems/install.sh | bash -s -- --to ~/bin
          echo "$HOME/bin" >> $GITHUB_PATH

      - name: Download all artifacts
        uses: actions/download-artifact@v4
        with:
          path: artifacts/

      - name: Reconstruct test directory structure
        run: |
          echo "==> Reconstructing .wstest directory structure for both NVX configurations..."

          # Debug: Show what artifacts we actually have
          echo "==> Available artifacts:"
          ls -la artifacts/ || echo "No artifacts directory"

          # Process each NVX configuration separately
          for nvx_config in "with-nvx" "without-nvx"; do
            echo "======================================================================"
            echo "==> Processing $nvx_config configuration"
            echo "======================================================================"

            mkdir -p .wstest/clients-$nvx_config .wstest/servers-$nvx_config

            # Combine client reports for this NVX configuration
            find artifacts -name "clients-all-*-$nvx_config" -type d | while read dir; do
              echo "Processing client reports: $dir"
              cp -r "$dir"/* .wstest/clients-$nvx_config/ || true
            done

            # Copy server reports for this NVX configuration
            find artifacts -name "servers-all-*-$nvx_config" -type d | while read dir; do
              echo "Processing server reports: $dir"
              cp -r "$dir"/* .wstest/servers-$nvx_config/ || true
            done

            echo "==> Reconstructed client reports ($nvx_config):"
            ls -la .wstest/clients-$nvx_config/ || echo "None"
            echo ""
            echo "==> Reconstructed server reports ($nvx_config):"
            ls -la .wstest/servers-$nvx_config/ || echo "None"
            echo ""
          done

      - name: Consolidate reports for documentation
        run: |
          # Process each NVX configuration separately
          for nvx_config in "with-nvx" "without-nvx"; do
            echo "======================================================================"
            echo "==> Consolidating reports for $nvx_config configuration"
            echo "======================================================================"

            # Create temporary .wstest structure expected by justfile recipe
            rm -rf .wstest/clients .wstest/servers || true
            mkdir -p .wstest/clients .wstest/servers
            cp -r .wstest/clients-$nvx_config/* .wstest/clients/ || true
            cp -r .wstest/servers-$nvx_config/* .wstest/servers/ || true

            # Run consolidation recipe
            just wstest-consolidate-reports

            # Move consolidated reports to nvx-specific subdirectory in docs
            mkdir -p docs/_static/websocket/conformance-$nvx_config/clients
            mkdir -p docs/_static/websocket/conformance-$nvx_config/servers
            cp -r docs/_static/websocket/conformance/clients/* docs/_static/websocket/conformance-$nvx_config/clients/ || true
            cp -r docs/_static/websocket/conformance/servers/* docs/_static/websocket/conformance-$nvx_config/servers/ || true

            echo "✅ Consolidation complete for $nvx_config"
          done

      - name: Generate summary tables from index.json files
        run: |
          echo "==> Generating conformance summary tables for both NVX configurations..."

          # Create summary filename using consistent identifier from identifiers workflow
          RELEASE_NAME="${{ needs.identifiers.outputs.release_name }}"
          SUMMARY_FILE="${{ github.workspace }}/.audit/${RELEASE_NAME}-wstest-summary.md"
          echo "Summary file: $SUMMARY_FILE"

          # Generate combined summary header
          echo "# WebSocket Conformance Test Results" > "$SUMMARY_FILE"
          echo "" >> "$SUMMARY_FILE"
          echo "Test Mode: \`${{ env.TEST_MODE }}\`" >> "$SUMMARY_FILE"
          echo "Release: $RELEASE_NAME" >> "$SUMMARY_FILE"
          echo "Repo: ${{ needs.identifiers.outputs.pr_repo }}" >> "$SUMMARY_FILE"
          echo "Branch: ${{ needs.identifiers.outputs.pr_branch }}" >> "$SUMMARY_FILE"
          echo "PR: ${{ needs.identifiers.outputs.pr_number }}" >> "$SUMMARY_FILE"
          echo "" >> "$SUMMARY_FILE"

          # Generate summaries for each NVX configuration
          for nvx_config in "with-nvx" "without-nvx"; do
            echo "---" >> "$SUMMARY_FILE"
            echo "" >> "$SUMMARY_FILE"
            echo "## Configuration: $nvx_config" >> "$SUMMARY_FILE"
            echo "" >> "$SUMMARY_FILE"

            # Generate client summary for this configuration
            python3 ${{ github.workspace }}/.github/workflows/generate_summary.py \
              "docs/_static/websocket/conformance-$nvx_config/clients/index.json" \
              "Client Conformance ($nvx_config)" >> "$SUMMARY_FILE"

            # Generate server summary for this configuration
            python3 ${{ github.workspace }}/.github/workflows/generate_summary.py \
              "docs/_static/websocket/conformance-$nvx_config/servers/index.json" \
              "Server Conformance ($nvx_config)" >> "$SUMMARY_FILE"
          done

          # Show summary in workflow log
          echo "==> Summary Report:"
          cat "$SUMMARY_FILE"

          # Store filename for next step
          echo "SUMMARY_FILE=$SUMMARY_FILE" >> $GITHUB_ENV

      - name: Upload consolidated reports (with-nvx)
        uses: actions/upload-artifact@v4
        with:
          name: websocket-conformance-docs-${{ env.TEST_MODE }}-with-nvx
          path: docs/_static/websocket/conformance-with-nvx/
          retention-days: 30

      - name: Upload consolidated reports (without-nvx)
        uses: actions/upload-artifact@v4
        with:
          name: websocket-conformance-docs-${{ env.TEST_MODE }}-without-nvx
          path: docs/_static/websocket/conformance-without-nvx/
          retention-days: 30

      - name: Upload summary report for PR comment
        uses: actions/upload-artifact@v4
        with:
          name: conformance-summary-${{ env.TEST_MODE }}
          path: ${{ env.SUMMARY_FILE }}
          retention-days: 30

      - name: Report summary
        run: |
          echo "==> WebSocket Conformance Testing Complete ✅"
          echo "Test Mode: ${{ env.TEST_MODE }}"
          echo "Matrix tested: (client|server) × (twisted|asyncio) × (cpy311|cpy314|pypy311) × (with-nvx|without-nvx)"
          echo ""
          echo "Reports generated for both NVX configurations:"
          for nvx_config in "with-nvx" "without-nvx"; do
            echo ""
            echo "==== $nvx_config ===="
            find docs/_static/websocket/conformance-$nvx_config -name "*.zip" 2>/dev/null | while read zip; do
              echo "📦 $(basename "$zip")"
            done || echo "No reports found for $nvx_config"
          done
          echo ""
          echo "📄 Documentation: docs/_static/websocket/conformance-{with-nvx,without-nvx}/"
          echo "📊 Summary: summary.md"

  # Verify all tests passed (100% conformance) - fails if any test failed
  verify-reports:
    name: Verify WebSocket Conformance (100% Pass Required)
    needs: [identifiers, consolidate-reports]
    runs-on: ubuntu-24.04
    if: always()

    env:
      BASE_REPO: ${{ needs.identifiers.outputs.base_repo }}
      BASE_BRANCH: ${{ needs.identifiers.outputs.base_branch }}
      PR_NUMBER: ${{ needs.identifiers.outputs.pr_number }}
      PR_REPO: ${{ needs.identifiers.outputs.pr_repo }}
      PR_BRANCH: ${{ needs.identifiers.outputs.pr_branch }}

    steps:
      - name: Checkout code
        uses: actions/checkout@v4
        with:
          submodules: recursive

      - name: Download consolidated reports (with-nvx)
        uses: actions/download-artifact@v4
        with:
          name: websocket-conformance-docs-${{ env.TEST_MODE }}-with-nvx
          path: docs/_static/websocket/conformance-with-nvx/

      - name: Download consolidated reports (without-nvx)
        uses: actions/download-artifact@v4
        with:
          name: websocket-conformance-docs-${{ env.TEST_MODE }}-without-nvx
          path: docs/_static/websocket/conformance-without-nvx/

      - name: Verify 100% conformance (both NVX configurations)
        run: |
          echo "======================================================================"
          echo "==> Verifying 100% conformance for BOTH NVX configurations"
          echo "======================================================================"

          # Verify each NVX configuration independently
          for nvx_config in "with-nvx" "without-nvx"; do
            echo ""
            echo "======================================================================"
            echo "==> Verifying $nvx_config configuration"
            echo "======================================================================"

            echo "==> Debugging directory structure ($nvx_config):"
            find docs/_static/websocket/conformance-$nvx_config -type f -name "*.json" | head -10
            echo ""
            ls -la docs/_static/websocket/conformance-$nvx_config/ || echo "conformance dir not found"
            echo ""
            ls -la docs/_static/websocket/conformance-$nvx_config/clients/ || echo "clients dir not found"
            echo ""
            ls -la docs/_static/websocket/conformance-$nvx_config/servers/ || echo "servers dir not found"
            echo ""

            python3 ${{ github.workspace }}/.github/workflows/verify_conformance.py \
              "docs/_static/websocket/conformance-$nvx_config/clients/index.json" \
              "docs/_static/websocket/conformance-$nvx_config/servers/index.json"

            echo "✅ Verification passed for $nvx_config"
          done

          echo ""
          echo "======================================================================"
          echo "✅ All NVX configurations passed 100% conformance verification!"
          echo "======================================================================"