part 4b rel v25.9.1 #152
Workflow file for this run
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: wstest | |
| on: | |
| push: | |
| branches: [master] | |
| pull_request: | |
| branches: [master] | |
| workflow_dispatch: | |
| inputs: | |
| test_mode: | |
| description: "Test mode to run" | |
| required: false | |
| default: "quick" | |
| type: choice | |
| options: | |
| - quick | |
| - full | |
| env: | |
| UV_CACHE_DIR: ${{ github.workspace }}/.uv-cache | |
| # Test mode: 'quick' for fast CI, 'full' for comprehensive testing | |
| TEST_MODE: ${{ github.event.inputs.test_mode || 'quick' }} | |
| jobs: | |
| identifiers: | |
| # GitHub needs to know where .cicd/workflows/identifiers.yml lives at parse time, | |
| # and submodules aren't included in that context! thus the following does NOT work: | |
| # uses: ./.cicd/workflows/identifiers.yml | |
| # we MUST reference the remote repo directly: | |
| uses: wamp-proto/wamp-cicd/.github/workflows/identifiers.yml@main | |
| # IMPORTANT: we still need .cicd as a Git submodule in the using repo though! | |
| # because e.g. identifiers.yml wants to access scripts/sanitize.sh ! | |
| # Client testing: Test autobahn-python clients against testsuite server | |
| client-conformance: | |
| name: WebSocket Client Testing (All Combinations) | |
| needs: identifiers | |
| runs-on: ubuntu-24.04 | |
| env: | |
| BASE_REPO: ${{ needs.identifiers.outputs.base_repo }} | |
| BASE_BRANCH: ${{ needs.identifiers.outputs.base_branch }} | |
| PR_NUMBER: ${{ needs.identifiers.outputs.pr_number }} | |
| PR_REPO: ${{ needs.identifiers.outputs.pr_repo }} | |
| PR_BRANCH: ${{ needs.identifiers.outputs.pr_branch }} | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| with: | |
| submodules: recursive | |
| - name: Install toolchain (just & uv) | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| run: | | |
| curl --proto '=https' --tlsv1.2 -sSf https://just.systems/install.sh | bash -s -- --to ~/bin | |
| curl -LsSf https://astral.sh/uv/install.sh | sh | |
| echo "$HOME/bin" >> $GITHUB_PATH | |
| echo "$HOME/.cargo/bin" >> $GITHUB_PATH | |
| - name: Setup uv cache | |
| uses: actions/cache@v4 | |
| with: | |
| path: ${{ env.UV_CACHE_DIR }} | |
| key: | |
| uv-cache-clients-${{ hashFiles('pyproject.toml') }} | |
| - name: | |
| Setup Python environments and install Autobahn|Python | |
| run: | | |
| just install cpy311 | |
| just install cpy314 | |
| just install pypy311 | |
| - name: Start fuzzingserver | |
| run: | | |
| echo "==> Starting fuzzingserver in background..." | |
| nohup just wstest-fuzzingserver "" "" ${{ env.TEST_MODE }} > fuzzingserver.log 2>&1 & | |
| echo $! > fuzzingserver.pid | |
| echo "==> Waiting for fuzzingserver to start..." | |
| for i in {1..30}; do | |
| if curl -f http://127.0.0.1:9001 >/dev/null 2>&1; then | |
| echo "✅ Fuzzingserver is running after ${i} seconds" | |
| break | |
| fi | |
| if [ $i -eq 30 ]; then | |
| echo "❌ Fuzzingserver failed to start after 30 seconds" | |
| echo "==> Fuzzingserver log:" | |
| cat fuzzingserver.log | |
| exit 1 | |
| fi | |
| sleep 1 | |
| done | |
| - name: Run all client test combinations | |
| run: | | |
| echo "==> Running client tests for all 6 combinations..." | |
| # Client 1: Twisted + CPython 3.11 | |
| echo "==> Testing Twisted + cpy311 client..." | |
| just wstest-testeeclient-twisted cpy311 | |
| # Client 2: asyncio + CPython 3.11 | |
| echo "==> Testing asyncio + cpy311 client..." | |
| just wstest-testeeclient-asyncio cpy311 | |
| # Client 3: Twisted + CPython 3.14 | |
| echo "==> Testing Twisted + cpy314 client..." | |
| just wstest-testeeclient-twisted cpy314 | |
| # Client 4: asyncio + CPython 3.14 | |
| echo "==> Testing asyncio + cpy314 client..." | |
| just wstest-testeeclient-asyncio cpy314 | |
| # Client 5: Twisted + PyPy 3.11 | |
| echo "==> Testing Twisted + pypy311 client..." | |
| just wstest-testeeclient-twisted pypy311 | |
| # Client 6: asyncio + PyPy 3.11 | |
| echo "==> Testing asyncio + pypy311 client..." | |
| just wstest-testeeclient-asyncio pypy311 | |
| echo "✅ All 6 client test combinations completed" | |
| - name: Stop fuzzingserver | |
| if: always() | |
| run: | | |
| echo "==> Stopping fuzzingserver..." | |
| if [ -f fuzzingserver.pid ]; then | |
| kill $(cat fuzzingserver.pid) || true | |
| fi | |
| docker stop fuzzingserver || true | |
| - name: Upload client reports | |
| uses: actions/upload-artifact@v4 | |
| if: always() | |
| with: | |
| name: clients-all-${{ env.TEST_MODE }} | |
| path: .wstest/clients/ | |
| retention-days: 30 | |
| # Server testing: Test all server combinations against testsuite client | |
| server-conformance: | |
| name: WebSocket Server Testing (All Combinations) | |
| needs: identifiers | |
| runs-on: ubuntu-24.04 | |
| env: | |
| BASE_REPO: ${{ needs.identifiers.outputs.base_repo }} | |
| BASE_BRANCH: ${{ needs.identifiers.outputs.base_branch }} | |
| PR_NUMBER: ${{ needs.identifiers.outputs.pr_number }} | |
| PR_REPO: ${{ needs.identifiers.outputs.pr_repo }} | |
| PR_BRANCH: ${{ needs.identifiers.outputs.pr_branch }} | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| with: | |
| submodules: recursive | |
| - name: Install toolchain (just & uv) | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| run: | | |
| curl --proto '=https' --tlsv1.2 -sSf https://just.systems/install.sh | bash -s -- --to ~/bin | |
| curl -LsSf https://astral.sh/uv/install.sh | sh | |
| echo "$HOME/bin" >> $GITHUB_PATH | |
| echo "$HOME/.cargo/bin" >> $GITHUB_PATH | |
| - name: Setup uv cache | |
| uses: actions/cache@v4 | |
| with: | |
| path: ${{ env.UV_CACHE_DIR }} | |
| key: | |
| uv-cache-servers-${{ hashFiles('pyproject.toml') }} | |
| - name: | |
| Setup Python environments and install Autobahn|Python | |
| run: | | |
| just install cpy311 | |
| just install cpy314 | |
| just install pypy311 | |
| - name: Start all WebSocket servers | |
| run: | | |
| echo "==> Starting 6 WebSocket server combinations sequentially..." | |
| # Server 1: Twisted + CPython 3.11 on port 9011 | |
| echo "==> Starting Twisted + cpy311 server on port 9011..." | |
| nohup .venvs/cpy311/bin/python ./wstest/testee_server_tx.py --url ws://127.0.0.1:9011 > twisted-cpy311.log 2>&1 & | |
| for i in {1..30}; do | |
| if netstat -tuln | grep :9011 >/dev/null 2>&1; then | |
| echo "✅ Twisted + cpy311 server running on port 9011" | |
| break | |
| fi | |
| if [ $i -eq 30 ]; then | |
| echo "❌ Twisted + cpy311 server failed to start on port 9011" | |
| cat twisted-cpy311.log | |
| exit 1 | |
| fi | |
| sleep 1 | |
| done | |
| # Server 2: asyncio + CPython 3.11 on port 9012 | |
| echo "==> Starting asyncio + cpy311 server on port 9012..." | |
| nohup .venvs/cpy311/bin/python ./wstest/testee_server_aio.py --url ws://127.0.0.1:9012 > asyncio-cpy311.log 2>&1 & | |
| for i in {1..30}; do | |
| if netstat -tuln | grep :9012 >/dev/null 2>&1; then | |
| echo "✅ asyncio + cpy311 server running on port 9012" | |
| break | |
| fi | |
| if [ $i -eq 30 ]; then | |
| echo "❌ asyncio + cpy311 server failed to start on port 9012" | |
| cat asyncio-cpy311.log | |
| exit 1 | |
| fi | |
| sleep 1 | |
| done | |
| # Server 3: Twisted + CPython 3.14 on port 9013 | |
| echo "==> Starting Twisted + cpy314 server on port 9013..." | |
| nohup .venvs/cpy314/bin/python ./wstest/testee_server_tx.py --url ws://127.0.0.1:9013 > twisted-cpy314.log 2>&1 & | |
| for i in {1..30}; do | |
| if netstat -tuln | grep :9013 >/dev/null 2>&1; then | |
| echo "✅ Twisted + cpy314 server running on port 9013" | |
| break | |
| fi | |
| if [ $i -eq 30 ]; then | |
| echo "❌ Twisted + cpy314 server failed to start on port 9013" | |
| cat twisted-cpy314.log | |
| exit 1 | |
| fi | |
| sleep 1 | |
| done | |
| # Server 4: asyncio + CPython 3.14 on port 9014 | |
| echo "==> Starting asyncio + cpy314 server on port 9014..." | |
| nohup .venvs/cpy314/bin/python ./wstest/testee_server_aio.py --url ws://127.0.0.1:9014 > asyncio-cpy314.log 2>&1 & | |
| for i in {1..30}; do | |
| if netstat -tuln | grep :9014 >/dev/null 2>&1; then | |
| echo "✅ asyncio + cpy314 server running on port 9014" | |
| break | |
| fi | |
| if [ $i -eq 30 ]; then | |
| echo "❌ asyncio + cpy314 server failed to start on port 9014" | |
| cat asyncio-cpy314.log | |
| exit 1 | |
| fi | |
| sleep 1 | |
| done | |
| # Server 5: Twisted + PyPy 3.11 on port 9015 | |
| echo "==> Starting Twisted + pypy311 server on port 9015..." | |
| nohup .venvs/pypy311/bin/python ./wstest/testee_server_tx.py --url ws://127.0.0.1:9015 > twisted-pypy311.log 2>&1 & | |
| for i in {1..30}; do | |
| if netstat -tuln | grep :9015 >/dev/null 2>&1; then | |
| echo "✅ Twisted + pypy311 server running on port 9015" | |
| break | |
| fi | |
| if [ $i -eq 30 ]; then | |
| echo "❌ Twisted + pypy311 server failed to start on port 9015" | |
| cat twisted-pypy311.log | |
| exit 1 | |
| fi | |
| sleep 1 | |
| done | |
| # Server 6: asyncio + PyPy 3.11 on port 9016 | |
| echo "==> Starting asyncio + pypy311 server on port 9016..." | |
| nohup .venvs/pypy311/bin/python ./wstest/testee_server_aio.py --url ws://127.0.0.1:9016 > asyncio-pypy311.log 2>&1 & | |
| for i in {1..30}; do | |
| if netstat -tuln | grep :9016 >/dev/null 2>&1; then | |
| echo "✅ asyncio + pypy311 server running on port 9016" | |
| break | |
| fi | |
| if [ $i -eq 30 ]; then | |
| echo "❌ asyncio + pypy311 server failed to start on port 9016" | |
| cat asyncio-pypy311.log | |
| exit 1 | |
| fi | |
| sleep 1 | |
| done | |
| echo "✅ All 6 WebSocket servers are running successfully" | |
| - name: Run server tests | |
| run: just wstest-fuzzingclient "" "" ${{ env.TEST_MODE }} | |
| - name: Upload server reports | |
| uses: actions/upload-artifact@v4 | |
| if: always() | |
| with: | |
| name: servers-all-${{ env.TEST_MODE }} | |
| path: .wstest/servers/ | |
| retention-days: 30 | |
| # Consolidate all reports using the proven justfile recipe | |
| consolidate-reports: | |
| name: Consolidate WebSocket Reports | |
| needs: [identifiers, client-conformance, server-conformance] | |
| runs-on: ubuntu-24.04 | |
| if: always() | |
| env: | |
| BASE_REPO: ${{ needs.identifiers.outputs.base_repo }} | |
| BASE_BRANCH: ${{ needs.identifiers.outputs.base_branch }} | |
| PR_NUMBER: ${{ needs.identifiers.outputs.pr_number }} | |
| PR_REPO: ${{ needs.identifiers.outputs.pr_repo }} | |
| PR_BRANCH: ${{ needs.identifiers.outputs.pr_branch }} | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| with: | |
| submodules: recursive | |
| - name: Install Just | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| run: | | |
| curl --proto '=https' --tlsv1.2 -sSf https://just.systems/install.sh | bash -s -- --to ~/bin | |
| echo "$HOME/bin" >> $GITHUB_PATH | |
| - name: Download all artifacts | |
| uses: actions/download-artifact@v4 | |
| with: | |
| path: artifacts/ | |
| - name: Reconstruct test directory structure | |
| run: | | |
| echo "==> Reconstructing .wstest directory structure..." | |
| mkdir -p .wstest/clients .wstest/servers | |
| # Debug: Show what artifacts we actually have | |
| echo "==> Available artifacts:" | |
| ls -la artifacts/ || echo "No artifacts directory" | |
| # Combine all client reports (now using clients-all naming) | |
| find artifacts -name "clients-all-*" -type d | while read dir; do | |
| echo "Processing client reports: $dir" | |
| cp -r "$dir"/* .wstest/clients/ || true | |
| done | |
| # Copy server reports (servers-all naming) | |
| find artifacts -name "servers-all-*" -type d | while read dir; do | |
| echo "Processing server reports: $dir" | |
| cp -r "$dir"/* .wstest/servers/ || true | |
| done | |
| echo "==> Reconstructed client reports:" | |
| ls -la .wstest/clients/ || echo "None" | |
| echo "" | |
| echo "==> Reconstructed server reports:" | |
| ls -la .wstest/servers/ || echo "None" | |
| - name: Consolidate reports for documentation | |
| run: | | |
| echo "==> Using proven justfile recipe..." | |
| just wstest-consolidate-reports | |
| - name: Generate summary tables from index.json files | |
| run: | | |
| echo "==> Generating conformance summary tables..." | |
| # Create unique filename for PR using -- as separator | |
| REPO_SAFE=$(echo '${{ github.event.pull_request.head.repo.full_name }}' | tr '/' '--') | |
| SUMMARY_FILE="${{ github.workspace }}/.audit/${REPO_SAFE}--${{ github.head_ref }}--${{ github.event.number }}--wstest-summary.md" | |
| echo "Summary file: $SUMMARY_FILE" | |
| # Generate summary for clients and servers | |
| echo "# WebSocket Conformance Test Results" > "$SUMMARY_FILE" | |
| echo "" >> "$SUMMARY_FILE" | |
| echo "Test Mode: \`${{ env.TEST_MODE }}\`" >> "$SUMMARY_FILE" | |
| echo "Repo: ${{ github.event.pull_request.head.repo.full_name }}" >> "$SUMMARY_FILE" | |
| echo "Branch: ${{ github.head_ref }}" >> "$SUMMARY_FILE" | |
| echo "PR: #${{ github.event.number }}" >> "$SUMMARY_FILE" | |
| # Generate client summary | |
| python3 ${{ github.workspace }}/.github/workflows/generate_summary.py \ | |
| "docs/_static/websocket/conformance/clients/index.json" \ | |
| "Client Conformance" >> "$SUMMARY_FILE" | |
| # Generate server summary | |
| python3 ${{ github.workspace }}/.github/workflows/generate_summary.py \ | |
| "docs/_static/websocket/conformance/servers/index.json" \ | |
| "Server Conformance" >> "$SUMMARY_FILE" | |
| # Show summary in workflow log | |
| echo "==> Summary Report:" | |
| cat "$SUMMARY_FILE" | |
| # Store filename for next step | |
| echo "SUMMARY_FILE=$SUMMARY_FILE" >> $GITHUB_ENV | |
| - name: Upload consolidated reports | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: websocket-conformance-docs-${{ env.TEST_MODE }} | |
| path: docs/_static/websocket/conformance/ | |
| retention-days: 30 | |
| - name: Upload summary report for PR comment | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: conformance-summary-${{ env.TEST_MODE }} | |
| path: ${{ env.SUMMARY_FILE }} | |
| retention-days: 30 | |
| - name: Report summary | |
| run: | | |
| echo "==> WebSocket Conformance Testing Complete ✅" | |
| echo "Test Mode: ${{ env.TEST_MODE }}" | |
| echo "Matrix tested: (client|server) × (twisted|asyncio) × (cpy311|cpy314|pypy311)" | |
| echo "" | |
| find docs/_static/websocket/conformance -name "*.zip" | while read zip; do | |
| echo "📦 $(basename "$zip")" | |
| done | |
| echo "📄 Documentation: docs/_static/websocket/conformance/" | |
| echo "📊 Summary: summary.md" | |
| # Verify all tests passed (100% conformance) - fails if any test failed | |
| verify-reports: | |
| name: Verify WebSocket Conformance (100% Pass Required) | |
| needs: [identifiers, consolidate-reports] | |
| runs-on: ubuntu-24.04 | |
| if: always() | |
| env: | |
| BASE_REPO: ${{ needs.identifiers.outputs.base_repo }} | |
| BASE_BRANCH: ${{ needs.identifiers.outputs.base_branch }} | |
| PR_NUMBER: ${{ needs.identifiers.outputs.pr_number }} | |
| PR_REPO: ${{ needs.identifiers.outputs.pr_repo }} | |
| PR_BRANCH: ${{ needs.identifiers.outputs.pr_branch }} | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| with: | |
| submodules: recursive | |
| - name: Download consolidated reports | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: websocket-conformance-docs-${{ env.TEST_MODE }} | |
| path: docs/_static/websocket/conformance/ | |
| - name: Verify 100% conformance | |
| run: | | |
| echo "==> Debugging directory structure:" | |
| find docs/_static/websocket/conformance -type f -name "*.json" | head -10 | |
| echo "" | |
| ls -la docs/_static/websocket/conformance/ || echo "conformance dir not found" | |
| echo "" | |
| ls -la docs/_static/websocket/conformance/clients/ || echo "clients dir not found" | |
| echo "" | |
| ls -la docs/_static/websocket/conformance/servers/ || echo "servers dir not found" | |
| echo "" | |
| python3 ${{ github.workspace }}/.github/workflows/verify_conformance.py \ | |
| "docs/_static/websocket/conformance/clients/index.json" \ | |
| "docs/_static/websocket/conformance/servers/index.json" |