diff --git a/scripts/windows-debug/check-tee-command.ps1 b/scripts/windows-debug/check-tee-command.ps1 new file mode 100644 index 000000000..46fa7f093 --- /dev/null +++ b/scripts/windows-debug/check-tee-command.ps1 @@ -0,0 +1,142 @@ +#!/usr/bin/env pwsh +# Script to check if tee command is available on Windows +# Usage: .\check-tee-command.ps1 + +Write-Host "Checking for 'tee' command availability on Windows..." -ForegroundColor Cyan +Write-Host "" + +# Store original PATH +$originalPath = $env:PATH + +# Method 1: Using where.exe +Write-Host "Method 1: Using where.exe" -ForegroundColor Yellow +try { + $whereResult = where.exe tee 2>&1 + if ($LASTEXITCODE -eq 0) { + Write-Host " Found tee at: $whereResult" -ForegroundColor Green + } else { + Write-Host " tee not found via where.exe" -ForegroundColor Red + } +} catch { + Write-Host " Error checking with where.exe: $_" -ForegroundColor Red +} + +Write-Host "" + +# Method 2: Using Get-Command +Write-Host "Method 2: Using Get-Command" -ForegroundColor Yellow +try { + $getCommandResult = Get-Command tee -ErrorAction Stop + Write-Host " Found tee:" -ForegroundColor Green + Write-Host " Name: $($getCommandResult.Name)" + Write-Host " CommandType: $($getCommandResult.CommandType)" + Write-Host " Source: $($getCommandResult.Source)" + Write-Host " Version: $($getCommandResult.Version)" +} catch { + Write-Host " tee not found via Get-Command" -ForegroundColor Red +} + +Write-Host "" + +# Method 3: Check common locations +Write-Host "Method 3: Checking common locations" -ForegroundColor Yellow +$commonPaths = @( + "C:\Program Files\Git\usr\bin\tee.exe", + "C:\Program Files (x86)\Git\usr\bin\tee.exe", + "C:\tools\msys64\usr\bin\tee.exe", + "C:\msys64\usr\bin\tee.exe", + "C:\cygwin64\bin\tee.exe", + "C:\cygwin\bin\tee.exe" +) + +$found = $false +foreach ($path in $commonPaths) { + if (Test-Path $path) { + Write-Host " Found at: $path" -ForegroundColor Green + $found = $true + } +} + +if (-not $found) { + Write-Host " tee not found in common locations" -ForegroundColor Red +} + +Write-Host "" + +# Method 4: Check if it's a PowerShell alias +Write-Host "Method 4: Checking PowerShell aliases" -ForegroundColor Yellow +$alias = Get-Alias tee -ErrorAction SilentlyContinue +if ($alias) { + Write-Host " Found PowerShell alias:" -ForegroundColor Green + Write-Host " Name: $($alias.Name)" + Write-Host " Definition: $($alias.Definition)" +} else { + Write-Host " No PowerShell alias for tee" -ForegroundColor Yellow +} + +Write-Host "" + +# Method 5: Python check (what the test uses) +Write-Host "Method 5: Python shutil.which() check" -ForegroundColor Yellow +$pythonCheck = python -c "import shutil; print(shutil.which('tee'))" +if ($pythonCheck -and $pythonCheck -ne "None") { + Write-Host " Python found tee at: $pythonCheck" -ForegroundColor Green +} else { + Write-Host " Python shutil.which() did not find tee" -ForegroundColor Red +} + +Write-Host "" + +# Method 6: Try adding Git for Windows to PATH if it exists +Write-Host "Method 6: Adding Git for Windows to PATH temporarily" -ForegroundColor Yellow +$gitPaths = @( + "C:\Program Files\Git\usr\bin", + "C:\Program Files (x86)\Git\usr\bin" +) + +$addedToPath = $false +foreach ($gitPath in $gitPaths) { + if (Test-Path $gitPath) { + Write-Host " Found Git directory: $gitPath" -ForegroundColor Green + $env:PATH = "$gitPath;$env:PATH" + $teeCheck = python -c "import shutil; print(shutil.which('tee'))" + if ($teeCheck -and $teeCheck -ne "None") { + Write-Host " tee is now available at: $teeCheck" -ForegroundColor Green + $addedToPath = $true + break + } + } +} + +if (-not $addedToPath) { + # Restore original PATH if we didn't find tee + $env:PATH = $originalPath + Write-Host " Could not add Git for Windows tee to PATH" -ForegroundColor Red +} + +Write-Host "" +Write-Host "========== SUMMARY ==========" -ForegroundColor Cyan +if ($whereResult -or $getCommandResult -or $found -or ($pythonCheck -and $pythonCheck -ne "None") -or $addedToPath) { + Write-Host "tee command is available" -ForegroundColor Green + Write-Host "" + Write-Host "The test_stdio_context_manager_exiting test should run." -ForegroundColor Green + if ($addedToPath) { + Write-Host "" + Write-Host "Note: Git for Windows tee was added to PATH for this session." -ForegroundColor Yellow + Write-Host "To make this permanent, add this to your PowerShell profile:" -ForegroundColor Yellow + Write-Host " `$env:PATH = `"C:\Program Files\Git\usr\bin;`$env:PATH`"" -ForegroundColor Cyan + } +} else { + Write-Host "tee command is NOT available" -ForegroundColor Red + Write-Host "" + Write-Host "The test_stdio_context_manager_exiting test will be SKIPPED." -ForegroundColor Yellow + Write-Host "" + Write-Host "To install tee on Windows, you can:" -ForegroundColor Cyan + Write-Host " 1. Install Git for Windows (includes tee in Git Bash)" + Write-Host " 2. Install WSL (Windows Subsystem for Linux)" + Write-Host " 3. Install MSYS2 or Cygwin" + Write-Host " 4. Use PowerShell's Tee-Object cmdlet (different syntax)" +} + +# Restore original PATH +$env:PATH = $originalPath \ No newline at end of file diff --git a/scripts/windows-debug/diagnose-environment.ps1 b/scripts/windows-debug/diagnose-environment.ps1 new file mode 100644 index 000000000..5d49480b2 --- /dev/null +++ b/scripts/windows-debug/diagnose-environment.ps1 @@ -0,0 +1,152 @@ +#!/usr/bin/env pwsh +# Script to diagnose environment differences that might cause CI flakiness +# Usage: .\diagnose-environment.ps1 + +Write-Host "Diagnosing Windows environment for stdio test issues..." -ForegroundColor Cyan +Write-Host "" + +# System Information +Write-Host "=== SYSTEM INFORMATION ===" -ForegroundColor Yellow +Write-Host "Windows Version:" +(Get-CimInstance Win32_OperatingSystem).Version +Write-Host "Windows Build:" +(Get-ItemProperty "HKLM:\SOFTWARE\Microsoft\Windows NT\CurrentVersion").DisplayVersion +Write-Host "" + +# Python Information +Write-Host "=== PYTHON INFORMATION ===" -ForegroundColor Yellow +Write-Host "Python Version:" +python --version +Write-Host "" +Write-Host "Python Build Info:" +python -c "import sys; print(f'Version: {sys.version}')" +python -c "import sys; print(f'Platform: {sys.platform}')" +python -c "import sys; print(f'Windows Version: {sys.getwindowsversion()}')" +Write-Host "" + +# Check subprocess configuration +Write-Host "=== SUBPROCESS CONFIGURATION ===" -ForegroundColor Yellow +python -c @" +import subprocess +import sys +print(f'CREATE_NO_WINDOW available: {hasattr(subprocess, "CREATE_NO_WINDOW")}') +print(f'CREATE_NEW_PROCESS_GROUP available: {hasattr(subprocess, "CREATE_NEW_PROCESS_GROUP")}') +print(f'Windows subprocess startup info: {hasattr(subprocess, "STARTUPINFO")}') + +# Check handle inheritance defaults +import os +print(f'\nHandle inheritance (os.O_NOINHERIT): {hasattr(os, "O_NOINHERIT")}') + +# Check asyncio event loop +import asyncio +try: + loop = asyncio.get_event_loop_policy() + print(f'Event loop policy: {type(loop).__name__}') +except: + print('Event loop policy: Unable to determine') +"@ +Write-Host "" + +# Check for Job Objects support +Write-Host "=== JOB OBJECTS SUPPORT ===" -ForegroundColor Yellow +python -c @" +try: + import win32job + import win32api + print('pywin32 available: Yes') + print(f'win32job version: {win32job.__file__}') + + # Try to create a job object + try: + job = win32job.CreateJobObject(None, '') + win32api.CloseHandle(job) + print('Job Object creation: Success') + except Exception as e: + print(f'Job Object creation: Failed - {e}') +except ImportError: + print('pywin32 available: No (Job Objects not available)') +"@ +Write-Host "" + +# Check process limits +Write-Host "=== PROCESS LIMITS ===" -ForegroundColor Yellow +python -c @" +import os +import psutil +proc = psutil.Process(os.getpid()) +print(f'Open handles: {proc.num_handles()}') +print(f'Open files: {len(proc.open_files())}') + +# Check system-wide limits +print(f'Total processes: {len(psutil.pids())}') +"@ +Write-Host "" + +# Check security software +Write-Host "=== SECURITY SOFTWARE ===" -ForegroundColor Yellow +Get-CimInstance -Namespace "root\SecurityCenter2" -ClassName AntiVirusProduct -ErrorAction SilentlyContinue | + Select-Object displayName, productState | Format-Table +Write-Host "" + +# Test rapid process creation +Write-Host "=== RAPID PROCESS CREATION TEST ===" -ForegroundColor Yellow +Write-Host "Testing rapid tee process creation/destruction..." +$testScript = @' +import time +import subprocess +import sys + +failures = 0 +times = [] + +for i in range(20): + start = time.time() + try: + # Create process with same flags as stdio client + proc = subprocess.Popen( + ['tee'], + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + creationflags=getattr(subprocess, 'CREATE_NO_WINDOW', 0) + ) + proc.stdin.close() + proc.wait(timeout=0.5) + elapsed = time.time() - start + times.append(elapsed) + except Exception as e: + failures += 1 + print(f" Iteration {i+1}: FAILED - {e}") + + if (i+1) % 5 == 0: + avg_time = sum(times) / len(times) if times else 0 + print(f" Completed {i+1}/20 (avg: {avg_time*1000:.1f}ms)") + +print(f"\nFailures: {failures}/20") +if times: + print(f"Average time: {sum(times)/len(times)*1000:.1f}ms") + print(f"Max time: {max(times)*1000:.1f}ms") +'@ + +python -c $testScript +Write-Host "" + +# Environment variables that might affect subprocess +Write-Host "=== RELEVANT ENVIRONMENT VARIABLES ===" -ForegroundColor Yellow +@("COMSPEC", "PATH", "PYTHONPATH", "PYTHONASYNCIODEBUG") | ForEach-Object { + $value = [Environment]::GetEnvironmentVariable($_) + if ($value) { + Write-Host "$_`:" + Write-Host " $value" -ForegroundColor Gray + } +} +Write-Host "" + +Write-Host "=== DIAGNOSIS COMPLETE ===" -ForegroundColor Cyan +Write-Host "" +Write-Host "Share this output when reporting the flakiness issue." -ForegroundColor Yellow +Write-Host "Key differences to look for between local and CI:" -ForegroundColor Yellow +Write-Host " - Windows version/build" -ForegroundColor Gray +Write-Host " - Python build details" -ForegroundColor Gray +Write-Host " - Security software presence" -ForegroundColor Gray +Write-Host " - Process creation timing" -ForegroundColor Gray \ No newline at end of file diff --git a/scripts/windows-debug/setup-environment.ps1 b/scripts/windows-debug/setup-environment.ps1 new file mode 100644 index 000000000..25dd44e27 --- /dev/null +++ b/scripts/windows-debug/setup-environment.ps1 @@ -0,0 +1,59 @@ +#!/usr/bin/env pwsh +# Script to set up the environment for running stdio tests on Windows +# This adds Git for Windows tools to PATH if available +# Usage: . .\setup-environment.ps1 (note the dot-sourcing) + +Write-Host "Setting up environment for stdio tests..." -ForegroundColor Cyan +Write-Host "" + +# Check if Git for Windows is installed +$gitPaths = @( + "C:\Program Files\Git\usr\bin", + "C:\Program Files (x86)\Git\usr\bin" +) + +$gitFound = $false +$gitPath = "" + +foreach ($path in $gitPaths) { + if (Test-Path $path) { + $gitPath = $path + $gitFound = $true + break + } +} + +if ($gitFound) { + Write-Host "Found Git for Windows at: $gitPath" -ForegroundColor Green + + # Add to PATH + $env:PATH = "$gitPath;$env:PATH" + + # Verify tee is available + $teeCheck = python -c "import shutil; print(shutil.which('tee'))" + if ($teeCheck -and $teeCheck -ne "None") { + Write-Host "Successfully added tee to PATH: $teeCheck" -ForegroundColor Green + Write-Host "" + Write-Host "Environment is ready for stdio tests!" -ForegroundColor Green + Write-Host "" + Write-Host "You can now run the test scripts or individual tests:" -ForegroundColor Cyan + Write-Host " .\test-stdio-flakiness-200-runs.ps1" + Write-Host " .\test-stdio-flakiness-until-failure.ps1" + Write-Host " .\test-stdio-verbose-debug.ps1" + Write-Host "" + Write-Host "Or run individual tests with:" -ForegroundColor Cyan + Write-Host " uv run pytest tests/client/test_stdio.py::test_stdio_context_manager_exiting -v -o addopts=""""" + } else { + Write-Host "WARNING: Git path was added but tee is still not available" -ForegroundColor Yellow + } +} else { + Write-Host "Git for Windows not found in standard locations." -ForegroundColor Red + Write-Host "" + Write-Host "Please install Git for Windows from: https://gitforwindows.org/" -ForegroundColor Yellow + Write-Host "Or manually add the Git usr\bin directory to your PATH." -ForegroundColor Yellow +} + +Write-Host "" +Write-Host "Note: This only affects the current PowerShell session." -ForegroundColor Gray +Write-Host "To make changes permanent, add to your PowerShell profile:" -ForegroundColor Gray +Write-Host ' $env:PATH = "C:\Program Files\Git\usr\bin;$env:PATH"' -ForegroundColor Cyan \ No newline at end of file diff --git a/scripts/windows-debug/test-stdio-flakiness-200-runs.ps1 b/scripts/windows-debug/test-stdio-flakiness-200-runs.ps1 new file mode 100644 index 000000000..26bdddd6b --- /dev/null +++ b/scripts/windows-debug/test-stdio-flakiness-200-runs.ps1 @@ -0,0 +1,55 @@ +#!/usr/bin/env pwsh +# Script to run test_stdio_context_manager_exiting 200 times to detect flakiness +# Usage: .\test-stdio-flakiness-200-runs.ps1 +# +# Prerequisites: Run . .\setup-environment.ps1 first to ensure tee is available + +Write-Host "Running test_stdio_context_manager_exiting 200 times to detect flakiness..." -ForegroundColor Cyan +Write-Host "Test: tests/client/test_stdio.py::test_stdio_context_manager_exiting" -ForegroundColor Yellow +Write-Host "" + +# Check if tee is available +$teeCheck = python -c "import shutil; print(shutil.which('tee'))" +if (-not $teeCheck -or $teeCheck -eq "None") { + Write-Host "ERROR: tee command not found!" -ForegroundColor Red + Write-Host "Please run: . .\setup-environment.ps1" -ForegroundColor Yellow + Write-Host "(Note the dot at the beginning to source the script)" -ForegroundColor Yellow + exit 1 +} + +$startTime = Get-Date +$count = 0 +$failures = 0 +$failedRuns = @() + +for ($i = 1; $i -le 200; $i++) { + Write-Host "Run $i of 200..." -NoNewline + + $output = uv run --frozen pytest tests/client/test_stdio.py::test_stdio_context_manager_exiting -xvs -o addopts="" 2>&1 + $exitCode = $LASTEXITCODE + + if ($exitCode -ne 0) { + $failures++ + $failedRuns += $i + Write-Host " FAILED" -ForegroundColor Red + Write-Host "Failure output:" -ForegroundColor Red + Write-Host $output + Write-Host "" + } else { + Write-Host " PASSED" -ForegroundColor Green + } +} + +$endTime = Get-Date +$duration = $endTime - $startTime + +Write-Host "" +Write-Host "========== SUMMARY ==========" -ForegroundColor Cyan +Write-Host "Total runs: 200" +Write-Host "Successful runs: $(200 - $failures)" -ForegroundColor Green +Write-Host "Failed runs: $failures" -ForegroundColor Red +if ($failures -gt 0) { + Write-Host "Failed on runs: $($failedRuns -join ', ')" -ForegroundColor Red +} +Write-Host "Duration: $($duration.ToString())" +Write-Host "Failure rate: $([math]::Round(($failures / 200) * 100, 2))%" \ No newline at end of file diff --git a/scripts/windows-debug/test-stdio-flakiness-until-failure.ps1 b/scripts/windows-debug/test-stdio-flakiness-until-failure.ps1 new file mode 100644 index 000000000..1a9df0e40 --- /dev/null +++ b/scripts/windows-debug/test-stdio-flakiness-until-failure.ps1 @@ -0,0 +1,53 @@ +#!/usr/bin/env pwsh +# Script to run test_stdio_context_manager_exiting until it fails +# Usage: .\test-stdio-flakiness-until-failure.ps1 +# +# Prerequisites: Run . .\setup-environment.ps1 first to ensure tee is available + +Write-Host "Running test_stdio_context_manager_exiting until failure..." -ForegroundColor Cyan +Write-Host "Test: tests/client/test_stdio.py::test_stdio_context_manager_exiting" -ForegroundColor Yellow +Write-Host "Press Ctrl+C to stop" -ForegroundColor Gray +Write-Host "" + +# Check if tee is available +$teeCheck = python -c "import shutil; print(shutil.which('tee'))" +if (-not $teeCheck -or $teeCheck -eq "None") { + Write-Host "ERROR: tee command not found!" -ForegroundColor Red + Write-Host "Please run: . .\setup-environment.ps1" -ForegroundColor Yellow + Write-Host "(Note the dot at the beginning to source the script)" -ForegroundColor Yellow + exit 1 +} + +$startTime = Get-Date +$i = 0 + +while ($true) { + $i++ + Write-Host "Run $i..." -NoNewline + + $output = uv run --frozen pytest tests/client/test_stdio.py::test_stdio_context_manager_exiting -xvs -o addopts="" 2>&1 + $exitCode = $LASTEXITCODE + + if ($exitCode -ne 0) { + $endTime = Get-Date + $duration = $endTime - $startTime + + Write-Host " FAILED!" -ForegroundColor Red + Write-Host "" + Write-Host "========== FAILURE DETECTED ==========" -ForegroundColor Red + Write-Host "Failed on run: $i" -ForegroundColor Red + Write-Host "Time until failure: $($duration.ToString())" -ForegroundColor Yellow + Write-Host "" + Write-Host "Failure output:" -ForegroundColor Red + Write-Host $output + break + } else { + Write-Host " PASSED" -ForegroundColor Green + } + + # Small delay to prevent overwhelming the system + Start-Sleep -Milliseconds 100 +} + +Write-Host "" +Write-Host "Exiting after failure detection." -ForegroundColor Cyan \ No newline at end of file diff --git a/scripts/windows-debug/test-stdio-parallel-flakiness.ps1 b/scripts/windows-debug/test-stdio-parallel-flakiness.ps1 new file mode 100644 index 000000000..af3280175 --- /dev/null +++ b/scripts/windows-debug/test-stdio-parallel-flakiness.ps1 @@ -0,0 +1,94 @@ +#!/usr/bin/env pwsh +# Script to test for flakiness when running tests in parallel (like CI does) +# This simulates the xdist environment where the issue occurs +# Usage: .\test-stdio-parallel-flakiness.ps1 +# +# Prerequisites: Run . .\setup-environment.ps1 first to ensure tee is available + +Write-Host "Testing stdio with parallel execution to simulate CI environment..." -ForegroundColor Cyan +Write-Host "" + +# Check if tee is available +$teeCheck = python -c "import shutil; print(shutil.which('tee'))" +if (-not $teeCheck -or $teeCheck -eq "None") { + Write-Host "ERROR: tee command not found!" -ForegroundColor Red + Write-Host "Please run: . .\setup-environment.ps1" -ForegroundColor Yellow + Write-Host "(Note the dot at the beginning to source the script)" -ForegroundColor Yellow + exit 1 +} + +Write-Host "Running tests with different parallel configurations..." -ForegroundColor Yellow +Write-Host "" + +# Test 1: Run with 4 workers (default CI behavior) +Write-Host "Test 1: Running with 4 parallel workers (CI default)..." -ForegroundColor Cyan +$failures1 = 0 +for ($i = 1; $i -le 20; $i++) { + Write-Host " Run $i..." -NoNewline + $output = uv run --frozen pytest tests/client/test_stdio.py::test_stdio_context_manager_exiting -v -n 4 2>&1 + if ($LASTEXITCODE -ne 0) { + $failures1++ + Write-Host " FAILED" -ForegroundColor Red + } else { + Write-Host " PASSED" -ForegroundColor Green + } +} +Write-Host " Result: $failures1 failures out of 20 runs" -ForegroundColor $(if ($failures1 -eq 0) { "Green" } else { "Red" }) +Write-Host "" + +# Test 2: Run with 2 workers +Write-Host "Test 2: Running with 2 parallel workers..." -ForegroundColor Cyan +$failures2 = 0 +for ($i = 1; $i -le 20; $i++) { + Write-Host " Run $i..." -NoNewline + $output = uv run --frozen pytest tests/client/test_stdio.py::test_stdio_context_manager_exiting -v -n 2 2>&1 + if ($LASTEXITCODE -ne 0) { + $failures2++ + Write-Host " FAILED" -ForegroundColor Red + } else { + Write-Host " PASSED" -ForegroundColor Green + } +} +Write-Host " Result: $failures2 failures out of 20 runs" -ForegroundColor $(if ($failures2 -eq 0) { "Green" } else { "Red" }) +Write-Host "" + +# Test 3: Run all stdio tests in parallel (simulates real CI) +Write-Host "Test 3: Running ALL stdio tests with 4 workers (full CI simulation)..." -ForegroundColor Cyan +$failures3 = 0 +for ($i = 1; $i -le 10; $i++) { + Write-Host " Run $i..." -NoNewline + $output = uv run --frozen pytest tests/client/test_stdio.py -v -n 4 2>&1 + if ($LASTEXITCODE -ne 0) { + $failures3++ + Write-Host " FAILED" -ForegroundColor Red + # Show which test failed + $failedTest = $output | Select-String "FAILED tests/client/test_stdio.py::" | Select-Object -First 1 + if ($failedTest) { + Write-Host " Failed: $failedTest" -ForegroundColor Red + } + } else { + Write-Host " PASSED" -ForegroundColor Green + } +} +Write-Host " Result: $failures3 failures out of 10 runs" -ForegroundColor $(if ($failures3 -eq 0) { "Green" } else { "Red" }) +Write-Host "" + +# Summary +Write-Host "========== SUMMARY ==========" -ForegroundColor Cyan +Write-Host "4 workers (single test): $failures1/20 failures" +Write-Host "2 workers (single test): $failures2/20 failures" +Write-Host "4 workers (all tests): $failures3/10 failures" +Write-Host "" + +if ($failures1 -gt 0 -or $failures2 -gt 0 -or $failures3 -gt 0) { + Write-Host "FLAKINESS DETECTED with parallel execution!" -ForegroundColor Red + Write-Host "" + Write-Host "This confirms the issue is related to parallel test execution." -ForegroundColor Yellow + Write-Host "The race condition likely involves:" -ForegroundColor Yellow + Write-Host " - Windows Job Object handle management" -ForegroundColor Gray + Write-Host " - Process cleanup timing with multiple workers" -ForegroundColor Gray + Write-Host " - Handle inheritance between test processes" -ForegroundColor Gray +} else { + Write-Host "No flakiness detected in this run." -ForegroundColor Green + Write-Host "The issue might require specific timing conditions to reproduce." -ForegroundColor Yellow +} \ No newline at end of file diff --git a/scripts/windows-debug/test-stdio-simple.ps1 b/scripts/windows-debug/test-stdio-simple.ps1 new file mode 100644 index 000000000..3cf877ca0 --- /dev/null +++ b/scripts/windows-debug/test-stdio-simple.ps1 @@ -0,0 +1,40 @@ +#!/usr/bin/env pwsh +# Simple script to run the test without xdist +# Usage: .\test-stdio-simple.ps1 +# +# Prerequisites: Run . .\setup-environment.ps1 first to ensure tee is available + +Write-Host "Running test_stdio_context_manager_exiting with xdist disabled..." -ForegroundColor Cyan +Write-Host "" + +# Check if tee is available +$teeCheck = python -c "import shutil; print(shutil.which('tee'))" +if (-not $teeCheck -or $teeCheck -eq "None") { + Write-Host "ERROR: tee command not found!" -ForegroundColor Red + Write-Host "Please run: . .\setup-environment.ps1" -ForegroundColor Yellow + Write-Host "(Note the dot at the beginning to source the script)" -ForegroundColor Yellow + exit 1 +} + +Write-Host "tee found at: $teeCheck" -ForegroundColor Green +Write-Host "" + +# Run the test with the working method +Write-Host "Running test with -o addopts='' to override xdist..." -ForegroundColor Yellow +uv run --frozen pytest tests/client/test_stdio.py::test_stdio_context_manager_exiting -xvs -o addopts="" +$exitCode = $LASTEXITCODE + +Write-Host "" +if ($exitCode -eq 0) { + Write-Host "Test PASSED!" -ForegroundColor Green + Write-Host "" + Write-Host "You can now use the other scripts to test for flakiness:" -ForegroundColor Cyan + Write-Host " .\test-stdio-flakiness-200-runs.ps1" + Write-Host " .\test-stdio-flakiness-until-failure.ps1" + Write-Host " .\test-stdio-verbose-debug.ps1" +} else { + Write-Host "Test FAILED with exit code $exitCode" -ForegroundColor Red + Write-Host "" + Write-Host "Try running with verbose debug:" -ForegroundColor Yellow + Write-Host " .\test-stdio-verbose-debug.ps1" +} \ No newline at end of file diff --git a/scripts/windows-debug/test-stdio-stress-race.ps1 b/scripts/windows-debug/test-stdio-stress-race.ps1 new file mode 100644 index 000000000..633fa7b20 --- /dev/null +++ b/scripts/windows-debug/test-stdio-stress-race.ps1 @@ -0,0 +1,125 @@ +#!/usr/bin/env pwsh +# Script to stress test the specific race condition in stdio cleanup +# This creates many processes rapidly to expose handle/job object races +# Usage: .\test-stdio-stress-race.ps1 +# +# Prerequisites: Run . .\setup-environment.ps1 first to ensure tee is available + +Write-Host "Stress testing stdio cleanup race conditions..." -ForegroundColor Cyan +Write-Host "This test creates many processes rapidly to expose timing issues." -ForegroundColor Yellow +Write-Host "" + +# Check if tee is available +$teeCheck = python -c "import shutil; print(shutil.which('tee'))" +if (-not $teeCheck -or $teeCheck -eq "None") { + Write-Host "ERROR: tee command not found!" -ForegroundColor Red + Write-Host "Please run: . .\setup-environment.ps1" -ForegroundColor Yellow + Write-Host "(Note the dot at the beginning to source the script)" -ForegroundColor Yellow + exit 1 +} + +# Create a Python script that runs the test many times in quick succession +$stressScript = @' +import asyncio +import sys +import time +from pathlib import Path + +# Add parent directories to path +sys.path.insert(0, str(Path(__file__).parent.parent.parent)) + +from mcp.client.stdio import stdio_client, StdioServerParameters + +async def rapid_test(test_id: int): + """Run a single test iteration""" + try: + async with stdio_client(StdioServerParameters(command="tee")) as (_, _): + pass + return True, None + except Exception as e: + return False, str(e) + +async def stress_test(iterations: int, concurrent: int): + """Run many tests concurrently""" + print(f"Running {iterations} tests with {concurrent} concurrent...") + + failures = 0 + errors = [] + start_time = time.time() + + # Run in batches + for batch in range(0, iterations, concurrent): + batch_size = min(concurrent, iterations - batch) + tasks = [rapid_test(batch + i) for i in range(batch_size)] + results = await asyncio.gather(*tasks) + + for success, error in results: + if not success: + failures += 1 + if error and error not in errors: + errors.append(error) + + # Progress indicator + if (batch + batch_size) % 100 == 0: + print(f" Completed {batch + batch_size}/{iterations} tests...") + + duration = time.time() - start_time + return failures, errors, duration + +async def main(): + # Test different concurrency levels + configs = [ + (100, 1), # Sequential + (100, 2), # Low concurrency + (100, 5), # Medium concurrency + (100, 10), # High concurrency + ] + + for iterations, concurrent in configs: + print(f"\nTest: {iterations} iterations, {concurrent} concurrent") + failures, errors, duration = await stress_test(iterations, concurrent) + + print(f" Duration: {duration:.2f}s") + print(f" Failures: {failures}/{iterations}") + if errors: + print(f" Unique errors: {len(errors)}") + for error in errors[:3]: # Show first 3 errors + print(f" - {error}") + + if failures > 0: + print(" RACE CONDITION DETECTED!" if concurrent > 1 else " FAILURE DETECTED!") + +if __name__ == "__main__": + asyncio.run(main()) +'@ + +# Save the stress test script +$scriptPath = Join-Path $PSScriptRoot "stress_test.py" +$stressScript | Out-File -FilePath $scriptPath -Encoding UTF8 + +Write-Host "Running stress tests..." -ForegroundColor Cyan +Write-Host "" + +# Run the stress test +uv run python $scriptPath + +$exitCode = $LASTEXITCODE + +# Clean up +Remove-Item $scriptPath -ErrorAction SilentlyContinue + +Write-Host "" +Write-Host "========== ANALYSIS ==========" -ForegroundColor Cyan + +if ($exitCode -ne 0) { + Write-Host "Stress test failed to complete." -ForegroundColor Red +} else { + Write-Host "Stress test completed." -ForegroundColor Green + Write-Host "" + Write-Host "If failures increased with concurrency, it indicates:" -ForegroundColor Yellow + Write-Host " - Race condition in process cleanup" -ForegroundColor Gray + Write-Host " - Job Object handle conflicts" -ForegroundColor Gray + Write-Host " - Windows handle inheritance issues" -ForegroundColor Gray + Write-Host "" + Write-Host "This matches the CI flakiness pattern where parallel tests fail." -ForegroundColor Yellow +} \ No newline at end of file diff --git a/scripts/windows-debug/test-stdio-verbose-debug.ps1 b/scripts/windows-debug/test-stdio-verbose-debug.ps1 new file mode 100644 index 000000000..593c216cf --- /dev/null +++ b/scripts/windows-debug/test-stdio-verbose-debug.ps1 @@ -0,0 +1,52 @@ +#!/usr/bin/env pwsh +# Script to run test_stdio_context_manager_exiting with maximum debugging output +# Usage: .\test-stdio-verbose-debug.ps1 +# +# Prerequisites: Run . .\setup-environment.ps1 first to ensure tee is available + +Write-Host "Running test_stdio_context_manager_exiting with verbose debug output..." -ForegroundColor Cyan +Write-Host "" + +# Check if tee is available +$teeCheck = python -c "import shutil; print(shutil.which('tee'))" +if (-not $teeCheck -or $teeCheck -eq "None") { + Write-Host "ERROR: tee command not found!" -ForegroundColor Red + Write-Host "Please run: . .\setup-environment.ps1" -ForegroundColor Yellow + Write-Host "(Note the dot at the beginning to source the script)" -ForegroundColor Yellow + exit 1 +} + +# Set environment variables for debugging +$env:PYTHONFAULTHANDLER = "1" +$env:PYTEST_CURRENT_TEST = "1" + +Write-Host "Environment variables set:" -ForegroundColor Yellow +Write-Host " PYTHONFAULTHANDLER = 1 (enables Python fault handler)" +Write-Host "" + +Write-Host "Running test with maximum verbosity..." -ForegroundColor Cyan +Write-Host "" + +# Run the test with all debugging options +uv run --frozen pytest ` + tests/client/test_stdio.py::test_stdio_context_manager_exiting ` + -xvs ` + -o addopts="" ` + --log-cli-level=DEBUG ` + --log-cli-format="%(asctime)s [%(levelname)s] %(name)s: %(message)s" ` + --capture=no ` + --tb=long ` + --full-trace + +$exitCode = $LASTEXITCODE + +Write-Host "" +if ($exitCode -eq 0) { + Write-Host "Test PASSED" -ForegroundColor Green +} else { + Write-Host "Test FAILED with exit code: $exitCode" -ForegroundColor Red +} + +# Clean up environment variables +Remove-Item Env:PYTHONFAULTHANDLER -ErrorAction SilentlyContinue +Remove-Item Env:PYTEST_CURRENT_TEST -ErrorAction SilentlyContinue \ No newline at end of file diff --git a/scripts/windows-debug/test.txt b/scripts/windows-debug/test.txt new file mode 100644 index 000000000..e69de29bb diff --git a/test.txt b/test.txt new file mode 100644 index 000000000..5ca294497 --- /dev/null +++ b/test.txt @@ -0,0 +1,6 @@ +hello +test +test +test +test +test