Update Testing

- Fix flakey tests in parallel with fixtures
- Generate stubs via nox
- Use nox in definitively
- Add more session types to nox
- add generate-stubs.sh for convinenence

Author: tempusfrangit

.github/workflows/ci.yml

@@ -53,7 +53,7 @@ jobs:
         node-version: '24'
 
     - name: Generate type stubs
-      run: script/check.sh stubs
+      run: uv tool run nox -s stubs
 
     - name: Check for stub changes
       run: |

noxfile.py

@@ -1,101 +1,174 @@
-import os
-
 import nox
 
 # Python versions from pyproject.toml
-PYTHON_VERSIONS = ["3.9", "3.10", "3.11", "3.12", "3.13"]
+PYTHON_VERSIONS = ['3.9', '3.10', '3.11', '3.12', '3.13']
+
+
+def _generate_stubs():
+    """Helper function to generate pyright stubs."""
+    import os
+    import shutil
+    import subprocess
+
+    # Check if npx is available
+    try:
+        subprocess.run(['npx', '--version'], capture_output=True, check=True)
+    except (subprocess.CalledProcessError, FileNotFoundError):
+        raise RuntimeError(
+            'npx not found. Please install Node.js to generate type stubs.\n'
+            'Visit: https://nodejs.org/ or use your package manager.'
+        )
+
+    # Remove existing stubs to avoid duplication
+    subprocess.run(['find', 'python', '-name', '*.pyi', '-type', 'f', '-delete'])
+
+    # Generate stubs using npx pyright
+    env = os.environ.copy()
+    env['PYTHONPATH'] = 'python'
+
+    try:
+        subprocess.run(
+            ['npx', '-y', 'pyright', '--createstub', 'coglet'], env=env, check=True
+        )
+    except subprocess.CalledProcessError:
+        print('Warning: coglet stub creation may have failed')
+
+    try:
+        subprocess.run(
+            ['npx', '-y', 'pyright', '--createstub', 'cog'], env=env, check=True
+        )
+    except subprocess.CalledProcessError:
+        print('Warning: cog stub creation may have failed')
+
+    # Move stubs from typings/ to alongside source
+    if os.path.exists('typings'):
+        shutil.copytree('typings', 'python', dirs_exist_ok=True)
+        shutil.rmtree('typings')
+
 
 # Use uv for faster package installs
-nox.options.default_venv_backend = "uv"
+nox.options.default_venv_backend = 'uv'
 
 
 @nox.session(python=PYTHON_VERSIONS)
 def tests(session):
     """Run tests with pytest-xdist parallelization."""
-    session.install(".[test]")
-    
+    session.install('.[test]')
+
     # Pass through any arguments to pytest
-    pytest_args = ["-vv", "-n", "auto"] + list(session.posargs)
-    
+    pytest_args = ['-vv', '-n', 'auto'] + list(session.posargs)
+
     # Only add -n auto if -n isn't already specified
-    if any(arg.startswith("-n") for arg in session.posargs):
-        pytest_args = ["-vv"] + list(session.posargs)
-    
-    session.run("pytest", *pytest_args)
+    if any(arg.startswith('-n') for arg in session.posargs):
+        pytest_args = ['-vv'] + list(session.posargs)
 
+    session.run('pytest', *pytest_args)
 
-@nox.session(python="3.11")  # Single version for regular testing
+
+@nox.session(python='3.11')  # Single version for regular testing
 def test(session):
     """Run tests on single Python version (faster for development)."""
-    session.install(".[test]")
-    
+    session.install('.[test]')
+
     # Pass through any arguments to pytest
-    pytest_args = ["-vv", "-n", "auto"] + list(session.posargs)
-    
+    pytest_args = ['-vv', '-n', 'auto'] + list(session.posargs)
+
     # Only add -n auto if -n isn't already specified
-    if any(arg.startswith("-n") for arg in session.posargs):
-        pytest_args = ["-vv"] + list(session.posargs)
-    
-    session.run("pytest", *pytest_args)
+    if any(arg.startswith('-n') for arg in session.posargs):
+        pytest_args = ['-vv'] + list(session.posargs)
+
+    session.run('pytest', *pytest_args)
 
 
 @nox.session(python=None)  # Use current system Python, but create venv
 def test_current(session):
     """Run tests using current system Python with isolated venv (for CI)."""
-    session.install(".[test]")
-    
+    session.install('.[test]')
+
     # Pass through any arguments to pytest
-    pytest_args = ["-vv", "-n", "auto"] + list(session.posargs)
-    
+    pytest_args = ['-vv', '-n', 'auto'] + list(session.posargs)
+
     # Only add -n auto if -n isn't already specified
-    if any(arg.startswith("-n") for arg in session.posargs):
-        pytest_args = ["-vv"] + list(session.posargs)
-    
-    session.run("pytest", *pytest_args)
+    if any(arg.startswith('-n') for arg in session.posargs):
+        pytest_args = ['-vv'] + list(session.posargs)
+
+    session.run('pytest', *pytest_args)
 
 
 @nox.session
 def lint(session):
     """Run ruff linting (check mode)."""
-    session.install(".[dev]")
+    session.install('.[dev]')
     # Check if --fix is in posargs for local dev
-    if "--fix" in session.posargs:
-        session.run("ruff", "check", "--fix", ".")
+    check_only = '--fix' not in session.posargs
+    session.log(f'running lint... {"[check_only]" if check_only else "[auto_fix]"}')
+    if '--fix' in session.posargs:
+        session.run('ruff', 'check', '--fix', '.')
     else:
-        session.run("ruff", "check", ".")
+        session.run('ruff', 'check', '.')
 
 
 @nox.session
 def format(session):
     """Format code with ruff."""
-    session.install(".[dev]")
+    session.install('.[dev]')
     # Check mode for CI, format mode for local
-    if "--check" in session.posargs:
-        session.run("ruff", "format", "--check", ".")
+    if '--check' in session.posargs:
+        session.run('ruff', 'format', '--check', '.')
     else:
-        session.run("ruff", "format", ".")
+        session.run('ruff', 'format', '.')
+
+
+@nox.session
+def stubs(session):
+    """Generate pyright stubs."""
+    session.log('stubbing python modules...')
+    try:
+        _generate_stubs()
+        session.log('python module stubs generated successfully')
+    except RuntimeError as e:
+        session.error(str(e))
 
 
 @nox.session
 def typecheck(session):
     """Run mypy type checking."""
-    session.install(".[dev]")
+    import os
+
+    session.log('running typecheck...')
+    # Only generate stubs if not in CI (CI validates existing stubs separately)
+    if not os.environ.get('CI'):
+        session.log('auto-generating python module stubs...')
+        # Run stubs generation inline instead of separate session
+        _generate_stubs()
+        session.log('done auto-generating python module stubs...')
+
+    # Install dev deps (mypy) + test deps (pytest) + provided deps (pydantic)
+    session.install('.[dev,test,provided]')
     session.run(
-        "mypy", ".", 
-        "--exclude", "build",
-        "--exclude", "python/tests/cases",
-        "--exclude", "python/tests/bad_inputs", 
-        "--exclude", "python/tests/bad_predictors",
-        "--exclude", "python/tests/runners",
-        "--exclude", "python/tests/schemas",
-        "--exclude", "python/.*\\.pyi",
-        *session.posargs
+        'mypy',
+        '.',
+        '--exclude',
+        'build',
+        '--exclude',
+        'python/tests/cases',
+        '--exclude',
+        'python/tests/bad_inputs',
+        '--exclude',
+        'python/tests/bad_predictors',
+        '--exclude',
+        'python/tests/runners',
+        '--exclude',
+        'python/tests/schemas',
+        '--exclude',
+        'python/.*\\.pyi',
+        *session.posargs,
     )
 
 
 @nox.session
 def check_all(session):
     """Run all checks (lint, format check, typecheck)."""
-    session.notify("lint")
-    session.notify("format", ["--check"])
-    session.notify("typecheck")
+    session.notify('lint')
+    session.notify('format', ['--check'])
+    session.notify('typecheck')

pyproject.toml

@@ -19,6 +19,7 @@ dev = [
     'ipython',
     'mypy==1.16.0', # pinned to fix CI
     'nox',
+    'ruff',
     'setuptools',
 ]
 

python/tests/test_file_runner.py

@@ -11,10 +11,9 @@
 from pathlib import Path
 from typing import Dict, List, Optional
 
-from coglet import file_runner
+import pytest
 
-ipc_port: Optional[int] = None
-ipc_popen: Optional[subprocess.Popen[bytes]] = None
+from coglet import file_runner
 
 
 def find_free_port() -> int:
@@ -24,26 +23,42 @@ def find_free_port() -> int:
         return s.getsockname()[1]
 
 
-def setup_module():
+@pytest.fixture(scope='module')
+def ipc_server():
+    """Start webhook server for IPC communication, one per test module."""
+    import urllib.error
+    import urllib.request
+
     # Webhook simulates /_ipc endpoint of Go server for receiving Python runner status updates
     cwd = str(Path(__file__).absolute().parent)
     env = os.environ.copy()
-    global ipc_port, ipc_popen
-    ipc_port = find_free_port()
-    env['PORT'] = str(ipc_port)
-    ipc_popen = subprocess.Popen(['python3', 'webhook.py'], cwd=cwd, env=env)
+    port = find_free_port()
+    env['PORT'] = str(port)
+    popen = subprocess.Popen(['python3', 'webhook.py'], cwd=cwd, env=env)
 
+    # Wait for server to actually start and be ready (more robust than fixed sleep)
+    for _ in range(50):  # Try for up to 5 seconds
+        try:
+            urllib.request.urlopen(f'http://localhost:{port}/_requests', timeout=0.1)
+            break
+        except (urllib.error.URLError, ConnectionRefusedError):
+            time.sleep(0.1)
+    else:
+        popen.terminate()
+        raise RuntimeError(f'Webhook server on port {port} failed to start')
 
-def teardown_module():
-    global ipc_popen
-    ipc_popen.terminate()
+    yield port
+
+    popen.terminate()
+    popen.wait()
 
 
 class FileRunnerTest:
     def __init__(
         self,
         tmp_path: Path,
         predictor: str,
+        ipc_port: int,
         env: Optional[Dict[str, str]] = None,
         max_concurrency: int = 1,
         predictor_class: str = 'Predictor',
@@ -53,7 +68,7 @@ def __init__(
         if env is not None:
             runner_env.update(env)
         runner_env['PYTHONPATH'] = str(Path(__file__).absolute().parent.parent)
-        global ipc_port
+        self.ipc_port = ipc_port
         self.name = f'runner-{uuid.uuid4()}'
         cmd = [
             sys.executable,
@@ -79,8 +94,7 @@ def __init__(
         )
 
     def statuses(self) -> List[str]:
-        global ipc_port
-        resp = urllib.request.urlopen(f'http://localhost:{ipc_port}/_requests')
+        resp = urllib.request.urlopen(f'http://localhost:{self.ipc_port}/_requests')
         requests = json.loads(resp.read()) or []
         statuses = []
         for r in requests:
@@ -105,8 +119,8 @@ def wait_for_file(path, exists: bool = True) -> None:
             return
 
 
-def test_file_runner(tmp_path):
-    rt = FileRunnerTest(tmp_path, 'sleep', env={'SETUP_SLEEP': '1'})
+def test_file_runner(tmp_path, ipc_server):
+    rt = FileRunnerTest(tmp_path, 'sleep', ipc_server, env={'SETUP_SLEEP': '1'})
 
     openapi_file = os.path.join(tmp_path, 'openapi.json')
     wait_for_file(openapi_file)
@@ -145,8 +159,10 @@ def test_file_runner(tmp_path):
     rt.stop()
 
 
-def test_file_runner_setup_failed(tmp_path):
-    rt = FileRunnerTest(tmp_path, 'sleep', predictor_class='SetupFailingPredictor')
+def test_file_runner_setup_failed(tmp_path, ipc_server):
+    rt = FileRunnerTest(
+        tmp_path, 'sleep', ipc_server, predictor_class='SetupFailingPredictor'
+    )
 
     openapi_file = os.path.join(tmp_path, 'openapi.json')
     wait_for_file(openapi_file)
@@ -160,9 +176,12 @@ def test_file_runner_setup_failed(tmp_path):
     rt.stop(1)
 
 
-def test_file_runner_predict_failed(tmp_path):
+def test_file_runner_predict_failed(tmp_path, ipc_server):
     rt = FileRunnerTest(
-        tmp_path, 'sleep', predictor_class='PredictionFailingPredictorWithTiming'
+        tmp_path,
+        'sleep',
+        ipc_server,
+        predictor_class='PredictionFailingPredictorWithTiming',
     )
 
     openapi_file = os.path.join(tmp_path, 'openapi.json')
@@ -202,8 +221,8 @@ def test_file_runner_predict_failed(tmp_path):
     rt.stop()
 
 
-def test_file_runner_predict_canceled(tmp_path):
-    rt = FileRunnerTest(tmp_path, 'sleep')
+def test_file_runner_predict_canceled(tmp_path, ipc_server):
+    rt = FileRunnerTest(tmp_path, 'sleep', ipc_server)
 
     openapi_file = os.path.join(tmp_path, 'openapi.json')
     wait_for_file(openapi_file)