ci-testing.yml

# Phase 9 Testing Infrastructure - Per-Package and Integration Testing
# Created by 09-05-PLAN: CI Matrix for per-package and integration testing
#
# Features:
# - Per-package isolated tests (TEST-06): Each package tested in isolated venv
# - Preset integration tests (TEST-07): Validates agent composition presets
# - Provenance export tests: Validates multi-agent workflow export
# - Python 3.12 only
# - fail-fast: false to identify ALL failures, not just first
#
# Test isolation ensures:
# - Missing dependency declarations are caught
# - Package boundaries are properly enforced
# - Integration issues are pinpointed to exact package/preset

name: Testing Infrastructure

on:
  push:
    branches: [main]
    paths:
      - 'lobster/**'
      - 'packages/**'
      - 'tests/**'
      - '.github/workflows/ci-testing.yml'
  pull_request:
    branches: [main]
    paths:
      - 'lobster/**'
      - 'packages/**'
      - 'tests/**'
      - '.github/workflows/ci-testing.yml'

env:
  PYTHONPATH: ${{ github.workspace }}
  PYTEST_ADDOPTS: "--color=yes --tb=short"

jobs:
  # ==========================================================================
  # TEST-06: Per-Package Isolated Tests
  # Each package tested in its own venv to catch missing dependency declarations
  # ==========================================================================
  package-tests:
    name: Test ${{ matrix.package }}
    runs-on: ubuntu-latest
    timeout-minutes: 15
    strategy:
      fail-fast: false  # Continue all tests to identify ALL failures
      matrix:
        package:
          - lobster-transcriptomics
          - lobster-proteomics
          - lobster-genomics
          - lobster-visualization
          - lobster-ml
          - lobster-research
          - lobster-metadata
          - lobster-structural-viz

    steps:
      - name: Checkout code
        uses: actions/checkout@v4

      - name: Set up Python 3.12
        uses: actions/setup-python@v5
        with:
          python-version: "3.12"

      - name: Free up disk space
        run: |
          sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc 2>/dev/null || true
          df -h

      - name: Create isolated venv for ${{ matrix.package }}
        run: |
          python -m venv .venv-${{ matrix.package }}
          source .venv-${{ matrix.package }}/bin/activate
          pip install --upgrade pip

      - name: Install lobster-ai core (provides lobster.testing)
        run: |
          source .venv-${{ matrix.package }}/bin/activate
          # Install core package with dev dependencies first
          pip install -e ".[dev]"

      - name: Install package under test
        run: |
          source .venv-${{ matrix.package }}/bin/activate
          # Install the specific package if it exists
          PACKAGE_DIR="packages/${{ matrix.package }}"
          if [ -d "$PACKAGE_DIR" ] && [ -f "$PACKAGE_DIR/pyproject.toml" ]; then
            echo "Installing ${{ matrix.package }} from $PACKAGE_DIR"
            pip install -e "$PACKAGE_DIR[dev]" 2>/dev/null || pip install -e "$PACKAGE_DIR"
          else
            echo "Package directory $PACKAGE_DIR not found, skipping package install"
          fi

      - name: Run package tests
        run: |
          source .venv-${{ matrix.package }}/bin/activate
          PACKAGE_TEST_DIR="packages/${{ matrix.package }}/tests"
          if [ -d "$PACKAGE_TEST_DIR" ]; then
            echo "Running tests from $PACKAGE_TEST_DIR"
            pytest "$PACKAGE_TEST_DIR" \
              -m "not slow and not real_api" \
              --import-mode=importlib \
              -v \
              --no-cov \
              --junit-xml=results-${{ matrix.package }}.xml \
              --tb=short
          else
            echo "No tests found at $PACKAGE_TEST_DIR, creating placeholder result"
            echo '<?xml version="1.0" encoding="utf-8"?><testsuites><testsuite name="placeholder" tests="0" failures="0" errors="0"></testsuite></testsuites>' > results-${{ matrix.package }}.xml
          fi

      - name: Upload test results
        uses: actions/upload-artifact@v4
        if: always()
        with:
          name: test-results-${{ matrix.package }}
          path: results-${{ matrix.package }}.xml
          retention-days: 14

  # ==========================================================================
  # TEST-07: Preset Integration Tests
  # Validates that agent composition presets work correctly
  # ==========================================================================
  preset-tests:
    name: Test preset ${{ matrix.preset }}
    runs-on: ubuntu-latest
    timeout-minutes: 20
    needs: [package-tests]  # Run after package tests validate individual packages
    strategy:
      fail-fast: false
      matrix:
        preset:
          - scrna-basic
          - scrna-full
          - multiomics-full

    steps:
      - name: Checkout code
        uses: actions/checkout@v4

      - name: Set up Python 3.12
        uses: actions/setup-python@v5
        with:
          python-version: "3.12"

      - name: Free up disk space
        run: |
          sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc 2>/dev/null || true
          df -h

      - name: Install all packages
        run: |
          pip install --upgrade pip
          # Install core with full extras for all agent dependencies
          pip install -e ".[dev,full]" 2>/dev/null || pip install -e ".[dev]"

      - name: Verify preset dependency resolution
        run: |
          python -c "
          from lobster.config.agent_presets import expand_preset, is_valid_preset

          preset_name = '${{ matrix.preset }}'

          # Verify preset is valid
          assert is_valid_preset(preset_name), f'Preset {preset_name} is not valid'

          # Expand preset and verify agents
          agents = expand_preset(preset_name)
          assert agents is not None, f'Preset {preset_name} expansion returned None'
          assert len(agents) > 0, f'Preset {preset_name} has no agents'

          print(f'Preset {preset_name} resolved to {len(agents)} agents:')
          for agent in agents:
              print(f'  - {agent}')
          "

      - name: Run preset integration tests
        run: |
          # Convert preset name to test file name (replace - with _)
          TEST_FILE="tests/integration/test_preset_$(echo '${{ matrix.preset }}' | tr '-' '_').py"
          if [ -f "$TEST_FILE" ]; then
            echo "Running tests from $TEST_FILE"
            pytest "$TEST_FILE" \
              -m "not slow and not real_api" \
              --import-mode=importlib \
              -v \
              --no-cov \
              --junit-xml=results-preset-${{ matrix.preset }}.xml \
              --tb=short
          else
            echo "Test file $TEST_FILE not found"
            exit 1
          fi

      - name: Upload test results
        uses: actions/upload-artifact@v4
        if: always()
        with:
          name: test-results-preset-${{ matrix.preset }}
          path: results-preset-${{ matrix.preset }}.xml
          retention-days: 14

  # ==========================================================================
  # Integration Tests: Provenance Export
  # Validates multi-agent workflow provenance and notebook export
  # ==========================================================================
  integration-tests:
    name: Integration tests
    runs-on: ubuntu-latest
    timeout-minutes: 20
    needs: [package-tests]  # Run after package tests pass

    steps:
      - name: Checkout code
        uses: actions/checkout@v4

      - name: Set up Python 3.12
        uses: actions/setup-python@v5
        with:
          python-version: "3.12"

      - name: Free up disk space
        run: |
          sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc 2>/dev/null || true
          df -h

      - name: Install with all extras
        run: |
          pip install --upgrade pip
          pip install -e ".[dev,full]" 2>/dev/null || pip install -e ".[dev]"

      - name: Run provenance export tests
        run: |
          TEST_FILE="tests/integration/test_provenance_export.py"
          if [ -f "$TEST_FILE" ]; then
            pytest "$TEST_FILE" \
              -m "not slow and not real_api" \
              --import-mode=importlib \
              -v \
              --no-cov \
              --junit-xml=results-integration.xml \
              --tb=short
          else
            echo "Provenance export tests not found at $TEST_FILE"
            exit 1
          fi

      - name: Upload test results
        uses: actions/upload-artifact@v4
        if: always()
        with:
          name: test-results-integration
          path: results-integration.xml
          retention-days: 14

  # ==========================================================================
  # Summary Job: Aggregate Results for Branch Protection
  # ==========================================================================
  test-summary:
    name: Test Summary
    runs-on: ubuntu-latest
    needs: [package-tests, preset-tests, integration-tests]
    if: always()

    steps:
      - name: Download all test results
        uses: actions/download-artifact@v4
        with:
          path: test-results
          pattern: test-results-*

      - name: Display test summary
        run: |
          echo "=== Test Results Summary ==="
          echo ""

          TOTAL_PASS=0
          TOTAL_FAIL=0

          for dir in test-results/*/; do
            if [ -d "$dir" ]; then
              for file in "$dir"results-*.xml; do
                if [ -f "$file" ]; then
                  name=$(basename "$file" .xml | sed 's/results-//')

                  # Count failures and errors from JUnit XML
                  failures=$(grep -oP 'failures="\K[0-9]+' "$file" 2>/dev/null | head -1 || echo "0")
                  errors=$(grep -oP 'errors="\K[0-9]+' "$file" 2>/dev/null | head -1 || echo "0")

                  # Handle empty values
                  failures=${failures:-0}
                  errors=${errors:-0}

                  if [ "$failures" = "0" ] && [ "$errors" = "0" ]; then
                    echo "PASS: $name"
                    TOTAL_PASS=$((TOTAL_PASS + 1))
                  else
                    echo "FAIL: $name (failures: $failures, errors: $errors)"
                    TOTAL_FAIL=$((TOTAL_FAIL + 1))
                  fi
                fi
              done
            fi
          done

          echo ""
          echo "=== Summary ==="
          echo "Passed: $TOTAL_PASS"
          echo "Failed: $TOTAL_FAIL"

      - name: Check job results
        run: |
          # Check if any upstream jobs failed
          PACKAGE_RESULT="${{ needs.package-tests.result }}"
          PRESET_RESULT="${{ needs.preset-tests.result }}"
          INTEGRATION_RESULT="${{ needs.integration-tests.result }}"

          echo "Package tests: $PACKAGE_RESULT"
          echo "Preset tests: $PRESET_RESULT"
          echo "Integration tests: $INTEGRATION_RESULT"

          if [ "$PACKAGE_RESULT" = "failure" ] || [ "$PRESET_RESULT" = "failure" ] || [ "$INTEGRATION_RESULT" = "failure" ]; then
            echo ""
            echo "One or more test jobs failed. See individual job logs for details."
            exit 1
          fi

          echo ""
          echo "All test jobs completed successfully!"