ci.yml

name: CI/CD Pipeline

on:
  push:
    branches: [ main, develop ]
  pull_request:
    branches: [ main, develop ]

env:
  PYTHON_DEFAULT_VERSION: "3.11"

jobs:
  test:
    name: Test Python ${{ env.PYTHON_DEFAULT_VERSION }}
    runs-on: ubuntu-latest
    
    steps:
    - name: Checkout code
      uses: actions/checkout@v4
    
    - name: Set up Python ${{ env.PYTHON_DEFAULT_VERSION }}
      uses: actions/setup-python@v5
      with:
        python-version: ${{ env.PYTHON_DEFAULT_VERSION }}
        cache: 'pip'
    
    - name: Install system dependencies
      run: |
        sudo apt-get update
        sudo apt-get install -y gcc g++ curl git
    
    - name: Upgrade pip, setuptools, and wheel
      run: |
        python -m pip install --upgrade pip setuptools wheel build
    
    - name: Install PyTorch CPU (for faster CI)
      run: |
        pip install --no-cache-dir --default-timeout=600 \
          --extra-index-url https://download.pytorch.org/whl/cpu \
          torch==2.1.0+cpu \
          torchvision==0.16.0+cpu \
          torchaudio==2.1.0+cpu
    
    - name: Install project dependencies
      run: |
        # Install package in editable mode first
        pip install --no-cache-dir -e ".[dev]"
        # Install remaining dependencies that aren't in dev extras
        pip install --no-cache-dir \
          transformers>=4.34.0 \
          scikit-learn>=1.3.0 \
          pandas>=2.1.1 \
          numpy>=1.26.0 \
          biopython>=1.81 \
          sentencepiece>=0.1.99 \
          accelerate>=0.24.0 \
          datasets>=2.14.0 \
          google-generativeai>=0.7.2 \
          tiktoken>=0.5.0 \
          litellm>=1.50.0 \
          requests>=2.31.0 \
          beautifulsoup4>=4.12.2 \
          lxml>=4.9.0 \
          openpyxl>=3.1.0 \
          xlrd>=2.0.1 \
          tqdm>=4.65.0 \
          python-dotenv>=1.0.0 \
          psutil>=5.9.0 \
          fastapi>=0.104.0 \
          "uvicorn[standard]>=0.23.2" \
          aiohttp>=3.8.6 \
          websockets>=11.0.3 \
          python-multipart>=0.0.5 \
          aiofiles>=0.7.0 \
          pydantic>=2.4.2 \
          typing-extensions>=3.10.0.2 \
          starlette>=0.31.1 \
          click>=8.0.1 \
          h11>=0.12.0 \
          httptools>=0.3.0 \
          PyYAML>=5.4.1 \
          "watchfiles[watchdog]>=1.0.0" \
          wsproto>=1.0.0 \
          tokenizers>=0.14.1 \
          pytz>=2023.3 \
          qdrant-client>=1.7.0
    
    - name: Install paper-qa from GitHub
      run: |
        # For Python 3.11, install latest paper-qa from GitHub (supports >=5.0.0)
        echo "Installing paper-qa from GitHub for Python 3.11"
        pip install --no-cache-dir git+https://github.com/Future-House/paper-qa.git || \
        pip install --no-cache-dir "paper-qa>=4.9.0,<5.0.0"
    
    - name: Verify installation
      env:
        PYTHONPATH: ${{ github.workspace }}
      run: |
        export PYTHONPATH=${{ github.workspace }}:$PYTHONPATH
        python -c "import sys; print('Python version:', sys.version)"
        python -c "import sys; print('Python path:', sys.path)"
        python -c "import app; print('✓ App imported successfully')" || (echo "✗ App import failed" && exit 1)
        python -c "from app.utils.config import GEMINI_API_KEY; print('✓ Config imported')" || echo "⚠ Config import warning"
    
    - name: Run tests
      env:
        # Set dummy API keys for testing (tests should handle missing keys gracefully)
        GEMINI_API_KEY: "test_key_1234567890abcdef"
        NCBI_API_KEY: "test_ncbi_key_1234567890"
        EMAIL: "test@example.com"
        PYTHONPATH: ${{ github.workspace }}
      run: |
        export PYTHONPATH=${{ github.workspace }}:$PYTHONPATH
        pytest tests/ -v --tb=short --maxfail=10 --cov=app --cov-report=xml --cov-report=term-missing --cov-fail-under=50
    
    - name: Upload coverage to Codecov
      uses: codecov/codecov-action@v4
      with:
        file: ./coverage.xml
        flags: unittests
        name: codecov-umbrella
        fail_ci_if_error: false

  lint:
    name: Lint Code
    runs-on: ubuntu-latest
    
    steps:
    - name: Checkout code
      uses: actions/checkout@v4
    
    - name: Lint with Black and flake8 using Docker
      run: |
        docker run --rm \
          -v ${{ github.workspace }}:/workspace \
          -w /workspace \
          python:3.11-slim \
          bash -c "
            apt-get update -qq && apt-get install -y -qq gcc > /dev/null 2>&1 && \
            pip install --quiet --no-cache-dir black>=23.0.0 flake8>=6.0.0 && \
            echo '=== Running Black (check mode) ===' && \
            black --check --diff app/ tests/ cli.py main.py && \
            echo '=== Running flake8 ===' && \
            flake8 app/ tests/ cli.py main.py --max-line-length=120 --extend-ignore=E203,W503,E501 --exclude=__pycache__,*.pyc
          "

  type-check:
    name: Type Check
    runs-on: ubuntu-latest
    
    steps:
    - name: Checkout code
      uses: actions/checkout@v4
    
    - name: Set up Python
      uses: actions/setup-python@v5
      with:
        python-version: ${{ env.PYTHON_DEFAULT_VERSION }}
        cache: 'pip'
    
    - name: Install type checking dependencies
      run: |
        python -m pip install --upgrade pip
        pip install mypy>=1.5.0 types-requests types-PyYAML
    
    - name: Run mypy
      run: |
        mypy app/ --config-file mypy.ini || (echo "⚠️ Type checking found issues. Review and fix type errors." && exit 1)
      continue-on-error: false  # Type checking is now required

  security:
    name: Security Scan
    runs-on: ubuntu-latest
    
    steps:
    - name: Checkout code
      uses: actions/checkout@v4
    
    - name: Set up Python
      uses: actions/setup-python@v5
      with:
        python-version: ${{ env.PYTHON_DEFAULT_VERSION }}
        cache: 'pip'
    
    - name: Install security scanning tools
      run: |
        python -m pip install --upgrade pip
        pip install bandit safety
    
    - name: Run Bandit security scan
      run: |
        echo "Running Bandit security scan..."
        # Generate JSON report
        bandit -r app/ -f json -o bandit-report.json -c .bandit -ll || true
        echo ""
        echo "=== Bandit Security Scan Results ==="
        bandit -r app/ -c .bandit -ll || true
        
        # Check for high/critical severity issues
        if [ -f bandit-report.json ]; then
          python3 << 'EOF'
import json
import sys

try:
    with open('bandit-report.json', 'r') as f:
        data = json.load(f)
    
    high_issues = [
        r for r in data.get('results', [])
        if r.get('issue_severity') in ['HIGH', 'CRITICAL']
    ]
    
    if high_issues:
        print(f"❌ Found {len(high_issues)} high/critical severity issues:")
        for issue in high_issues[:5]:
            print(f"  - {issue.get('test_id')}: {issue.get('issue_text', '')[:80]}")
        sys.exit(1)
    else:
        print("✅ No high/critical severity issues found.")
        sys.exit(0)
except Exception as e:
    print(f"⚠️ Could not parse bandit report: {e}")
    sys.exit(0)
EOF
        fi
      continue-on-error: false  # High/critical security issues must be addressed
    
    - name: Check dependencies with Safety
      run: |
        pip install -e ".[dev]"
        safety check --json
        safety check
      continue-on-error: false  # Vulnerable dependencies must be fixed

  docker-build:
    name: Docker Build Test
    runs-on: ubuntu-latest
    
    steps:
    - name: Checkout code
      uses: actions/checkout@v4
    
    - name: Set up Docker Buildx
      uses: docker/setup-buildx-action@v3
    
    - name: Build Docker image
      uses: docker/build-push-action@v5
      with:
        context: .
        file: ./Dockerfile
        push: false
        load: true
        tags: bioanalyzer-backend:test
        cache-from: type=gha
        cache-to: type=gha,mode=max
    
    - name: Test Docker image
      run: |
        docker run --rm bioanalyzer-backend:test python -c "import app; print('Docker image works!')"

  all-checks:
    name: All Checks Summary
    runs-on: ubuntu-latest
    needs: [test, lint, type-check, security, docker-build]
    if: always()
    
    steps:
    - name: Check job status
      run: |
        echo "## CI/CD Pipeline Status" >> $GITHUB_STEP_SUMMARY
        echo "" >> $GITHUB_STEP_SUMMARY
        echo "| Job | Status |" >> $GITHUB_STEP_SUMMARY
        echo "|-----|--------|" >> $GITHUB_STEP_SUMMARY
        echo "| Tests | ${{ needs.test.result }} |" >> $GITHUB_STEP_SUMMARY
        echo "| Lint | ${{ needs.lint.result }} |" >> $GITHUB_STEP_SUMMARY
        echo "| Type Check | ${{ needs.type-check.result }} |" >> $GITHUB_STEP_SUMMARY
        echo "| Security | ${{ needs.security.result }} |" >> $GITHUB_STEP_SUMMARY
        echo "| Docker Build | ${{ needs.docker-build.result }} |" >> $GITHUB_STEP_SUMMARY
        echo "" >> $GITHUB_STEP_SUMMARY
        
        # Fail if any critical check failed
        if [ "${{ needs.test.result }}" != "success" ] || \
           [ "${{ needs.lint.result }}" != "success" ] || \
           [ "${{ needs.type-check.result }}" != "success" ] || \
           [ "${{ needs.security.result }}" != "success" ] || \
           [ "${{ needs.docker-build.result }}" != "success" ]; then
          echo "❌ One or more critical checks failed!" >> $GITHUB_STEP_SUMMARY
          echo "Test: ${{ needs.test.result }}" >> $GITHUB_STEP_SUMMARY
          echo "Lint: ${{ needs.lint.result }}" >> $GITHUB_STEP_SUMMARY
          echo "Type-check: ${{ needs.type-check.result }}" >> $GITHUB_STEP_SUMMARY
          echo "Security: ${{ needs.security.result }}" >> $GITHUB_STEP_SUMMARY
          echo "Docker-build: ${{ needs.docker-build.result }}" >> $GITHUB_STEP_SUMMARY
          exit 1
        else
          echo "✅ All checks passed!" >> $GITHUB_STEP_SUMMARY
        fi