Format code with Black and isort #12
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: CI/CD Pipeline | ||
| on: | ||
| push: | ||
| branches: [ main, develop ] | ||
| pull_request: | ||
| branches: [ main, develop ] | ||
| env: | ||
| PYTHON_DEFAULT_VERSION: "3.11" | ||
| jobs: | ||
| test: | ||
| name: Test Python ${{ env.PYTHON_DEFAULT_VERSION }} | ||
| runs-on: ubuntu-latest | ||
| steps: | ||
| - name: Checkout code | ||
| uses: actions/checkout@v4 | ||
| - name: Set up Python ${{ env.PYTHON_DEFAULT_VERSION }} | ||
| uses: actions/setup-python@v5 | ||
| with: | ||
| python-version: ${{ env.PYTHON_DEFAULT_VERSION }} | ||
| cache: 'pip' | ||
| - name: Install system dependencies | ||
| run: | | ||
| sudo apt-get update | ||
| sudo apt-get install -y gcc g++ curl git | ||
| - name: Upgrade pip, setuptools, and wheel | ||
| run: | | ||
| python -m pip install --upgrade pip setuptools wheel build | ||
| - name: Install PyTorch CPU (for faster CI) | ||
| run: | | ||
| pip install --no-cache-dir --default-timeout=600 \ | ||
| --extra-index-url https://download.pytorch.org/whl/cpu \ | ||
| torch==2.1.0+cpu \ | ||
| torchvision==0.16.0+cpu \ | ||
| torchaudio==2.1.0+cpu | ||
| - name: Install project dependencies | ||
| run: | | ||
| # Install package in editable mode first | ||
| pip install --no-cache-dir -e ".[dev]" | ||
| # Install remaining dependencies that aren't in dev extras | ||
| pip install --no-cache-dir \ | ||
| transformers>=4.34.0 \ | ||
| scikit-learn>=1.3.0 \ | ||
| pandas>=2.1.1 \ | ||
| numpy>=1.26.0 \ | ||
| biopython>=1.81 \ | ||
| sentencepiece>=0.1.99 \ | ||
| accelerate>=0.24.0 \ | ||
| datasets>=2.14.0 \ | ||
| google-generativeai>=0.7.2 \ | ||
| tiktoken>=0.5.0 \ | ||
| litellm>=1.50.0 \ | ||
| requests>=2.31.0 \ | ||
| beautifulsoup4>=4.12.2 \ | ||
| lxml>=4.9.0 \ | ||
| openpyxl>=3.1.0 \ | ||
| xlrd>=2.0.1 \ | ||
| tqdm>=4.65.0 \ | ||
| python-dotenv>=1.0.0 \ | ||
| psutil>=5.9.0 \ | ||
| fastapi>=0.104.0 \ | ||
| "uvicorn[standard]>=0.23.2" \ | ||
| aiohttp>=3.8.6 \ | ||
| websockets>=11.0.3 \ | ||
| python-multipart>=0.0.5 \ | ||
| aiofiles>=0.7.0 \ | ||
| pydantic>=2.4.2 \ | ||
| typing-extensions>=3.10.0.2 \ | ||
| starlette>=0.31.1 \ | ||
| click>=8.0.1 \ | ||
| h11>=0.12.0 \ | ||
| httptools>=0.3.0 \ | ||
| PyYAML>=5.4.1 \ | ||
| "watchfiles[watchdog]>=1.0.0" \ | ||
| wsproto>=1.0.0 \ | ||
| tokenizers>=0.14.1 \ | ||
| pytz>=2023.3 \ | ||
| qdrant-client>=1.7.0 | ||
| - name: Install paper-qa from GitHub | ||
| run: | | ||
| # For Python 3.11, install latest paper-qa from GitHub (supports >=5.0.0) | ||
| echo "Installing paper-qa from GitHub for Python 3.11" | ||
| pip install --no-cache-dir git+https://github.com/Future-House/paper-qa.git || \ | ||
| pip install --no-cache-dir "paper-qa>=4.9.0,<5.0.0" | ||
| - name: Verify installation | ||
| env: | ||
| PYTHONPATH: ${{ github.workspace }} | ||
| run: | | ||
| export PYTHONPATH=${{ github.workspace }}:$PYTHONPATH | ||
| python -c "import sys; print('Python version:', sys.version)" | ||
| python -c "import sys; print('Python path:', sys.path)" | ||
| python -c "import app; print('✓ App imported successfully')" || (echo "✗ App import failed" && exit 1) | ||
| python -c "from app.utils.config import GEMINI_API_KEY; print('✓ Config imported')" || echo "⚠ Config import warning" | ||
| - name: Run tests | ||
| env: | ||
| # Set dummy API keys for testing (tests should handle missing keys gracefully) | ||
| GEMINI_API_KEY: "test_key_1234567890abcdef" | ||
| NCBI_API_KEY: "test_ncbi_key_1234567890" | ||
| EMAIL: "test@example.com" | ||
| PYTHONPATH: ${{ github.workspace }} | ||
| run: | | ||
| export PYTHONPATH=${{ github.workspace }}:$PYTHONPATH | ||
| pytest tests/ -v --tb=short --maxfail=10 --cov=app --cov-report=xml --cov-report=term-missing --cov-fail-under=50 | ||
| - name: Upload coverage to Codecov | ||
| uses: codecov/codecov-action@v4 | ||
| with: | ||
| file: ./coverage.xml | ||
| flags: unittests | ||
| name: codecov-umbrella | ||
| fail_ci_if_error: false | ||
| lint: | ||
| name: Lint Code | ||
| runs-on: ubuntu-latest | ||
| steps: | ||
| - name: Checkout code | ||
| uses: actions/checkout@v4 | ||
| - name: Lint with Black and flake8 using Docker | ||
| run: | | ||
| docker run --rm \ | ||
| -v ${{ github.workspace }}:/workspace \ | ||
| -w /workspace \ | ||
| python:3.11-slim \ | ||
| bash -c " | ||
| apt-get update -qq && apt-get install -y -qq gcc > /dev/null 2>&1 && \ | ||
| pip install --quiet --no-cache-dir black>=23.0.0 flake8>=6.0.0 && \ | ||
| echo '=== Running Black (check mode) ===' && \ | ||
| black --check --diff app/ tests/ cli.py main.py && \ | ||
| echo '=== Running flake8 ===' && \ | ||
| flake8 app/ tests/ cli.py main.py --max-line-length=120 --extend-ignore=E203,W503,E501 --exclude=__pycache__,*.pyc | ||
| " | ||
| type-check: | ||
| name: Type Check | ||
| runs-on: ubuntu-latest | ||
| steps: | ||
| - name: Checkout code | ||
| uses: actions/checkout@v4 | ||
| - name: Set up Python | ||
| uses: actions/setup-python@v5 | ||
| with: | ||
| python-version: ${{ env.PYTHON_DEFAULT_VERSION }} | ||
| cache: 'pip' | ||
| - name: Install type checking dependencies | ||
| run: | | ||
| python -m pip install --upgrade pip | ||
| pip install mypy>=1.5.0 types-requests types-PyYAML | ||
| - name: Run mypy | ||
| run: | | ||
| mypy app/ --config-file mypy.ini || (echo "⚠️ Type checking found issues. Review and fix type errors." && exit 1) | ||
| continue-on-error: false # Type checking is now required | ||
| security: | ||
| name: Security Scan | ||
| runs-on: ubuntu-latest | ||
| steps: | ||
| - name: Checkout code | ||
| uses: actions/checkout@v4 | ||
| - name: Set up Python | ||
| uses: actions/setup-python@v5 | ||
| with: | ||
| python-version: ${{ env.PYTHON_DEFAULT_VERSION }} | ||
| cache: 'pip' | ||
| - name: Install security scanning tools | ||
| run: | | ||
| python -m pip install --upgrade pip | ||
| pip install bandit safety | ||
| - name: Run Bandit security scan | ||
| run: | | ||
| echo "Running Bandit security scan..." | ||
| # Generate JSON report | ||
| bandit -r app/ -f json -o bandit-report.json -c .bandit -ll || true | ||
| echo "" | ||
| echo "=== Bandit Security Scan Results ===" | ||
| bandit -r app/ -c .bandit -ll || true | ||
| # Check for high/critical severity issues | ||
| if [ -f bandit-report.json ]; then | ||
| python3 << 'EOF' | ||
| import json | ||
| import sys | ||
| try: | ||
| with open('bandit-report.json', 'r') as f: | ||
| data = json.load(f) | ||
| high_issues = [ | ||
| r for r in data.get('results', []) | ||
| if r.get('issue_severity') in ['HIGH', 'CRITICAL'] | ||
| ] | ||
| if high_issues: | ||
| print(f"❌ Found {len(high_issues)} high/critical severity issues:") | ||
| for issue in high_issues[:5]: | ||
| print(f" - {issue.get('test_id')}: {issue.get('issue_text', '')[:80]}") | ||
| sys.exit(1) | ||
| else: | ||
| print("✅ No high/critical severity issues found.") | ||
| sys.exit(0) | ||
| except Exception as e: | ||
| print(f"⚠️ Could not parse bandit report: {e}") | ||
| sys.exit(0) | ||
| EOF | ||
| fi | ||
| continue-on-error: false # High/critical security issues must be addressed | ||
| - name: Check dependencies with Safety | ||
| run: | | ||
| pip install -e ".[dev]" | ||
| safety check --json | ||
| safety check | ||
| continue-on-error: false # Vulnerable dependencies must be fixed | ||
| docker-build: | ||
| name: Docker Build Test | ||
| runs-on: ubuntu-latest | ||
| steps: | ||
| - name: Checkout code | ||
| uses: actions/checkout@v4 | ||
| - name: Set up Docker Buildx | ||
| uses: docker/setup-buildx-action@v3 | ||
| - name: Build Docker image | ||
| uses: docker/build-push-action@v5 | ||
| with: | ||
| context: . | ||
| file: ./Dockerfile | ||
| push: false | ||
| load: true | ||
| tags: bioanalyzer-backend:test | ||
| cache-from: type=gha | ||
| cache-to: type=gha,mode=max | ||
| - name: Test Docker image | ||
| run: | | ||
| docker run --rm bioanalyzer-backend:test python -c "import app; print('Docker image works!')" | ||
| all-checks: | ||
| name: All Checks Summary | ||
| runs-on: ubuntu-latest | ||
| needs: [test, lint, type-check, security, docker-build] | ||
| if: always() | ||
| steps: | ||
| - name: Check job status | ||
| run: | | ||
| echo "## CI/CD Pipeline Status" >> $GITHUB_STEP_SUMMARY | ||
| echo "" >> $GITHUB_STEP_SUMMARY | ||
| echo "| Job | Status |" >> $GITHUB_STEP_SUMMARY | ||
| echo "|-----|--------|" >> $GITHUB_STEP_SUMMARY | ||
| echo "| Tests | ${{ needs.test.result }} |" >> $GITHUB_STEP_SUMMARY | ||
| echo "| Lint | ${{ needs.lint.result }} |" >> $GITHUB_STEP_SUMMARY | ||
| echo "| Type Check | ${{ needs.type-check.result }} |" >> $GITHUB_STEP_SUMMARY | ||
| echo "| Security | ${{ needs.security.result }} |" >> $GITHUB_STEP_SUMMARY | ||
| echo "| Docker Build | ${{ needs.docker-build.result }} |" >> $GITHUB_STEP_SUMMARY | ||
| echo "" >> $GITHUB_STEP_SUMMARY | ||
| # Fail if any critical check failed | ||
| if [ "${{ needs.test.result }}" != "success" ] || \ | ||
| [ "${{ needs.lint.result }}" != "success" ] || \ | ||
| [ "${{ needs.type-check.result }}" != "success" ] || \ | ||
| [ "${{ needs.security.result }}" != "success" ] || \ | ||
| [ "${{ needs.docker-build.result }}" != "success" ]; then | ||
| echo "❌ One or more critical checks failed!" >> $GITHUB_STEP_SUMMARY | ||
| echo "Test: ${{ needs.test.result }}" >> $GITHUB_STEP_SUMMARY | ||
| echo "Lint: ${{ needs.lint.result }}" >> $GITHUB_STEP_SUMMARY | ||
| echo "Type-check: ${{ needs.type-check.result }}" >> $GITHUB_STEP_SUMMARY | ||
| echo "Security: ${{ needs.security.result }}" >> $GITHUB_STEP_SUMMARY | ||
| echo "Docker-build: ${{ needs.docker-build.result }}" >> $GITHUB_STEP_SUMMARY | ||
| exit 1 | ||
| else | ||
| echo "✅ All checks passed!" >> $GITHUB_STEP_SUMMARY | ||
| fi | ||
