Updated Docs #23
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: CI/CD Pipeline | ||
|
Check failure on line 1 in .github/workflows/ci-cd.yml
|
||
| on: | ||
| push: | ||
| branches: [ main, develop ] | ||
| pull_request: | ||
| branches: [ main ] | ||
| workflow_dispatch: | ||
| env: | ||
| DOCKER_BUILDKIT: 1 | ||
| COMPOSE_DOCKER_CLI_BUILD: 1 | ||
| PYTHON_VERSION: 3.12.7 | ||
| jobs: | ||
| test: | ||
| name: Test Suite | ||
| runs-on: ubuntu-latest | ||
| services: | ||
| postgres: | ||
| image: postgres:15-alpine | ||
| env: | ||
| POSTGRES_PASSWORD: test_password | ||
| POSTGRES_DB: test_db | ||
| options: >- | ||
| --health-cmd pg_isready | ||
| --health-interval 10s | ||
| --health-timeout 5s | ||
| --health-retries 5 | ||
| ports: | ||
| - 5432:5432 | ||
| redis: | ||
| image: redis:7-alpine | ||
| options: >- | ||
| --health-cmd "redis-cli ping" | ||
| --health-interval 10s | ||
| --health-timeout 5s | ||
| --health-retries 5 | ||
| ports: | ||
| - 6379:6379 | ||
| steps: | ||
| - name: Checkout code | ||
| uses: actions/checkout@v4 | ||
| - name: Set up Python | ||
| uses: actions/setup-python@v4 | ||
| with: | ||
| python-version: ${{ env.PYTHON_VERSION }} | ||
| - name: Cache pip dependencies | ||
| uses: actions/cache@v3 | ||
| with: | ||
| path: ~/.cache/pip | ||
| key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements*.txt') }} | ||
| restore-keys: | | ||
| ${{ runner.os }}-pip- | ||
| - name: Install dependencies | ||
| run: | | ||
| python -m pip install --upgrade pip | ||
| pip install -r requirements.txt | ||
| pip install -r requirements-dev.txt | ||
| pip install pytest pytest-asyncio pytest-cov | ||
| - name: Create test environment file | ||
| run: | | ||
| cat > .env.test << EOF | ||
| DATABASE_URL=postgresql://postgres:test_password@localhost:5432/test_db | ||
| REDIS_URL=redis://localhost:6379 | ||
| SECRET_KEY=test_secret_key_for_testing_only | ||
| ENABLE_API_KEYS=true | ||
| LOG_LEVEL=INFO | ||
| TESTING=true | ||
| EOF | ||
| - name: Run database migrations | ||
| run: | | ||
| export $(cat .env.test | xargs) | ||
| alembic upgrade head | ||
| - name: Run tests with coverage | ||
| run: | | ||
| export $(cat .env.test | xargs) | ||
| pytest --cov=api --cov-report=xml --cov-report=html --cov-report=term-missing -v | ||
| - name: Upload coverage to Codecov | ||
| uses: codecov/codecov-action@v3 | ||
| with: | ||
| file: ./coverage.xml | ||
| flags: unittests | ||
| name: codecov-umbrella | ||
| fail_ci_if_error: false | ||
| - name: Generate coverage report | ||
| run: | | ||
| echo "## Test Coverage Report" >> $GITHUB_STEP_SUMMARY | ||
| echo "$(coverage report)" >> $GITHUB_STEP_SUMMARY | ||
| - name: Archive test results | ||
| uses: actions/upload-artifact@v3 | ||
| if: always() | ||
| with: | ||
| name: test-results | ||
| path: | | ||
| htmlcov/ | ||
| coverage.xml | ||
| pytest-report.xml | ||
| lint: | ||
| name: Code Quality | ||
| runs-on: ubuntu-latest | ||
| steps: | ||
| - name: Checkout code | ||
| uses: actions/checkout@v4 | ||
| - name: Set up Python | ||
| uses: actions/setup-python@v4 | ||
| with: | ||
| python-version: ${{ env.PYTHON_VERSION }} | ||
| - name: Install linting tools | ||
| run: | | ||
| python -m pip install --upgrade pip | ||
| pip install black flake8 mypy isort bandit safety | ||
| - name: Run Black (code formatting) | ||
| run: black --check --diff api/ tests/ | ||
| - name: Run isort (import sorting) | ||
| run: isort --check-only --diff api/ tests/ | ||
| - name: Run flake8 (linting) | ||
| run: flake8 api/ tests/ | ||
| - name: Run mypy (type checking) | ||
| run: mypy api/ | ||
| - name: Run bandit (security) | ||
| run: bandit -r api/ | ||
| - name: Run safety (dependency security) | ||
| run: safety check | ||
| build: | ||
| name: Build Docker Images | ||
| runs-on: ubuntu-latest | ||
| needs: [test, lint] | ||
| strategy: | ||
| matrix: | ||
| component: [api, worker-cpu, worker-gpu] | ||
| include: | ||
| - component: api | ||
| dockerfile: docker/api/Dockerfile | ||
| context: . | ||
| build_args: "PYTHON_VERSION=${{ env.PYTHON_VERSION }}" | ||
| - component: worker-cpu | ||
| dockerfile: docker/worker/Dockerfile | ||
| context: . | ||
| build_args: "PYTHON_VERSION=${{ env.PYTHON_VERSION }}\nWORKER_TYPE=cpu" | ||
| - component: worker-gpu | ||
| dockerfile: docker/worker/Dockerfile | ||
| context: . | ||
| build_args: "PYTHON_VERSION=${{ env.PYTHON_VERSION }}\nWORKER_TYPE=gpu" | ||
| steps: | ||
| - name: Checkout code | ||
| uses: actions/checkout@v4 | ||
| - name: Set up Docker Buildx | ||
| uses: docker/setup-buildx-action@v3 | ||
| - name: Login to Container Registry | ||
| if: github.ref == 'refs/heads/main' | ||
| uses: docker/login-action@v3 | ||
| with: | ||
| registry: ghcr.io | ||
| username: ${{ github.actor }} | ||
| password: ${{ secrets.GITHUB_TOKEN }} | ||
| - name: Extract metadata | ||
| id: meta | ||
| uses: docker/metadata-action@v5 | ||
| with: | ||
| images: ghcr.io/${{ github.repository }}/${{ matrix.component }} | ||
| tags: | | ||
| type=ref,event=branch | ||
| type=ref,event=pr | ||
| type=sha,prefix={{branch}}- | ||
| type=raw,value=latest,enable={{is_default_branch}} | ||
| - name: Build and push | ||
| uses: docker/build-push-action@v5 | ||
| with: | ||
| context: ${{ matrix.context }} | ||
| file: ${{ matrix.dockerfile }} | ||
| build-args: ${{ matrix.build_args }} | ||
| push: ${{ github.ref == 'refs/heads/main' }} | ||
| tags: ${{ steps.meta.outputs.tags }} | ||
| labels: ${{ steps.meta.outputs.labels }} | ||
| cache-from: type=gha | ||
| cache-to: type=gha,mode=max | ||
| platforms: linux/amd64,linux/arm64 | ||
| security-scan: | ||
| name: Security Scan | ||
| runs-on: ubuntu-latest | ||
| needs: build | ||
| if: github.ref == 'refs/heads/main' | ||
| steps: | ||
| - name: Checkout code | ||
| uses: actions/checkout@v4 | ||
| - name: Run Trivy vulnerability scanner | ||
| uses: aquasecurity/trivy-action@master | ||
| with: | ||
| image-ref: ghcr.io/${{ github.repository }}/api:latest | ||
| format: 'sarif' | ||
| output: 'trivy-results.sarif' | ||
| - name: Upload Trivy scan results | ||
| uses: github/codeql-action/upload-sarif@v2 | ||
| with: | ||
| sarif_file: 'trivy-results.sarif' | ||
| integration-test: | ||
| name: Integration Tests | ||
| runs-on: ubuntu-latest | ||
| needs: build | ||
| steps: | ||
| - name: Checkout code | ||
| uses: actions/checkout@v4 | ||
| - name: Set up Docker Buildx | ||
| uses: docker/setup-buildx-action@v3 | ||
| - name: Build test environment | ||
| run: | | ||
| docker-compose -f docker-compose.yml -f docker-compose.test.yml build | ||
| - name: Run integration tests | ||
| run: | | ||
| docker-compose -f docker-compose.yml -f docker-compose.test.yml up -d | ||
| sleep 30 | ||
| # Run API health check | ||
| curl -f http://localhost:8000/api/v1/health || exit 1 | ||
| # Run basic API tests | ||
| python -m pytest tests/integration/ -v | ||
| - name: Cleanup | ||
| if: always() | ||
| run: | | ||
| docker-compose -f docker-compose.yml -f docker-compose.test.yml down -v | ||
| deploy-staging: | ||
| name: Deploy to Staging | ||
| runs-on: ubuntu-latest | ||
| needs: [test, lint, build, integration-test] | ||
| if: github.ref == 'refs/heads/develop' | ||
| environment: staging | ||
| steps: | ||
| - name: Checkout code | ||
| uses: actions/checkout@v4 | ||
| - name: Deploy to staging | ||
| run: | | ||
| echo "Deploying to staging environment..." | ||
| # Add deployment commands here | ||
| # Example: kubectl apply -f k8s/staging/ | ||
| - name: Run staging tests | ||
| run: | | ||
| echo "Running staging tests..." | ||
| # Add staging test commands here | ||
| - name: Notify deployment | ||
| if: always() | ||
| run: | | ||
| echo "Staging deployment completed" | ||
| deploy-production: | ||
| name: Deploy to Production | ||
| runs-on: ubuntu-latest | ||
| needs: [test, lint, build, integration-test, security-scan] | ||
| if: github.ref == 'refs/heads/main' | ||
| environment: production | ||
| steps: | ||
| - name: Checkout code | ||
| uses: actions/checkout@v4 | ||
| - name: Deploy to production | ||
| run: | | ||
| echo "Deploying to production environment..." | ||
| # Add production deployment commands here | ||
| # Example: kubectl apply -f k8s/production/ | ||
| - name: Run production smoke tests | ||
| run: | | ||
| echo "Running production smoke tests..." | ||
| # Add production smoke test commands here | ||
| - name: Create deployment issue | ||
| if: failure() | ||
| uses: actions/github-script@v6 | ||
| with: | ||
| script: | | ||
| github.rest.issues.create({ | ||
| owner: context.repo.owner, | ||
| repo: context.repo.repo, | ||
| title: 'Production deployment failed', | ||
| body: 'Production deployment failed. Please check the logs and take necessary action.', | ||
| labels: ['bug', 'production', 'deployment'] | ||
| }) | ||
| - name: Notify deployment | ||
| if: always() | ||
| run: | | ||
| echo "Production deployment completed" | ||
| backup-database: | ||
| name: Database Backup | ||
| runs-on: ubuntu-latest | ||
| if: github.ref == 'refs/heads/main' | ||
| steps: | ||
| - name: Checkout code | ||
| uses: actions/checkout@v4 | ||
| - name: Run database backup | ||
| run: | | ||
| echo "Running database backup..." | ||
| # Add database backup commands here | ||
| # Example: ./scripts/backup-database.sh | ||
| - name: Upload backup artifacts | ||
| uses: actions/upload-artifact@v3 | ||
| with: | ||
| name: database-backup | ||
| path: backups/ | ||
| retention-days: 7 | ||
| performance-test: | ||
| name: Performance Tests | ||
| runs-on: ubuntu-latest | ||
| needs: build | ||
| if: github.ref == 'refs/heads/main' | ||
| steps: | ||
| - name: Checkout code | ||
| uses: actions/checkout@v4 | ||
| - name: Run performance tests | ||
| run: | | ||
| echo "Running performance tests..." | ||
| # Add performance test commands here | ||
| # Example: locust -f tests/performance/locustfile.py | ||
| - name: Generate performance report | ||
| run: | | ||
| echo "Generating performance report..." | ||
| # Add performance report generation here | ||
| notify: | ||
| name: Notify Results | ||
| runs-on: ubuntu-latest | ||
| needs: [test, lint, build, integration-test] | ||
| if: always() | ||
| steps: | ||
| - name: Notify success | ||
| if: needs.test.result == 'success' && needs.lint.result == 'success' && needs.build.result == 'success' | ||
| run: | | ||
| echo "All CI/CD jobs completed successfully!" | ||
| - name: Notify failure | ||
| if: needs.test.result == 'failure' || needs.lint.result == 'failure' || needs.build.result == 'failure' | ||
| run: | | ||
| echo "Some CI/CD jobs failed. Please check the logs." | ||
| exit 1 | ||
