chore: speed-up of CI tests #81
Workflow file for this run
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Backend CI | |
| on: | |
| push: | |
| branches: [main, dev] | |
| paths: | |
| - 'backend/**' | |
| - '.github/workflows/backend-ci.yml' | |
| - 'docker-compose.ci.yaml' | |
| pull_request: | |
| branches: [main, dev] | |
| paths: | |
| - 'backend/**' | |
| - '.github/workflows/backend-ci.yml' | |
| - 'docker-compose.ci.yaml' | |
| workflow_dispatch: | |
| # Pin image versions for cache key consistency | |
| env: | |
| MONGO_IMAGE: mongo:8.0 | |
| REDIS_IMAGE: redis:7-alpine | |
| KAFKA_IMAGE: apache/kafka:3.9.0 | |
| SCHEMA_REGISTRY_IMAGE: confluentinc/cp-schema-registry:7.5.0 | |
| jobs: | |
| unit: | |
| name: Unit Tests | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v6 | |
| - name: Set up uv | |
| uses: astral-sh/setup-uv@v7 | |
| with: | |
| enable-cache: true | |
| cache-dependency-glob: "backend/uv.lock" | |
| - name: Install Python dependencies | |
| run: | | |
| cd backend | |
| uv python install 3.12 | |
| uv sync --frozen | |
| - name: Run unit tests | |
| timeout-minutes: 5 | |
| env: | |
| COVERAGE_CORE: sysmon | |
| run: | | |
| cd backend | |
| uv run pytest tests/unit -v -rs \ | |
| --cov=app --cov-branch \ | |
| --cov-report=xml --cov-report=term | |
| - name: Upload coverage to Codecov | |
| uses: codecov/codecov-action@v5 | |
| if: always() | |
| with: | |
| token: ${{ secrets.CODECOV_TOKEN }} | |
| files: backend/coverage.xml | |
| flags: backend-unit | |
| name: backend-unit-coverage | |
| fail_ci_if_error: false | |
| verbose: true | |
| integration: | |
| name: Integration Tests | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v6 | |
| # ========== DOCKER IMAGE CACHING ========== | |
| - name: Cache Docker images | |
| uses: actions/cache@v5 | |
| id: docker-cache | |
| with: | |
| path: /tmp/docker-cache | |
| key: docker-${{ runner.os }}-${{ env.MONGO_IMAGE }}-${{ env.REDIS_IMAGE }}-${{ env.KAFKA_IMAGE }}-${{ env.SCHEMA_REGISTRY_IMAGE }} | |
| - name: Load cached Docker images | |
| if: steps.docker-cache.outputs.cache-hit == 'true' | |
| run: | | |
| echo "Loading cached images..." | |
| for f in /tmp/docker-cache/*.tar.zst; do | |
| zstd -d -c "$f" | docker load & | |
| done | |
| wait | |
| docker images | |
| - name: Pull and save Docker images | |
| if: steps.docker-cache.outputs.cache-hit != 'true' | |
| run: | | |
| mkdir -p /tmp/docker-cache | |
| echo "Pulling images in parallel..." | |
| docker pull $MONGO_IMAGE & | |
| docker pull $REDIS_IMAGE & | |
| docker pull $KAFKA_IMAGE & | |
| docker pull $SCHEMA_REGISTRY_IMAGE & | |
| wait | |
| echo "Saving images with zstd compression..." | |
| docker save $MONGO_IMAGE | zstd -T0 -3 > /tmp/docker-cache/mongo.tar.zst & | |
| docker save $REDIS_IMAGE | zstd -T0 -3 > /tmp/docker-cache/redis.tar.zst & | |
| docker save $KAFKA_IMAGE | zstd -T0 -3 > /tmp/docker-cache/kafka.tar.zst & | |
| docker save $SCHEMA_REGISTRY_IMAGE | zstd -T0 -3 > /tmp/docker-cache/schema-registry.tar.zst & | |
| wait | |
| echo "Cache size:" | |
| du -sh /tmp/docker-cache/ | |
| # ========================================== | |
| - name: Set up uv | |
| uses: astral-sh/setup-uv@v7 | |
| with: | |
| enable-cache: true | |
| cache-dependency-glob: "backend/uv.lock" | |
| - name: Install Python dependencies | |
| run: | | |
| cd backend | |
| uv python install 3.12 | |
| uv sync --frozen | |
| - name: Start infrastructure services | |
| run: | | |
| docker compose -f docker-compose.ci.yaml up -d --wait --wait-timeout 120 | |
| docker compose -f docker-compose.ci.yaml ps | |
| - name: Run integration tests (non-k8s) | |
| timeout-minutes: 10 | |
| env: | |
| MONGO_ROOT_USER: root | |
| MONGO_ROOT_PASSWORD: rootpassword | |
| MONGODB_HOST: 127.0.0.1 | |
| MONGODB_PORT: 27017 | |
| MONGODB_URL: mongodb://root:[email protected]:27017/?authSource=admin | |
| KAFKA_BOOTSTRAP_SERVERS: localhost:9092 | |
| SCHEMA_REGISTRY_URL: http://localhost:8081 | |
| REDIS_HOST: localhost | |
| REDIS_PORT: 6379 | |
| SCHEMA_SUBJECT_PREFIX: "ci.${{ github.run_id }}." | |
| COVERAGE_CORE: sysmon | |
| run: | | |
| cd backend | |
| uv run pytest tests/integration -v -rs \ | |
| --ignore=tests/integration/k8s \ | |
| --cov=app --cov-branch \ | |
| --cov-report=xml --cov-report=term | |
| - name: Upload coverage to Codecov | |
| uses: codecov/codecov-action@v5 | |
| if: always() | |
| with: | |
| token: ${{ secrets.CODECOV_TOKEN }} | |
| files: backend/coverage.xml | |
| flags: backend-integration | |
| name: backend-integration-coverage | |
| fail_ci_if_error: false | |
| verbose: true | |
| - name: Collect logs | |
| if: failure() | |
| run: | | |
| mkdir -p logs | |
| docker compose -f docker-compose.ci.yaml logs > logs/docker-compose.log 2>&1 | |
| docker compose -f docker-compose.ci.yaml logs kafka > logs/kafka.log 2>&1 | |
| docker compose -f docker-compose.ci.yaml logs schema-registry > logs/schema-registry.log 2>&1 | |
| - name: Upload logs | |
| if: failure() | |
| uses: actions/upload-artifact@v6 | |
| with: | |
| name: backend-logs | |
| path: logs/ | |
| k8s-integration: | |
| name: K8s Integration Tests | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v6 | |
| # ========== DOCKER IMAGE CACHING ========== | |
| - name: Cache Docker images | |
| uses: actions/cache@v5 | |
| id: docker-cache | |
| with: | |
| path: /tmp/docker-cache | |
| key: docker-${{ runner.os }}-${{ env.MONGO_IMAGE }}-${{ env.REDIS_IMAGE }}-${{ env.KAFKA_IMAGE }}-${{ env.SCHEMA_REGISTRY_IMAGE }} | |
| - name: Load cached Docker images | |
| if: steps.docker-cache.outputs.cache-hit == 'true' | |
| run: | | |
| echo "Loading cached images..." | |
| for f in /tmp/docker-cache/*.tar.zst; do | |
| zstd -d -c "$f" | docker load & | |
| done | |
| wait | |
| docker images | |
| - name: Pull and save Docker images | |
| if: steps.docker-cache.outputs.cache-hit != 'true' | |
| run: | | |
| mkdir -p /tmp/docker-cache | |
| echo "Pulling images in parallel..." | |
| docker pull $MONGO_IMAGE & | |
| docker pull $REDIS_IMAGE & | |
| docker pull $KAFKA_IMAGE & | |
| docker pull $SCHEMA_REGISTRY_IMAGE & | |
| wait | |
| echo "Saving images with zstd compression..." | |
| docker save $MONGO_IMAGE | zstd -T0 -3 > /tmp/docker-cache/mongo.tar.zst & | |
| docker save $REDIS_IMAGE | zstd -T0 -3 > /tmp/docker-cache/redis.tar.zst & | |
| docker save $KAFKA_IMAGE | zstd -T0 -3 > /tmp/docker-cache/kafka.tar.zst & | |
| docker save $SCHEMA_REGISTRY_IMAGE | zstd -T0 -3 > /tmp/docker-cache/schema-registry.tar.zst & | |
| wait | |
| echo "Cache size:" | |
| du -sh /tmp/docker-cache/ | |
| # ========================================== | |
| - name: Set up uv | |
| uses: astral-sh/setup-uv@v7 | |
| with: | |
| enable-cache: true | |
| cache-dependency-glob: "backend/uv.lock" | |
| - name: Install Python dependencies | |
| run: | | |
| cd backend | |
| uv python install 3.12 | |
| uv sync --frozen | |
| - name: Start infrastructure services | |
| run: | | |
| docker compose -f docker-compose.ci.yaml up -d --wait --wait-timeout 120 | |
| docker compose -f docker-compose.ci.yaml ps | |
| - name: Setup Kubernetes (k3s) | |
| run: | | |
| curl -sfL https://get.k3s.io | INSTALL_K3S_EXEC="--disable=traefik" sh - | |
| mkdir -p /home/runner/.kube | |
| sudo k3s kubectl config view --raw > /home/runner/.kube/config | |
| sudo chmod 600 /home/runner/.kube/config | |
| export KUBECONFIG=/home/runner/.kube/config | |
| timeout 90 bash -c 'until sudo k3s kubectl cluster-info; do sleep 5; done' | |
| kubectl create namespace integr8scode --dry-run=client -o yaml | kubectl apply -f - | |
| - name: Run k8s integration tests | |
| timeout-minutes: 10 | |
| env: | |
| MONGO_ROOT_USER: root | |
| MONGO_ROOT_PASSWORD: rootpassword | |
| MONGODB_URL: mongodb://root:[email protected]:27017/?authSource=admin | |
| KAFKA_BOOTSTRAP_SERVERS: localhost:9092 | |
| SCHEMA_REGISTRY_URL: http://localhost:8081 | |
| REDIS_HOST: localhost | |
| REDIS_PORT: 6379 | |
| SCHEMA_SUBJECT_PREFIX: "ci.${{ github.run_id }}." | |
| KUBECONFIG: /home/runner/.kube/config | |
| K8S_NAMESPACE: integr8scode | |
| COVERAGE_CORE: sysmon | |
| run: | | |
| cd backend | |
| uv run pytest tests/integration/k8s -v -rs \ | |
| --cov=app --cov-branch \ | |
| --cov-report=xml --cov-report=term | |
| - name: Upload coverage to Codecov | |
| uses: codecov/codecov-action@v5 | |
| if: always() | |
| with: | |
| token: ${{ secrets.CODECOV_TOKEN }} | |
| files: backend/coverage.xml | |
| flags: backend-e2e | |
| name: backend-e2e-coverage | |
| fail_ci_if_error: false | |
| verbose: true | |
| - name: Collect logs | |
| if: failure() | |
| run: | | |
| mkdir -p logs | |
| docker compose -f docker-compose.ci.yaml logs > logs/docker-compose.log 2>&1 | |
| kubectl get events --sort-by='.metadata.creationTimestamp' -A > logs/k8s-events.log 2>&1 || true | |
| kubectl describe pods -A > logs/k8s-describe-pods.log 2>&1 || true | |
| - name: Upload logs | |
| if: failure() | |
| uses: actions/upload-artifact@v6 | |
| with: | |
| name: k8s-logs | |
| path: logs/ |