ci/cd update: all tests in corresponding docker containers

Author: HardMax71

.github/workflows/backend-ci.yml

@@ -76,26 +76,30 @@ jobs:
         with:
           images: ${{ env.MONGO_IMAGE }} ${{ env.REDIS_IMAGE }} ${{ env.KAFKA_IMAGE }} ${{ env.ZOOKEEPER_IMAGE }} ${{ env.SCHEMA_REGISTRY_IMAGE }}
 
-      - name: Set up uv
-        uses: astral-sh/setup-uv@v7
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Build base image
+        uses: docker/build-push-action@v6
         with:
-          enable-cache: true
-          cache-dependency-glob: "backend/uv.lock"
+          context: ./backend
+          file: ./backend/Dockerfile.base
+          load: true
+          tags: base:latest
+          cache-from: type=gha,scope=backend-base
+          cache-to: type=gha,mode=max,scope=backend-base
 
-      - name: Install Python dependencies
-        run: |
-          cd backend
-          uv python install 3.12
-          uv sync --frozen
+      - name: Build backend image
+        run: docker build -t integr8scode-backend:latest -f ./backend/Dockerfile ./backend
 
       - name: Start infrastructure services
         run: ./deploy.sh infra --wait
 
       - name: Run integration tests
         timeout-minutes: 10
         run: |
-          cd backend
-          uv run pytest tests/integration -v -rs \
+          docker compose run --rm -T backend \
+            uv run pytest tests/integration -v -rs \
             --durations=0 \
             --cov=app \
             --cov-report=xml --cov-report=term
@@ -138,17 +142,21 @@ jobs:
         with:
           images: ${{ env.MONGO_IMAGE }} ${{ env.REDIS_IMAGE }} ${{ env.KAFKA_IMAGE }} ${{ env.ZOOKEEPER_IMAGE }} ${{ env.SCHEMA_REGISTRY_IMAGE }}
 
-      - name: Set up uv
-        uses: astral-sh/setup-uv@v7
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Build base image
+        uses: docker/build-push-action@v6
         with:
-          enable-cache: true
-          cache-dependency-glob: "backend/uv.lock"
+          context: ./backend
+          file: ./backend/Dockerfile.base
+          load: true
+          tags: base:latest
+          cache-from: type=gha,scope=backend-base
+          cache-to: type=gha,mode=max,scope=backend-base
 
-      - name: Install Python dependencies
-        run: |
-          cd backend
-          uv python install 3.12
-          uv sync --frozen
+      - name: Build backend image
+        run: docker build -t integr8scode-backend:latest -f ./backend/Dockerfile ./backend
 
       - name: Start infrastructure services
         run: ./deploy.sh infra --wait
@@ -158,19 +166,21 @@ jobs:
           curl -sfL https://get.k3s.io | INSTALL_K3S_EXEC="--disable=traefik" sh -
           mkdir -p /home/runner/.kube
           sudo k3s kubectl config view --raw > /home/runner/.kube/config
-          sudo chmod 600 /home/runner/.kube/config
+          # Use host IP instead of localhost so containers can reach k3s API
+          HOST_IP=$(hostname -I | awk '{print $1}')
+          sed -i "s/127.0.0.1/${HOST_IP}/g" /home/runner/.kube/config
+          sudo chmod 644 /home/runner/.kube/config
           export KUBECONFIG=/home/runner/.kube/config
           timeout 90 bash -c 'until sudo k3s kubectl cluster-info; do sleep 5; done'
           kubectl create namespace integr8scode --dry-run=client -o yaml | kubectl apply -f -
 
       - name: Run E2E tests
         timeout-minutes: 10
-        env:
-          KUBECONFIG: /home/runner/.kube/config
-          K8S_NAMESPACE: integr8scode
         run: |
-          cd backend
-          uv run pytest tests/e2e -v -rs \
+          docker compose run --rm -T \
+            -v /home/runner/.kube/config:/app/kubeconfig.yaml:ro \
+            backend \
+            uv run pytest tests/e2e -v -rs \
             --durations=0 \
             --cov=app \
             --cov-report=xml --cov-report=term

.github/workflows/frontend-ci.yml

@@ -54,12 +54,6 @@ jobs:
     needs: unit
     runs-on: ubuntu-latest
 
-    services:
-      registry:
-        image: registry:2
-        ports:
-          - 5000:5000
-
     env:
       MONGO_IMAGE: mongo:8.0
       REDIS_IMAGE: redis:7-alpine
@@ -92,8 +86,6 @@ jobs:
 
       - name: Setup Docker Buildx
         uses: docker/setup-buildx-action@v3
-        with:
-          driver-opts: network=host
 
       - name: Setup Kubernetes (k3s)
         run: |
@@ -111,32 +103,18 @@ jobs:
             /home/runner/.kube/config > backend/kubeconfig.yaml
           chmod 644 backend/kubeconfig.yaml
 
-      - name: Build and push base image
+      - name: Build base image
         uses: docker/build-push-action@v6
         with:
           context: ./backend
           file: ./backend/Dockerfile.base
-          push: true
-          tags: localhost:5000/integr8scode-base:latest
+          load: true
+          tags: base:latest
           cache-from: type=gha,scope=backend-base
           cache-to: type=gha,mode=max,scope=backend-base
 
-      - name: Load base image to Docker daemon
-        run: |
-          docker pull localhost:5000/integr8scode-base:latest
-          docker tag localhost:5000/integr8scode-base:latest integr8scode-base:latest
-
       - name: Build backend image
-        uses: docker/build-push-action@v6
-        with:
-          context: ./backend
-          file: ./backend/Dockerfile
-          load: true
-          tags: integr8scode-backend:latest
-          build-contexts: |
-            base=docker-image://localhost:5000/integr8scode-base:latest
-          cache-from: type=gha,scope=backend
-          cache-to: type=gha,mode=max,scope=backend
+        run: docker build -t integr8scode-backend:latest -f ./backend/Dockerfile ./backend
 
       - name: Build cert-generator image
         uses: docker/build-push-action@v6
@@ -160,11 +138,11 @@ jobs:
 
       - name: Build worker images
         run: |
-          docker build -t integr8scode-coordinator:latest -f backend/workers/Dockerfile.coordinator --build-context base=docker-image://integr8scode-base:latest backend
-          docker build -t integr8scode-k8s-worker:latest -f backend/workers/Dockerfile.k8s_worker --build-context base=docker-image://integr8scode-base:latest backend
-          docker build -t integr8scode-pod-monitor:latest -f backend/workers/Dockerfile.pod_monitor --build-context base=docker-image://integr8scode-base:latest backend
-          docker build -t integr8scode-result-processor:latest -f backend/workers/Dockerfile.result_processor --build-context base=docker-image://integr8scode-base:latest backend
-          docker build -t integr8scode-saga-orchestrator:latest -f backend/workers/Dockerfile.saga_orchestrator --build-context base=docker-image://integr8scode-base:latest backend
+          docker build -t integr8scode-coordinator:latest -f backend/workers/Dockerfile.coordinator --build-context base=docker-image://base:latest ./backend
+          docker build -t integr8scode-k8s-worker:latest -f backend/workers/Dockerfile.k8s_worker --build-context base=docker-image://base:latest ./backend
+          docker build -t integr8scode-pod-monitor:latest -f backend/workers/Dockerfile.pod_monitor --build-context base=docker-image://base:latest ./backend
+          docker build -t integr8scode-result-processor:latest -f backend/workers/Dockerfile.result_processor --build-context base=docker-image://base:latest ./backend
+          docker build -t integr8scode-saga-orchestrator:latest -f backend/workers/Dockerfile.saga_orchestrator --build-context base=docker-image://base:latest ./backend
 
       - name: Start full stack
         run: ./deploy.sh dev --ci

deploy.sh

@@ -265,27 +265,22 @@ cmd_check() {
 cmd_test() {
     print_header "Running Test Suite"
 
-    print_info "Starting services..."
-    docker compose up -d --build
+    print_info "Building images..."
+    docker build -t base:latest -f ./backend/Dockerfile.base ./backend
+    docker build -t integr8scode-backend:latest -f ./backend/Dockerfile ./backend
 
-    print_info "Waiting for backend to be healthy..."
-    if ! curl --retry 60 --retry-delay 5 --retry-all-errors -ksfo /dev/null https://localhost:443/api/v1/health/live; then
-        print_error "Backend failed to become healthy"
-        docker compose logs
-        exit 1
-    fi
-    print_success "Backend is healthy"
+    print_info "Starting infrastructure..."
+    cmd_infra --wait
 
-    print_info "Running tests..."
-    cd backend
-    if uv run pytest tests/integration tests/unit -v --cov=app --cov-report=term; then
+    print_info "Running tests inside Docker..."
+    if docker compose run --rm -T backend \
+        uv run pytest tests/integration tests/unit -v --cov=app --cov-report=term; then
         print_success "All tests passed!"
         TEST_RESULT=0
     else
         print_error "Tests failed"
         TEST_RESULT=1
     fi
-    cd ..
 
     print_info "Cleaning up..."
     docker compose down

docs/operations/cicd.md

@@ -125,56 +125,60 @@ After each image builds, [Trivy](https://trivy.dev/) scans it for known vulnerab
 dependencies. The scan fails if it finds any critical or high severity issues with available fixes. Results upload to
 GitHub Security for tracking. The backend scan respects `.trivyignore` for acknowledged vulnerabilities.
 
-## Integration tests
+## Backend tests
 
-The integration test workflow is the most complex. It spins up the entire stack on a GitHub Actions runner to verify
-that services work together correctly.
+The backend CI workflow runs three test jobs: unit tests (no infrastructure needed), integration tests (requires
+infrastructure), and E2E tests (requires infrastructure and Kubernetes).
+
+### Integration tests
+
+Integration tests verify that services work together correctly. Tests run inside Docker containers to ensure the same
+environment as production.
 
 ```mermaid
 sequenceDiagram
     participant GHA as GitHub Actions
-    participant K3s as K3s Cluster
     participant Docker as Docker Compose
-    participant Tests as pytest
-
-    GHA->>K3s: Install k3s
-    GHA->>Docker: Pre-pull base images
-    GHA->>Docker: Build services (bake)
-    GHA->>Docker: Start compose stack
-    Docker->>Docker: Wait for health checks
-    GHA->>Tests: Run pytest
-    Tests->>Docker: HTTP requests
-    Tests-->>GHA: Coverage report
+    participant Tests as pytest (in container)
+
+    GHA->>Docker: Build base + backend images
+    GHA->>Docker: Start infrastructure (infra --wait)
+    GHA->>Docker: docker compose run backend pytest
+    Tests->>Docker: Connect to kafka:29092, mongo:27017
+    Tests-->>GHA: Coverage report (via volume mount)
     GHA->>GHA: Upload to Codecov
 ```
 
-The workflow starts by installing [k3s](https://k3s.io/), a lightweight Kubernetes distribution, so the backend can
-interact with a real cluster during tests. It pre-pulls container images in parallel to avoid cold-start delays during
-the build step.
+The workflow builds the base image with GHA layer caching using [docker/build-push-action](https://github.com/docker/build-push-action),
+then builds the backend image on top. Infrastructure services start via `./deploy.sh infra --wait`.
 
-The CI workflow uses `deploy.sh` to start the infrastructure, ensuring consistency between local development and CI
-environments. The `deploy.sh dev --ci` command starts the full stack without observability services (Jaeger, Grafana,
-etc.) and waits for all services to be healthy before proceeding. For backend-only tests, `deploy.sh infra --wait`
-starts just the infrastructure services (MongoDB, Redis, Kafka, Zookeeper, Schema Registry).
+Tests run inside a container using `docker compose run --rm -T backend`, which:
 
-The [docker/bake-action](https://github.com/docker/bake-action) builds all services with GitHub Actions cache support.
-It reads cache layers from previous runs and writes new layers back, so unchanged dependencies don't rebuild. The cache
-scopes are branch-specific with a fallback to main, meaning feature branches benefit from the main branch cache even on
-their first run.
+- Uses the same Docker network as infrastructure services
+- Connects to services using Docker hostnames (`kafka:29092`, `mongo:27017`)
+- Writes coverage reports to the mounted volume for upload
 
-Once images are built, `docker compose up -d` starts the stack. The workflow then uses curl's built-in retry mechanism
-to wait for the backend health endpoint:
+The `.env.test` file contains Docker-internal hostnames, ensuring tests use the same configuration locally and in CI.
+
+### E2E tests
+
+E2E tests require Kubernetes for code execution. The workflow installs [k3s](https://k3s.io/) on the runner and
+configures the kubeconfig to be accessible from inside Docker containers:
 
 ```bash
-curl --retry 60 --retry-delay 5 --retry-all-errors -ksf https://127.0.0.1:443/api/v1/health/live
+# Update kubeconfig to use host IP instead of localhost
+HOST_IP=$(hostname -I | awk '{print $1}')
+sed -i "s/127.0.0.1/${HOST_IP}/g" /home/runner/.kube/config
 ```
 
-This approach is cleaner than shell loops and more reliable than Docker Compose's `--wait` flag (which has issues with
-init containers that exit after completion). The backend's `depends_on` configuration ensures MongoDB, Redis, Kafka,
-and Schema Registry are healthy before backend starts, so waiting for backend health implicitly waits for all
-dependencies. Once the health check passes, the workflow runs pytest against the integration and unit test suites with
-coverage reporting. Test isolation uses
-per-worker database names and schema registry prefixes to avoid conflicts when pytest-xdist runs tests in parallel.
+Tests run inside Docker with the kubeconfig mounted:
+
+```bash
+docker compose run --rm -T \
+  -v /home/runner/.kube/config:/app/kubeconfig.yaml:ro \
+  backend \
+  uv run pytest tests/e2e -v
+```
 
 Coverage reports go to [Codecov](https://codecov.io/) for tracking over time. The workflow always collects container
 logs and Kubernetes events as artifacts, which helps debug failures without reproducing them locally.
@@ -209,17 +213,28 @@ uv run mypy .
 # Security scan
 uv tool run bandit -r . -x tests/ -ll
 
-# Unit tests only (fast)
+# Unit tests only (fast, no infrastructure needed)
 uv run pytest tests/unit -v
+```
 
-# Full integration tests (requires docker compose up)
-uv run pytest tests/integration tests/unit -v
+For integration and E2E tests, use Docker to match the CI environment:
+
+```bash
+# Start infrastructure
+./deploy.sh infra --wait
+
+# Run integration tests inside Docker
+docker compose run --rm -T backend \
+  uv run pytest tests/integration -v
+
+# Run E2E tests (requires k8s configured)
+docker compose run --rm -T \
+  -v ~/.kube/config:/app/kubeconfig.yaml:ro \
+  backend \
+  uv run pytest tests/e2e -v
 ```
 
-For the full integration test experience, start the stack with `docker compose up -d`, wait for the backend to be
-healthy, then run pytest. Alternatively, use `./deploy.sh test` which handles startup, health checks, testing, and
-cleanup automatically. The CI workflow's yq modifications aren't necessary locally since your environment
-likely has the expected configuration already.
+Alternatively, use `./deploy.sh test` which handles startup, health checks, testing, and cleanup automatically.
 
 ## Build optimizations