ci: stabilize pipeline, fix integration tests and docker builds

- CI/CD: Resolved "no space left on device" error by cleaning up runner disk space.
- Tests: Fixed integration tests by mocking Airflow, MLflow, and API startup events.
- Dependencies: Added missing packages (dill, prometheus-client) to requirements.txt.
- Quality: Applied formatting (Black, Isort) and fixed Flake8/MyPy errors.
- Security: Configured correct permissions for Trivy and CodeQL scans.

Author: Western-1

.coverage

@@ -2,7 +2,7 @@ name: CD Pipeline
 
 on:
   push:
-    branches: [ main ]
+    branches: [ main, 'feature/CI-CD-improve' ]
     tags:
       - 'v*.*.*'
 
@@ -26,13 +26,23 @@ jobs:
       - name: Checkout code
         uses: actions/checkout@v3
 
+      - name: Downcase REPO NAME
+        run: |
+          echo "IMAGE_NAME=${GITHUB_REPOSITORY,,}" >> ${GITHUB_ENV}
+
       - name: Log in to Container Registry
         uses: docker/login-action@v2
         with:
           registry: ${{ env.REGISTRY }}
           username: ${{ github.actor }}
           password: ${{ secrets.GITHUB_TOKEN }}
 
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v2
+        with:
+          driver: docker-container
+          driver-opts: network=host
+
       - name: Extract metadata
         id: meta
         uses: docker/metadata-action@v4
@@ -72,12 +82,10 @@ jobs:
         run: |
           echo "Deploying to staging environment..."
           # Add your deployment commands here
-          # Example: kubectl apply -f k8s/staging/
 
       - name: Run smoke tests
         run: |
           echo "Running smoke tests..."
-          # Add smoke test commands
 
   deploy-production:
     name: Deploy to Production
@@ -95,7 +103,6 @@ jobs:
       - name: Deploy to production
         run: |
           echo "Deploying to production environment..."
-          # Add your production deployment commands
 
       - name: Create GitHub Release
         uses: actions/create-release@v1

.github/workflows/cd.yml

@@ -2,7 +2,7 @@ name: CI Pipeline
 
 on:
   push:
-    branches: [ main, develop ]
+    branches: [ main, develop, 'feature/CI-CD-improve' ]
   pull_request:
     branches: [ main ]
 
@@ -16,18 +16,18 @@ jobs:
 
     steps:
       - name: Checkout code
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Set up Python
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
           python-version: ${{ env.PYTHON_VERSION }}
           cache: 'pip'
 
       - name: Install dependencies
         run: |
           python -m pip install --upgrade pip
-          pip install black flake8 mypy isort
+          pip install black flake8 mypy isort types-PyYAML
           pip install -r requirements.txt
 
       - name: Run Black (code formatting)
@@ -63,10 +63,10 @@ jobs:
 
     steps:
       - name: Checkout code
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Set up Python
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
           python-version: ${{ env.PYTHON_VERSION }}
           cache: 'pip'
@@ -94,15 +94,16 @@ jobs:
           DATABASE_URL: postgresql://postgres:postgres@localhost:5432/test_db
 
       - name: Upload coverage to Codecov
-        uses: codecov/codecov-action@v3
+        uses: codecov/codecov-action@v4
         with:
           file: ./coverage.xml
           flags: unittests
           name: codecov-umbrella
           fail_ci_if_error: true
+          token: ${{ secrets.CODECOV_TOKEN }}
 
       - name: Upload coverage report as artifact
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: coverage-report
           path: htmlcov/
@@ -114,10 +115,20 @@ jobs:
 
     steps:
       - name: Checkout code
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
+
+      # 👇 РУЧНЕ ОЧИЩЕННЯ ДИСКА (ПРАЦЮЄ ЗАВЖДИ) 👇
+      - name: Free Disk Space (Manual)
+        run: |
+          sudo rm -rf /usr/share/dotnet
+          sudo rm -rf /usr/local/lib/android
+          sudo rm -rf /opt/ghc
+          sudo rm -rf /opt/hostedtoolcache/CodeQL
+          sudo docker image prune --all --force
+      # 👆 КІНЕЦЬ БЛОКУ 👆
 
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v2
+        uses: docker/setup-buildx-action@v3
 
       - name: Build Airflow image
         run: |
@@ -144,10 +155,14 @@ jobs:
     name: Security Scanning
     runs-on: ubuntu-latest
     needs: lint
+    permissions:
+      contents: read
+      security-events: write
+      actions: read
 
     steps:
       - name: Checkout code
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Run Trivy vulnerability scanner
         uses: aquasecurity/trivy-action@master
@@ -158,7 +173,7 @@ jobs:
           output: 'trivy-results.sarif'
 
       - name: Upload Trivy results to GitHub Security
-        uses: github/codeql-action/upload-sarif@v2
+        uses: github/codeql-action/upload-sarif@v3
         with:
           sarif_file: 'trivy-results.sarif'
 
@@ -168,7 +183,7 @@ jobs:
           bandit -r src/ -f json -o bandit-report.json || true
 
       - name: Upload Bandit report
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: bandit-security-report
           path: bandit-report.json

.github/workflows/ci.yml

@@ -0,0 +1,46 @@
+name: Scheduled Tests
+
+on:
+  schedule:
+    - cron: '0 2 * * *'
+  workflow_dispatch:
+
+jobs:
+  integration-tests:
+    name: Full Integration Tests
+    runs-on: ubuntu-latest
+
+    # FIX: Removed 'services' block entirely.
+    # We are using docker-compose below, which will start Postgres, MinIO, etc.
+    # Defining it here causes a port 5432/9000 conflict.
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v3
+
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.10'
+
+      - name: Start all services
+        run: |
+          docker compose up -d
+          sleep 60  # Wait for services to be ready
+
+      - name: Run full test suite
+        run: |
+          pip install pytest pytest-cov
+          pip install -r requirements.txt
+          # Using localhost because docker-compose mapped ports to host
+          pytest tests/ -v --cov=src --cov-report=xml
+
+      - name: Check service health
+        run: |
+          curl -f http://localhost:8000/health || exit 1
+          curl -f http://localhost:5000/health || exit 1
+          # curl -f http://localhost:8080/health || exit 1 # Airflow might take longer
+
+      - name: Cleanup
+        if: always()
+        run: docker compose down -v

.github/workflows/scheduled.yml

@@ -75,7 +75,7 @@ services:
     networks:
       - churn-network
 
-  # MLflow Server (ТЕПЕР НА SQLITE, ЩОБ НЕ КОНФЛІКТУВАТИ)
+  # MLflow Server
   mlflow:
     build:
       context: .
@@ -91,7 +91,6 @@ services:
     ports:
       - "5000:5000"
     volumes:
-      # Зберігаємо файл бази даних
       - mlflow_data:/mlflow
     command: >
       mlflow server

docker-compose.yml

@@ -36,3 +36,7 @@ python-dateutil
 matplotlib
 cryptography
 dill==0.3.7
+seaborn
+prometheus-client
+prometheus-client
+dill

requirements.txt

@@ -6,61 +6,57 @@
 url = "http://localhost:8000/predict"
 
 
-# 🔴 Генерируем клиента, который ГАРАНТИРОВАННО должен уйти
-# (Максимально плохие условия для удержания)
 def generate_toxic_customer():
     return {
         "gender": "Female",
-        "SeniorCitizen": 1,  # Пенсионеры чаще уходят
-        "Partner": "No",  # Нет привязки к семье
+        "SeniorCitizen": 1,
+        "Partner": "No",
         "Dependents": "No",
-        "tenure": 1,  # Только пришел (1 месяц)
+        "tenure": 1,
         "PhoneService": "Yes",
         "MultipleLines": "Yes",
-        "InternetService": "Fiber optic",  # Самый дорогой и проблемный инет
+        "InternetService": "Fiber optic",
         "OnlineSecurity": "No",
         "OnlineBackup": "No",
         "DeviceProtection": "No",
-        "TechSupport": "No",  # Никакой поддержки
-        "StreamingTV": "Yes",  # Набрал услуг...
-        "StreamingMovies": "Yes",  # ...чтобы чек был огромным
-        "Contract": "Month-to-month",  # Никаких обязательств
+        "TechSupport": "No",
+        "StreamingTV": "Yes",
+        "StreamingMovies": "Yes",
+        "Contract": "Month-to-month",
         "PaperlessBilling": "Yes",
-        "PaymentMethod": "Electronic check",  # Самый "ненадежный" метод
-        "MonthlyCharges": 118.75,  # МАКСИМАЛЬНО возможная цена в датасете
-        "TotalCharges": 118.75,  # Равно месячной, так как 1й месяц
+        "PaymentMethod": "Electronic check",
+        "MonthlyCharges": 118.75,
+        "TotalCharges": 118.75,
     }
 
 
-# 🟢 Идеальный клиент (Лояльный)
 def generate_loyal_customer():
     return {
         "gender": "Male",
         "SeniorCitizen": 0,
         "Partner": "Yes",
         "Dependents": "Yes",
-        "tenure": 72,  # С нами 6 лет (максимум)
+        "tenure": 72,
         "PhoneService": "Yes",
         "MultipleLines": "No",
-        "InternetService": "No",  # Нет интернета - нет проблем
+        "InternetService": "No",
         "OnlineSecurity": "No internet service",
         "OnlineBackup": "No internet service",
         "DeviceProtection": "No internet service",
         "TechSupport": "No internet service",
         "StreamingTV": "No internet service",
         "StreamingMovies": "No internet service",
-        "Contract": "Two year",  # Контракт на 2 года
+        "Contract": "Two year",
         "PaperlessBilling": "No",
         "PaymentMethod": "Mailed check",
-        "MonthlyCharges": 20.0,  # Минимальная цена
+        "MonthlyCharges": 20.0,
         "TotalCharges": 1400.0,
     }
 
 
 print("🚀 Starting EXTREME Load Test...")
 
 while True:
-    # Чередуем: Плохой -> Хороший
     if random.random() > 0.5:
         data = generate_toxic_customer()
         type_cust = "🤬 TOXIC"
@@ -84,4 +80,4 @@ def generate_loyal_customer():
     except Exception as e:
         print(f"🚨 Connection error: {e}")
 
-    time.sleep(0.2)  # Шлем быстро, чтобы забить графики
+    time.sleep(0.2)

scripts/create_test_data.py

@@ -57,7 +57,7 @@ def plot_confusion_matrix(y_true, y_pred, save_path):
             plt.text(
                 j + 0.5,
                 i + 0.7,
-                f"{cm[i, j]/total*100:.1f}%",  # ВИПРАВЛЕНО ТУТ: додано пробіл після коми [i, j]
+                f"{cm[i, j]/total*100:.1f}%",
                 ha="center",
                 va="center",
                 fontsize=10,

scripts/evaluate.py

@@ -3,7 +3,6 @@ set -e
 
 echo "📊 Setting up monitoring stack..."
 
-# 1. Проверяем наличие сети (она должна быть создана основным компоузом)
 NETWORK_NAME="churn-pipeline_churn-network"
 
 if [ -z "$(docker network ls --filter name=^${NETWORK_NAME}$ -q)" ]; then
@@ -14,15 +13,12 @@ else
   echo "✅ Network '${NETWORK_NAME}' found."
 fi
 
-# 2. Удаляем старые контейнеры мониторинга (чтобы избежать конфликтов)
 echo "🧹 Cleaning up old monitoring containers..."
 docker rm -f churn-prometheus churn-grafana churn-alertmanager churn-node-exporter 2>/dev/null || true
 
-# 3. Запускаем
 echo "🚀 Starting monitoring services..."
 docker compose -f monitoring/docker-compose.monitoring.yml up -d
 
-# 4. Ждем
 echo "⏳ Waiting for services to initialize..."
 sleep 10