v2.1: 80% coverage, updated tests, updated readmes

Author: Max Azatian

.github/workflows/tests.yml

@@ -124,7 +124,7 @@ jobs:
         run: |
           cd backend
           echo "Using BACKEND_BASE_URL=$BACKEND_BASE_URL"
-          python -m pytest tests/integration tests/unit -v --cov=app --cov-report=xml --cov-report=term
+          python -m pytest tests/integration tests/unit -v --cov=app --cov-branch --cov-report=xml --cov-report=term --cov-report=term-missing
 
       - name: Upload coverage to Codecov
         uses: codecov/codecov-action@v5

ARCHITECTURE_IN_DETAILS.md

@@ -7,12 +7,8 @@ This document sketches the system as it actually exists in this repo, using ASCI
 
 ```plantuml
 @startuml
-skinparam monochrome true
-skinparam shadowing false
-
 rectangle "Public Internet\n(Browser SPA)" as Browser
 rectangle "Frontend\n(Nginx + Svelte)" as Frontend
-
 node "Backend" as Backend {
   [FastAPI / Uvicorn\n(routers, Dishka DI, middlewares)] as FastAPI
   [SSE Service\n(Partitioned router + Redis bus)] as SSE
@@ -21,12 +17,11 @@ node "Backend" as Backend {
   cloud "Kafka" as Kafka
   [Schema Registry] as Schema
   cloud "Kubernetes API" as K8s
-  [Prometheus] as Prom
+  [OTel Collector] as OTel
+  [VictoriaMetrics] as VM
   [Jaeger] as Jaeger
 }
-
 rectangle "Cert Generator\n(setup-k8s.sh, TLS)" as CertGen
-
 Browser --> Frontend : HTTPS 443\nSPA + static assets
 Frontend --> FastAPI : HTTPS /api/v1/*\nCookies/CSRF
 FastAPI <--> SSE     : /api/v1/events/*\nJSON frames
@@ -36,10 +31,10 @@ FastAPI --> Kafka    : UnifiedProducer\n(events)
 Kafka --> FastAPI    : UnifiedConsumer\n(dispatch)
 Kafka -- Schema
 FastAPI <--> K8s     : pod create/monitor\nworker + pod monitor
-FastAPI --> Prom     : metrics (pull)
+FastAPI --> OTel     : metrics/traces (export)
+OTel --> VM          : remote_write (metrics)
 FastAPI --> Jaeger   : traces (export)
 CertGen .. K8s       : cluster setup / certs
-
 @enduml
 ```
 
@@ -76,7 +71,7 @@ Frontend serves the SPA; the SPA calls FastAPI over HTTPS. Backend exposes REST
 *** /admin/users (api/routes/admin/users.py)
 *** /admin/events (api/routes/admin/events.py)
 *** /admin/settings (api/routes/admin/settings.py)
-*** /alertmanager (api/routes/alertmanager.py)
+*** /alerts (api/routes/grafana_alerts.py)
 ** DI & Providers (Dishka)
 *** Container (core/container.py, core/providers.py)
 *** Exception handlers (core/exceptions/handlers.py)
@@ -89,7 +84,7 @@ Frontend serves the SPA; the SPA calls FastAPI over HTTPS. Backend exposes REST
 *** SSEService (services/sse/sse_service.py)
 **** SSERedisBus, PartitionedSSERouter, SSEShutdownManager, EventBuffer
 *** NotificationService (services/notification_service.py)
-**** UnifiedConsumer handlers (completed/failed/timeout), templates, throttle
+**** UnifiedConsumer handlers (completed/failed/timeout), SSE, throttle
 *** UserSettingsService (services/user_settings_service.py)
 **** LRU cache, USER_* events to EventStore/Kafka
 *** SavedScriptService (services/saved_script_service.py)
@@ -134,7 +129,7 @@ Frontend serves the SPA; the SPA calls FastAPI over HTTPS. Backend exposes REST
 *** Redis (rate limit, SSE bus)
 *** Kafka + Schema Registry
 *** Kubernetes API (pods)
-*** Prometheus (metrics)
+*** OTel Collector + VictoriaMetrics (metrics)
 *** Jaeger (traces)
 ** Settings (app/settings.py)
 *** Runtimes/limits, Kafka/Redis/Mongo endpoints, SSE, rate limiting
@@ -266,7 +261,7 @@ Sagas use explicit DI (no context-based injection). Only serializable public dat
             v
   NotificationService (private)
      |-- UnifiedConsumer (typed handlers for completed/failed/timeout)
-     |-- Repository: templates, notifications (Mongo)
+     |-- Repository: notifications + subscriptions (Mongo)
      |-- Channels:
      |     - IN_APP: persist + publish SSE bus (Redis)
      |     - WEBHOOK: httpx POST
@@ -331,7 +326,7 @@ Saved scripts are simple CRUD per user. User settings are reconstructed from sna
 ## DLQ and admin tooling
 
 ```
-  Kafka DLQ topic <-> DLQ consumer/manager (retry/backoff, thresholds)
+  Kafka DLQ topic <-> DLQ manager (retry/backoff, thresholds)
   /api/v1/admin/events/* -> admin repos (Mongo) for events query/delete
   /api/v1/admin/users/* -> users repo (Mongo) + rate limit config
   /api/v1/admin/settings/* -> system settings (Mongo)

README.md

@@ -54,7 +54,7 @@ things safe and efficient. You'll get the results back in no time.
 - Backend: `https://127.0.0.1:443/`
   - To check if it works, you can use `curl -k https://127.0.0.1/api/v1/k8s-limits`, should return JSON with current limits
 - Grafana: `http://127.0.0.1:3000` (login - `admin`, pw - `admin123`)
-- Prometheus: `http://127.0.0.1:9090/targets` (`integr8scode` must be `1/1 up`)
+  
 
 You may also find out that k8s doesn't capture metrics (`CPU` and `Memory` params are `null`), it may well be that metrics server
 for k8s is turned off/not enabled. To enable, execute:
@@ -175,6 +175,5 @@ The platform is built on three main pillars:
 - **Monitoring Tools**: Using OpenTelemetry and Grafana to keep an eye on system health.
 - **Alerts**: Set up notifications for when things go wrong.
 
-Link for accessing Prometheus is shown in `/editor` web page. 
-
+  
 

backend/.env.test

@@ -1,42 +1,39 @@
 # Test environment configuration
-# This file is loaded by tests/conftest.py for integration tests
-
-# MongoDB Configuration
-MONGODB_URL="mongodb://localhost:27017"
-PROJECT_NAME="integr8scode_test"
-
-# Redis Configuration  
-REDIS_URL="redis://localhost:6379/0"
-
-# Authentication
-SECRET_KEY="test-secret-key-for-testing-only"
-JWT_SECRET_KEY="test-jwt-secret-key-for-testing-only"
-JWT_ALGORITHM="HS256"
-ACCESS_TOKEN_EXPIRE_MINUTES=30
-
-# Rate Limiting - DISABLED for tests
-RATE_LIMIT_ENABLED=false
-RATE_LIMIT_DEFAULT_REQUESTS=1000
-RATE_LIMIT_DEFAULT_WINDOW=1
+PROJECT_NAME=integr8scode_test
+API_V1_STR=/api/v1
+SECRET_KEY=test-secret-key-for-testing-only-32chars!!
+ENVIRONMENT=testing
+TESTING=true
 
-# Disable tracing for tests
+# MongoDB - use localhost for tests
+MONGODB_URL=mongodb://root:rootpassword@localhost:27017/?authSource=admin
+MONGO_ROOT_USER=root
+MONGO_ROOT_PASSWORD=rootpassword
+
+# Redis - use localhost for tests
+REDIS_HOST=localhost
+REDIS_PORT=6379
+REDIS_DB=0
+REDIS_PASSWORD=
+REDIS_SSL=false
+REDIS_MAX_CONNECTIONS=50
+REDIS_DECODE_RESPONSES=true
+
+# Kafka - use localhost for tests
+KAFKA_BOOTSTRAP_SERVERS=localhost:9092
+SCHEMA_REGISTRY_URL=http://localhost:8081
+
+# Security
+SECURE_COOKIES=true
+CORS_ALLOWED_ORIGINS=["http://localhost:3000","https://localhost:3000"]
+
+# Features
+RATE_LIMIT_ENABLED=true
 ENABLE_TRACING=false
 OTEL_SDK_DISABLED=true
 OTEL_METRICS_EXPORTER=none
 OTEL_TRACES_EXPORTER=none
 
-# API Settings
-BACKEND_BASE_URL="https://[::1]:443"
-BACKEND_CORS_ORIGINS=["http://localhost:3000", "http://localhost:5173"]
-
-# Kafka Configuration (minimal for tests)
-KAFKA_BOOTSTRAP_SERVERS="localhost:9092"
-KAFKA_SECURITY_PROTOCOL="PLAINTEXT"
-
-# Kubernetes Configuration (mocked in tests)
-K8S_IN_CLUSTER=false
-K8S_NAMESPACE="default"
-
-# Test Mode
-TESTING=true
-DEBUG=false
+# Development
+DEVELOPMENT_MODE=false
+LOG_LEVEL=INFO

backend/Dockerfile.test

@@ -0,0 +1,25 @@
+# Test runner container - lightweight, uses same network as services
+FROM python:3.12-slim
+
+WORKDIR /app
+
+# Install system dependencies
+RUN apt-get update && apt-get install -y \
+    gcc \
+    curl \
+    && rm -rf /var/lib/apt/lists/*
+
+# Copy requirements
+COPY requirements.txt requirements-dev.txt ./
+
+# Install Python dependencies
+RUN pip install --no-cache-dir -r requirements.txt -r requirements-dev.txt
+
+# Copy application code
+COPY . .
+
+# Set Python path
+ENV PYTHONPATH=/app
+
+# Default command runs all tests
+CMD ["pytest", "-v", "--tb=short"]

backend/alertmanager/alertmanager.yml

@@ -1,100 +1,24 @@
-from typing import Optional
-
 from dishka import FromDishka
 from dishka.integrations.fastapi import inject
-from fastapi import HTTPException, Request, status
-
-from app.core.logging import logger
-from app.core.security import security_service
-from app.db.repositories.user_repository import UserRepository
-from app.domain.enums.user import UserRole
-from app.schemas_pydantic.user import User, UserResponse
-
-
-class AuthService:
-    def __init__(self, user_repo: UserRepository):
-        self.user_repo = user_repo
-
-    async def get_current_user(self, request: Request) -> UserResponse:
-        try:
-            token = request.cookies.get("access_token")
-            if not token:
-                raise HTTPException(
-                    status_code=status.HTTP_401_UNAUTHORIZED,
-                    detail="Not authenticated",
-                    headers={"WWW-Authenticate": "Bearer"},
-                )
+from fastapi import Request
 
-            user_in_db = await security_service.get_current_user(token, self.user_repo)
-
-            return UserResponse(
-                user_id=user_in_db.user_id,
-                username=user_in_db.username,
-                email=user_in_db.email,
-                role=user_in_db.role,
-                is_superuser=user_in_db.is_superuser,
-                created_at=user_in_db.created_at,
-                updated_at=user_in_db.updated_at
-            )
-        except Exception as e:
-            logger.error(f"Authentication failed: {e}", exc_info=True)
-            raise HTTPException(
-                status_code=status.HTTP_401_UNAUTHORIZED,
-                detail="Not authenticated",
-                headers={"WWW-Authenticate": "Bearer"},
-            ) from e
-
-    async def require_admin(self, request: Request) -> UserResponse:
-        user = await self.get_current_user(request)
-        if user.role != UserRole.ADMIN:
-            logger.warning(
-                f"Admin access denied for user: {user.username} (role: {user.role})"
-            )
-            raise HTTPException(
-                status_code=status.HTTP_403_FORBIDDEN,
-                detail="Admin access required"
-            )
-        return user
-
-
-@inject
-async def require_auth_guard(
-        request: Request,
-        auth_service: FromDishka[AuthService],
-) -> None:
-    await auth_service.get_current_user(request)
+from app.schemas_pydantic.user import UserResponse
+from app.services.auth_service import AuthService
 
 
 @inject
-async def require_admin_guard(
-        request: Request,
-        auth_service: FromDishka[AuthService],
-) -> None:
-    await auth_service.require_admin(request)
+async def CurrentUser(
+    request: Request,
+    auth_service: FromDishka[AuthService]
+) -> UserResponse:
+    """Get authenticated user."""
+    return await auth_service.get_current_user(request)
 
 
 @inject
-async def get_current_user_optional(
-        request: Request,
-        auth_service: FromDishka[AuthService],
-) -> Optional[User]:
-    """
-    Get current user if authenticated, otherwise return None.
-    This is used for optional authentication, like rate limiting.
-    """
-    try:
-        user_response = await auth_service.get_current_user(request)
-        # Convert UserResponse to User for compatibility
-        return User(
-            user_id=user_response.user_id,
-            username=user_response.username,
-            email=user_response.email,
-            role=user_response.role,
-            is_active=True,  # If they can authenticate, they're active
-            is_superuser=user_response.is_superuser,
-            created_at=user_response.created_at,
-            updated_at=user_response.updated_at
-        )
-    except HTTPException:
-        # User is not authenticated, return None
-        return None
+async def AdminUser(
+    request: Request,
+    auth_service: FromDishka[AuthService]
+) -> UserResponse:
+    """Get authenticated admin user."""
+    return await auth_service.get_admin(request)