feat: implemented AI for log analysis for alert and recommendation

Author: Nazzcodek

.dockerignore

@@ -30,8 +30,8 @@ ENV/
 *~
 
 # Environment
-.env
-.env.local
+# .env
+# .env.local
 
 # Documentation
 *.md

.env.example

@@ -1,85 +1,35 @@
 # Application Settings
-APP_NAME=log-analysis-service
-APP_VERSION=1.0.0
 ENVIRONMENT=development
-DEBUG=True
+DEBUG=true
 LOG_LEVEL=INFO
 
-# Server Configuration
-HOST=0.0.0.0
-PORT=8000
-WORKERS=4
-
-# Database Configuration
+# Database
 DATABASE_URL=postgresql+asyncpg://postgres:password@localhost:5432/log_analysis_db
-DATABASE_POOL_SIZE=20
-DATABASE_MAX_OVERFLOW=10
-DATABASE_POOL_TIMEOUT=30
-DATABASE_POOL_RECYCLE=3600
 POSTGRES_USER=postgres
 POSTGRES_PASSWORD=password
 POSTGRES_DB=log_analysis_db
-POSTGRES_HOST=localhost
-POSTGRES_PORT=5432
-
 
-# RabbitMQ Configuration
+# RabbitMQ
 RABBITMQ_HOST=localhost
 RABBITMQ_PORT=5672
 RABBITMQ_USER=guest
 RABBITMQ_PASSWORD=guest
 RABBITMQ_VHOST=/
-RABBITMQ_PREFETCH_COUNT=10
-RABBITMQ_RECONNECT_DELAY=5
-RABBITMQ_ERLANG_COOKIE=your-secret-cookie-change-this
-
-# Queue Names
-LOG_ANALYSIS_QUEUE=log_analysis_queue
-RECOMMENDATION_QUEUE=recommendation_queue
-ALERTS_QUEUE=alerts_queue
+RABBITMQ_ERLANG_COOKIE=secret_cookie_change_me
 
-# JWT Authentication
+# Security
 JWT_SECRET_KEY=your-super-secret-jwt-key-change-this-in-production
 JWT_ALGORITHM=HS256
-JWT_EXPIRATION_MINUTES=60
-API_GATEWAY_PUBLIC_KEY=
-
-# Processing Configuration
-BATCH_SIZE=100
-PROCESSING_INTERVAL_SECONDS=5
-MAX_CONCURRENT_BATCHES=5
-
-# Anomaly Detection Thresholds
-ERROR_RATE_THRESHOLD=10.0
-ANOMALY_SCORE_THRESHOLD=0.85
-MIN_PATTERN_FREQUENCY=5
-
-# Metrics & Monitoring
-PROMETHEUS_PORT=9090
-ENABLE_METRICS=True
-METRICS_PATH=/metrics
-
-# Rate Limiting
-RATE_LIMIT_ENABLED=True
-RATE_LIMIT_REQUESTS=100
-RATE_LIMIT_PERIOD=60
-
-# CORS Settings
-CORS_ENABLED=True
-CORS_ORIGINS=["http://localhost:3000", "http://localhost:8080"]
-CORS_METHODS=["GET", "POST", "PUT", "DELETE", "OPTIONS"]
-CORS_HEADERS=["*"]
-
-# Security
-ALLOWED_HOSTS=["localhost", "127.0.0.1"]
-TRUSTED_PROXIES=[]
-
-# Retry Configuration
-MAX_RETRY_ATTEMPTS=3
-RETRY_BACKOFF_FACTOR=2
-RETRY_MAX_DELAY_SECONDS=60
 
-# Feature Flags
-ENABLE_REAL_TIME_PROCESSING=True
-ENABLE_PATTERN_CLUSTERING=True
-ENABLE_ML_ANOMALY_DETECTION=True
+# AI Services (Optional)
+ENABLE_OPENAI_ANALYSIS=false
+OPENAI_API_KEY=sk-your-api-key-here
+OPENAI_MODEL=gpt-4-turbo-preview
+AI_ANALYSIS_MIN_SEVERITY=ERROR
+
+# Groq AI Configuration (Recommended - Fast & Affordable)
+ENABLE_GROQ_ANALYSIS=false
+GROQ_API_KEY=gsk_your_api_key_here
+GROQ_MODEL=llama-3.1-70b-versatile
+GROQ_TEMPERATURE=0.3
+GROQ_MAX_TOKENS=1000

.gitignore

@@ -205,3 +205,5 @@ cython_debug/
 marimo/_static/
 marimo/_lsp/
 __marimo__/
+
+*.md

Dockerfile

@@ -1,6 +1,6 @@
 # Multi-stage build for smaller image size
 
-FROM python:3.11-slim as builder
+FROM python:3.11-slim AS builder
 
 WORKDIR /build
 

Makefile

@@ -1,54 +1,219 @@
-.PHONY: help install dev test lint format clean docker-build docker-up docker-down migrate
+# Log Analysis Service - Makefile
 
+.PHONY: help start stop restart test clean logs status
+
+# Default target
 help:
-	@echo "Log Analysis Service - Available commands:"
-	@echo "  install       - Install dependencies"
-	@echo "  dev           - Run development server"
-	@echo "  test          - Run tests"
-	@echo "  lint          - Run linters"
-	@echo "  format        - Format code"
-	@echo "  clean         - Clean generated files"
-	@echo "  docker-build  - Build Docker image"
-	@echo "  docker-up     - Start Docker services"
-	@echo "  docker-down   - Stop Docker services"
-	@echo "  migrate       - Run database migrations"
-
-install:
-	pip install -r requirements.txt
+	@echo "Log Analysis Service - Available Commands:"
+	@echo ""
+	@echo "🚀 Service Management:"
+	@echo "  start     - Start all services with Docker Compose"
+	@echo "  stop      - Stop all services"
+	@echo "  restart   - Restart all services"
+	@echo "  logs      - View application logs"
+	@echo "  status    - Check service status"
+	@echo "  clean     - Clean up containers and volumes"
+	@echo ""
+	@echo "🔑 Authentication:"
+	@echo "  get-token      - Generate admin authentication token"
+	@echo "  get-token-user - Generate user authentication token"
+	@echo "  get-token-admin- Generate admin authentication token"
+	@echo ""
+	@echo "⚙️  Configuration:"
+	@echo "  get-config     - Show application configuration"
+	@echo ""
+	@echo "🧪 Testing:"
+	@echo "  test-logs      - Run comprehensive log processing tests"
+	@echo "  test-critical   - Test critical error scenarios"
+	@echo "  test-pattern    - Test pattern detection"
+	@echo "  test-anomaly    - Test anomaly detection"
+	@echo "  test-api-status - Test AI status endpoint"
+	@echo "  test-api-logs   - Test logs endpoint"
+	@echo "  test-api-stats  - Test stats endpoint"
+	@echo ""
+	@echo "📊 Monitoring:"
+	@echo "  check-logs-db   - View recent logs in database"
+	@echo "  check-patterns  - View detected patterns"
+	@echo "  check-queues    - View RabbitMQ queue status"
+	@echo "  metrics         - View Prometheus metrics"
+	@echo ""
 
-dev:
-	uvicorn app.main:app --reload --host 0.0.0.0 --port 8000
+# Start all services
+start:
+	@echo "🚀 Starting Log Analysis Service..."
+	docker compose up -d
+	@echo "⏳ Waiting for services to be ready..."
+	@sleep 10
+	@echo "✅ Services started! Check status with 'make status'"
+
+# Stop all services
+stop:
+	@echo "🛑 Stopping services..."
+	docker compose down
+
+# Restart services
+restart:
+	@echo "🔄 Restarting services..."
+	docker compose restart
 
-test:
-	pytest tests/ -v --cov=app --cov-report=html --cov-report=term
+restart-build:
+	@echo "🔄 Restarting services..."
+	docker compose up -d --build
+	
 
-lint:
-	flake8 app/ tests/
-	mypy app/
+# View logs
+logs:
+	@echo "📋 Viewing application logs..."
+	docker compose logs -f log_analysis_service
 
-format:
-	black app/ tests/
-	isort app/ tests/
+# Check service status
+status:
+	@echo "📊 Service Status:"
+	@echo ""
+	@echo "Docker Services:"
+	@docker compose ps
+	@echo ""
+	@echo "Health Check:"
+	@curl -s http://localhost:8000/health | jq . || echo "❌ Health check failed"
+	@echo ""
+	@echo "Readiness Check:"
+	@curl -s http://localhost:8000/health/ready | jq . || echo "❌ Readiness check failed"
 
+# Clean up
 clean:
-	find . -type d -name "__pycache__" -exec rm -rf {} +
-	find . -type f -name "*.pyc" -delete
-	find . -type f -name "*.pyo" -delete
-	find . -type d -name "*.egg-info" -exec rm -rf {} +
-	rm -rf .pytest_cache .mypy_cache .coverage htmlcov/
+	@echo "🧹 Cleaning up..."
+	docker compose down -v
+	docker system prune -f
+
+# Local development
+dev:
+	@echo "🔧 Starting for local development..."
+	python start_app.py
+
+# Initial setup
+setup:
+	@echo "⚙️  Setting up Log Analysis Service..."
+	@echo "1. Creating .env file..."
+	@if [ ! -f .env ]; then cp .env.example .env; echo "✅ .env file created"; else echo "✅ .env file already exists"; fi
+	@echo "2. Starting services..."
+	docker compose up -d
+	@echo "3. Waiting for services to be ready..."
+	@sleep 15
+	@echo "4. Running initial tests..."
+	python test_all_endpoints.py
+	@echo "✅ Setup complete! Service is ready to use."
+
+# Quick test
+quick-test:
+	@echo "⚡ Quick API test..."
+	@curl -s http://localhost:8000/health | jq . && echo "✅ Health check passed" || echo "❌ Health check failed"
+	@curl -s http://localhost:8000/health/ready | jq . && echo "✅ Readiness check passed" || echo "❌ Readiness check failed"
+
+# AI toggle commands
+ai-on:
+	@echo "🤖 Enabling AI processing..."
+	@curl -s -X POST -H "Authorization: Bearer test-token" "http://localhost:8000/api/v1/ai/toggle?enable_ai=true" | jq .
+
+ai-off:
+	@echo "🔄 Disabling AI processing..."
+	@curl -s -X POST -H "Authorization: Bearer test-token" "http://localhost:8000/api/v1/ai/toggle?enable_ai=false" | jq .
+
+ai-status:
+	@echo "📊 AI Status:"
+	@curl -s -H "Authorization: Bearer test-token" "http://localhost:8000/api/v1/ai/status" | jq .
+
+# Test AI with custom message
+test-ai:
+	@echo "🧪 Testing AI with custom message..."
+	@curl -s -X POST -H "Authorization: Bearer test-token" "http://localhost:8000/api/v1/ai/test?test_message=Database connection timeout&service_name=test-service" | jq .
+
+# View all logs
+logs-all:
+	@echo "📋 Viewing all service logs..."
+	docker compose logs -f
+
+# Database commands
+db-shell:
+	@echo "🗄️  Opening database shell..."
+	docker compose exec postgres psql -U postgres -d log_analysis_db
+
+db-tables:
+	@echo "📊 Database tables:"
+	docker compose exec postgres psql -U postgres -d log_analysis_db -c "\dt"
+
+# Metrics
+metrics:
+	@echo "📈 Prometheus metrics:"
+	@curl -s http://localhost:8000/metrics | head -20
+
+# Authentication
+get-token:
+	@echo "🔑 Generating authentication token..."
+	@docker compose exec -T log_analysis_service python -c "from app.core.auth_external import create_test_token; print('Token:', create_test_token('test-user', ['admin']))"
+
+get-token-user:
+	@echo "🔑 Generating user token..."
+	@docker compose exec -T log_analysis_service python -c "from app.core.auth_external import create_test_token; print('Token:', create_test_token('user-123', ['user']))"
+
+get-token-admin:
+	@echo "🔑 Generating admin token..."
+	@docker compose exec -T log_analysis_service python -c "from app.core.auth_external import create_test_token; print('Token:', create_test_token('admin-456', ['admin', 'user']))"
+
+# Configuration
+get-config:
+	@echo "⚙️  Application Configuration:"
+	@docker compose exec -T log_analysis_service python -c "from app.core.config import get_settings; s = get_settings(); print(f'Database URL: {s.database_url}'); print(f'RabbitMQ URL: {s.rabbitmq_url}'); print(f'Server: {s.host}:{s.port}')"
+
+# Testing
+test-logs:
+	@echo "🧪 Running comprehensive log tests..."
+	docker compose exec -T log_analysis_service python /app/test_send_log.py --scenario all
+
+test-critical:
+	@echo "🚨 Testing critical errors..."
+	docker compose exec -T log_analysis_service python /app/test_send_log.py --scenario critical
+
+test-pattern:
+	@echo "💡 Testing pattern detection..."
+	docker compose exec -T log_analysis_service python /app/test_send_log.py --scenario pattern
+
+test-anomaly:
+	@echo "⚠️  Testing anomaly detection..."
+	docker compose exec -T log_analysis_service python /app/test_send_log.py --scenario anomaly
+
+test-normal:
+	@echo "✅ Testing normal logs..."
+	docker compose exec -T log_analysis_service python /app/test_send_log.py --scenario normal
+
+test-mixed:
+	@echo "🎯 Testing mixed scenario..."
+	docker compose exec -T log_analysis_service python /app/test_send_log.py --scenario mixed
+
+# API Testing (requires token)
+test-api-status:
+	@echo "🌐 Testing AI status endpoint..."
+	@TOKEN=$$(docker compose exec -T log_analysis_service python -c "from app.core.auth_external import create_test_token; print(create_test_token('test-user', ['admin']))"); \
+	curl -H "Authorization: Bearer $$TOKEN" http://localhost:8000/api/v1/ai/status
 
-docker-build:
-	docker-compose build
+test-api-logs:
+	@echo "📋 Testing logs endpoint..."
+	@TOKEN=$$(docker compose exec -T log_analysis_service python -c "from app.core.auth_external import create_test_token; print(create_test_token('test-user', ['admin']))"); \
+	curl -H "Authorization: Bearer $$TOKEN" http://localhost:8000/api/v1/logs
 
-docker-up:
-	docker-compose up -d
+test-api-stats:
+	@echo "📊 Testing stats endpoint..."
+	@TOKEN=$$(docker compose exec -T log_analysis_service python -c "from app.core.auth_external import create_test_token; print(create_test_token('test-user', ['admin']))"); \
+	curl -H "Authorization: Bearer $$TOKEN" http://localhost:8000/api/v1/stats
 
-docker-down:
-	docker-compose down
+# View results
+check-logs-db:
+	@echo "📊 Recent logs in database:"
+	docker compose exec postgres psql -U postgres -d log_analysis_db -c "SELECT id, service_name, log_level, LEFT(message, 50) as message, timestamp FROM log_entries ORDER BY timestamp DESC LIMIT 10;"
 
-migrate:
-	alembic upgrade head
+check-patterns:
+	@echo "🔍 Detected patterns:"
+	docker compose exec postgres psql -U postgres -d log_analysis_db -c "SELECT pattern_id, LEFT(template, 40) as template, occurrence_count, service_name FROM patterns WHERE is_active = true ORDER BY occurrence_count DESC LIMIT 10;"
 
-migrate-create:
-	@read -p "Enter migration message: " msg; \
-	alembic revision --autogenerate -m "$$msg"
+check-queues:
+	@echo "📬 RabbitMQ queues:"
+	docker compose exec rabbitmq rabbitmqctl list_queues name messages