Feat: Implement RAG pipeline for advanced conversational AI

.github/workflows/build_docker.yml

@@ -152,12 +152,13 @@ jobs:
         run: |
           cd genai-svc
           pip install -r requirements.txt
+          pip install -r requirements-dev.txt
 
-      - name: Run Python tests (if any)
+      - name: Run Python tests
         run: |
+          export SKIP_WEAVIATE=true
           cd genai-svc
-          # Add test commands here when tests are available
-          python -m py_compile app.py
+          python -m pytest tests/ -v
 
   build:
     name: Build Docker Images

.gitignore

@@ -383,7 +383,7 @@ sketch
 /gateway/src/main/java/de/tum/aet/devops25/api/generated/
 /concept-svc/src/main/java/de/tum/aet/devops25/api/generated/
 /user-svc/src/main/java/de/tum/aet/devops25/api/generated/
-/genai-svc/client/
+/genai-svc/genai_models/
 
 # OpenAPI Generator CLI configuration
 openapitools.json

README.md

@@ -222,6 +222,8 @@ This will start all services:
 - User Service at [http://localhost:8081](http://localhost:8081)
 - Concept Service at [http://localhost:8082](http://localhost:8082)
 - GenAI Service at [http://localhost:8083](http://localhost:8083)
+- Weaviate Vector Database at [http://localhost:8087](http://localhost:8087)
+- MinIO Object Storage at [http://localhost:9000](http://localhost:9000) (API) and [http://localhost:9001](http://localhost:9001) (Console)
 
 ### Option 2: Manual Startup
 

api/genai-service.yaml

@@ -31,6 +31,7 @@ paths:
       description: Check if the GenAI Service is running
       tags:
         - Health
+      x-openapi-router-controller: controllers.health_controller
       security: []
       responses:
         '200':
@@ -74,6 +75,7 @@ paths:
       description: Create the first welcome message when a user creates a new concept, personalized with user and concept details
       tags:
         - Chat Interface
+      x-openapi-router-controller: controllers.chat_controller
       security:
         - bearerAuth: []
       requestBody:
@@ -136,6 +138,7 @@ paths:
       description: Send a message to the AI assistant for concept development and receive intelligent responses with optional concept suggestions
       tags:
         - Chat Interface
+      x-openapi-router-controller: controllers.chat_controller
       security:
         - bearerAuth: []
       requestBody:
@@ -177,6 +180,7 @@ paths:
       description: Upload documents for AI analysis and RAG processing
       tags:
         - Document Processing
+      x-openapi-router-controller: controllers.document_controller
       security:
         - bearerAuth: []
       parameters:
@@ -234,6 +238,7 @@ paths:
       description: Retrieve all processed documents associated with a concept
       tags:
         - Document Processing
+      x-openapi-router-controller: controllers.document_controller
       security:
         - bearerAuth: []
       parameters:
@@ -278,6 +283,7 @@ paths:
       description: Delete a specific document from a concept and remove it from the knowledge base
       tags:
         - Document Processing
+      x-openapi-router-controller: controllers.document_controller
       security:
         - bearerAuth: []
       parameters:

api/scripts/gen-all.sh

@@ -28,7 +28,8 @@ generate_java() {
   echo "Generating Java code..."
   check_command openapi-generator-cli
 
-openapi-generator-cli generate \
+  # Generate for API Gateway
+  openapi-generator-cli generate \
   -i api/gateway.yaml \
   -g spring \
   -o gateway \
@@ -61,21 +62,23 @@ openapi-generator-cli generate \
   echo "Java code generation complete!"
 }
 
-# Python client generation
+# Python server generation
 generate_python() {
-  echo "Generating Python client..."
-  check_command openapi-python-client
+  echo "Generating Python client and server..."
+  check_command openapi-generator-cli
 
   # Create output directories
-  create_dir "genai-svc/client"
+  create_dir "genai-svc"
 
-  # Generate client for GenAI service
-  openapi-python-client generate \
-    --path api/genai-service.yaml \
-    --output-path genai-svc/client \
-    --overwrite
+  # Generate server stubs for GenAI service
+  openapi-generator-cli generate \
+    -i api/genai-service.yaml \
+    -g python-flask \
+    -o genai-svc \
+    --skip-validate-spec \
+    --additional-properties=packageName=genai_models,serverPort=8083
 
-  echo "Python client generation complete!"
+  echo "Python server generation complete!"
 }
 
 # TypeScript SDK generation

concept-svc/.openapi-generator-ignore

@@ -22,4 +22,4 @@
 # Then explicitly reverse the ignore rule for a single file:
 #!docs/README.md
 
-README.md
+README.md

docker-compose.yml

@@ -9,7 +9,6 @@ services:
       - gateway
     networks:
       - frontend-network
-      - backend-network
 
   # API Gateway (Spring Boot)
   gateway:
@@ -28,7 +27,8 @@ services:
       - concept-svc
       - genai-svc
     networks:
-      - backend-network
+      - frontend-network
+      - service-network
 
   # User Service (Spring Boot)
   user-svc:
@@ -45,7 +45,8 @@ services:
       - SPRING_JPA_SHOW_SQL=true
       - SPRING_JPA_PROPERTIES_HIBERNATE_FORMAT_SQL=true
     networks:
-      - backend-network
+      - service-network
+      - user-db-network
     depends_on:
       - user-svc-db
 
@@ -58,7 +59,7 @@ services:
       - POSTGRES_USER=postgres
       - POSTGRES_DB=eventdb
     networks:
-      - backend-network
+      - user-db-network
     #volumes:
       #- eventdb_data:/var/lib/postgresql/data
 
@@ -77,7 +78,8 @@ services:
       - SPRING_JPA_SHOW_SQL=true
       - SPRING_JPA_PROPERTIES_HIBERNATE_FORMAT_SQL=true
     networks:
-      - backend-network
+      - service-network
+      - concept-db-network
     depends_on:
       - concept-svc-db
 
@@ -90,7 +92,7 @@ services:
       - POSTGRES_USER=postgres
       - POSTGRES_DB=conceptdb
     networks:
-      - backend-network
+      - concept-db-network
     #volumes:
       #- conceptdb_data:/var/lib/postgresql/data
 
@@ -102,11 +104,83 @@ services:
       - "8083:8083"
     environment:
       - FLASK_ENV=production
+      - WEAVIATE_URL=http://genai-svc-weaviate:8080  # Updated for new service name
+      - WEAVIATE_GRPC_PORT=50051  # Explicitly set gRPC port
+      - WEAVIATE_API_KEY=${WEAVIATE_API_KEY:-}
+      - MINIO_URL=http://genai-svc-minio:9000  # Updated for new service name
+      - MINIO_ACCESS_KEY=minioadmin
+      - MINIO_SECRET_KEY=minioadmin
+      - MINIO_BUCKET=concepts
+      - CONCEPT_SERVICE_URL=http://concept-svc:8080
+      - OPENWEBUI_MODEL=llama3.3:latest
+    depends_on:
+      - genai-svc-weaviate
+      - genai-svc-minio
+      - concept-svc
+    networks:
+      - service-network
+      - genai-network
+    env_file:
+      - genai-svc/prod/.env
+
+  # Weaviate Vector Database for GenAI Service
+  genai-svc-weaviate:
+    image: semitechnologies/weaviate:1.30.5
+    ports:
+      - "8087:8080"  # HTTP port
+      - "50051:50051"  # gRPC port
+    environment:
+      - QUERY_DEFAULTS_LIMIT=20
+      - AUTHENTICATION_ANONYMOUS_ACCESS_ENABLED=true
+      - PERSISTENCE_DATA_PATH=/var/lib/weaviate
+      - DEFAULT_VECTORIZER_MODULE=none
+      - ENABLE_MODULES=text2vec-transformers
+      - TRANSFORMERS_INFERENCE_API=http://genai-svc-t2v-transformers:8080
+      # Minimal single-node configuration
+      - CLUSTER_HOSTNAME=genai-svc-weaviate
+      - RAFT_BOOTSTRAP_EXPECT=1
+    volumes:
+      - weaviate_data:/var/lib/weaviate
     networks:
-      - backend-network
+      - genai-network
+
+  # Text2Vec Transformers for Weaviate
+  genai-svc-t2v-transformers:
+    image: semitechnologies/transformers-inference:sentence-transformers-all-MiniLM-L6-v2
+    environment:
+      - ENABLE_CUDA=0
+    networks:
+      - genai-network
+
+  # MinIO Object Storage for GenAI Service
+  genai-svc-minio:
+    image: minio/minio:latest
+    ports:
+      - "9000:9000"
+      - "9001:9001"
+    environment:
+      - MINIO_ROOT_USER=minioadmin
+      - MINIO_ROOT_PASSWORD=minioadmin
+    command: server /data --console-address ":9001"
+    volumes:
+      - minio_data:/data
+    networks:
+      - genai-network
 
 networks:
   frontend-network:
     driver: bridge
-  backend-network:
+  service-network:
     driver: bridge
+  user-db-network:
+    driver: bridge
+  concept-db-network:
+    driver: bridge
+  genai-network:
+    driver: bridge
+
+volumes:
+  weaviate_data:
+    driver: local
+  minio_data:
+    driver: local

genai-svc/.dockerignore

@@ -0,0 +1,72 @@
+.travis.yaml
+.openapi-generator-ignore
+README.md
+tox.ini
+git_push.sh
+test-requirements.txt
+setup.py
+
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+env/
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+*.egg-info/
+.installed.cfg
+*.egg
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*,cover
+.hypothesis/
+venv/
+.python-version
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+target/
+
+#Ipython Notebook
+.ipynb_checkpoints

genai-svc/.env.example

@@ -0,0 +1,19 @@
+# GenAI Service Environment Variables
+
+# Weaviate Configuration
+WEAVIATE_URL=http://weaviate:8080
+WEAVIATE_API_KEY=
+
+# OpenWebUI Configuration
+OPENWEBUI_API_URL=https://gpu.aet.cit.tum.de/api
+OPENWEBUI_API_TOKEN=your_token_here
+OPENWEBUI_MODEL=llama3.3:latest
+
+# AWS S3 Configuration (Optional, for document storage)
+AWS_ACCESS_KEY_ID=
+AWS_SECRET_ACCESS_KEY=
+AWS_REGION=us-east-1
+S3_BUCKET=concepter-documents
+
+# Flask Configuration
+FLASK_ENV=production