v0.1.5-alpha: Major documentation overhaul, async backend migration, 50+ bug fixes

- Complete documentation restructure with example reports from 7 models
- Full async backend migration for 2-3x performance improvement
- Fixed mission resume/recovery capabilities
- Enhanced UI/UX with LaTeX support
- Removed evaluation folder from tracking
- Updated to PostgreSQL-only architecture

Author: Murtaza Nsair

.env.example

@@ -89,8 +89,46 @@ CORS_ALLOWED_ORIGINS=*
 # Development mode flag - allows wildcard CORS when true
 ALLOW_CORS_WILDCARD=true
 
-# Worker and performance settings
-MAX_WORKER_THREADS=10
+# =============================================================================
+# CONCURRENCY AND PERFORMANCE SETTINGS
+# =============================================================================
+# These settings control different types of parallel operations:
+#
+# 1. MAX_WORKER_THREADS: General background tasks (NOT for LLM calls)
+#    - Web page fetching, document processing, file operations
+#
+# 2. GLOBAL_MAX_CONCURRENT_LLM_REQUESTS: System-wide LLM API limit
+#    - Total concurrent LLM calls across ALL users/sessions
+#
+# 3. MAX_CONCURRENT_REQUESTS: Per-session LLM API limit  
+#    - How many LLM calls one research session can make
+#    - Users can override this in UI settings
+#
+# 4. Web Search: Always limited to 2 concurrent (hardcoded)
+# =============================================================================
+
+# General purpose thread pool for background tasks
+# Controls: Web scraping, document processing, file operations
+# Does NOT control: LLM API calls (those use separate limits below)
+# Default: 20, Recommended: 10-50 depending on CPU cores and RAM
+MAX_WORKER_THREADS=20
+
+# System-wide limit for concurrent LLM API calls
+# This is the maximum across ALL users and research sessions combined
+# Example: If set to 200, and you have 10 users each wanting 50 concurrent
+#          requests, only 200 total will run at once
+# Default: 200, Recommended: 50-500 depending on your LLM provider
+GLOBAL_MAX_CONCURRENT_LLM_REQUESTS=200
+
+# Per-session limit for concurrent LLM API calls
+# This is a FALLBACK - users typically set this in the UI
+# Settings > Research > Performance > Concurrent Requests
+# Precedence: Mission settings > User UI settings > This env var > Default (10)
+# Default: 10, Recommended: 10-50 per session
+MAX_CONCURRENT_REQUESTS=10
+
+# Note: Web search APIs (Tavily, Jina, etc.) are hardcoded to 2 concurrent
+#       requests to avoid rate limiting from search providers
 
 # Timezone configuration
 # Use your local timezone (e.g., America/New_York, Europe/London, Asia/Tokyo)

.github/workflows/docs.yml

@@ -0,0 +1,71 @@
+name: Deploy Documentation
+
+on:
+  push:
+    branches:
+      - main
+    paths:
+      - 'docs/**'
+      - 'mkdocs.yml'
+      - '.github/workflows/docs.yml'
+  pull_request:
+    branches:
+      - main
+    paths:
+      - 'docs/**'
+      - 'mkdocs.yml'
+  workflow_dispatch:
+
+permissions:
+  contents: read
+  pages: write
+  id-token: write
+
+concurrency:
+  group: "pages"
+  cancel-in-progress: false
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0  # Fetch all history for git revision date plugin
+
+      - name: Setup Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.11'
+          cache: 'pip'
+
+      - name: Install MkDocs and dependencies
+        run: |
+          pip install --upgrade pip
+          pip install mkdocs-material
+          pip install mkdocs-minify-plugin
+          pip install mkdocs-git-revision-date-localized-plugin
+          pip install mike
+          pip install mkdocs-mermaid2-plugin
+
+      - name: Build documentation
+        run: mkdocs build --verbose --clean --strict
+
+      - name: Upload artifact
+        if: github.ref == 'refs/heads/main' && github.event_name == 'push'
+        uses: actions/upload-pages-artifact@v3
+        with:
+          path: ./site
+
+  deploy:
+    if: github.ref == 'refs/heads/main' && github.event_name == 'push'
+    environment:
+      name: github-pages
+      url: ${{ steps.deployment.outputs.page_url }}
+    runs-on: ubuntu-latest
+    needs: build
+    steps:
+      - name: Deploy to GitHub Pages
+        id: deployment
+        uses: actions/deploy-pages@v4

.gitignore

@@ -13,6 +13,12 @@
 *.log
 docker-logs.txt
 
+# Release notes (not for version control)
+release-notes/
+
+# Evaluation folder (local testing only)
+evaluation/
+
 # =============================================================================
 # PYTHON
 # =============================================================================
@@ -410,7 +416,6 @@ htmlcov/
 .pytest_cache/
 .tox/
 pdfs*/
-docs/
 # =============================================================================
 # STREAMLIT & EVALUATION
 # =============================================================================
@@ -434,7 +439,7 @@ docs/
 # =============================================================================
 .claude/
 CLAUDE.md
-docs/*
+own_docs/*
 notes.md
 WEBSOCKET_REWRITE_TODO.md
 # Temporary test files

.streamlit/config.toml

@@ -22,7 +22,7 @@ This guide explains how to configure Maestro to run in CPU-only mode, which is p
 
 3. **Start the application**:
    ```bash
-   docker-compose up -d
+   docker compose up -d
    ```
 
 ### Method 2: Using CPU-Only Docker Compose
@@ -31,7 +31,7 @@ We provide a dedicated CPU-only Docker Compose configuration that doesn't requir
 
 ```bash
 # Use the CPU-only configuration
-docker-compose -f docker-compose.cpu.yml up -d
+docker compose -f docker-compose.cpu.yml up -d
 ```
 
 This configuration automatically sets `FORCE_CPU_MODE=true` and removes all GPU-related settings.
@@ -137,7 +137,7 @@ MAX_CONCURRENT_REQUESTS=1
 
 **Solution**: Use the CPU-only Docker Compose file:
 ```bash
-docker-compose -f docker-compose.cpu.yml up -d
+docker compose -f docker-compose.cpu.yml up -d
 ```
 
 ## Community Contributions

CPU_MODE_SETUP.md

@@ -5,23 +5,33 @@
 # MAESTRO: Your Self-Hosted AI Research Assistant
 
 [![License: AGPL v3](https://img.shields.io/badge/License-AGPL_v3-blue.svg)](https://www.gnu.org/licenses/agpl-3.0)
-[![Version](https://img.shields.io/badge/Version-2.1.0-green.svg)](https://github.com/murtaza-nasir/maestro.git)
+[![Version](https://img.shields.io/badge/Version-0.1.5--alpha-green.svg)](https://github.com/murtaza-nasir/maestro.git)
 [![Docker](https://img.shields.io/badge/Docker-Ready-blue.svg)](https://hub.docker.com/r/yourusername/maestro)
+[![Documentation](https://img.shields.io/badge/Docs-Available-brightgreen.svg)](https://yourusername.github.io/maestro/)
 
-> **Version 0.1.4 Update (08/20/2025)**
+> **Version 0.1.5-alpha (January 2025) - Major Update**
 > 
-> - **New**: Jina.ai integration for web search and content fetching
-> - **Improved**: Search queries now expand vague references automatically  
-> - **Enhanced**: Better handling of JavaScript-heavy sites with Jina fetcher but will be slower
+> - **Performance**: Complete async backend migration (2-3x faster)
+> - **Stability**: 50+ bug fixes and mission recovery improvements
+> - **Documentation**: Complete overhaul with example reports and guides
+> - **UI/UX**: Enhanced interface with LaTeX support and better navigation
+> - **[Full Release Notes](RELEASE_NOTES_v0.1.5-alpha.md)** | **[Documentation](https://yourusername.github.io/maestro/)**
 >
-> **⚠️ Version 0.1.3 - BREAKING CHANGE (08/15/2025)** 
-> 
-> Complete migration from SQLite/ChromaDB to PostgreSQL with pgvector. 
-> - **Action Required**: If upgrading, you must rebuild from scratch with `docker compose down -v`
-> - **New Requirements**: PostgreSQL with pgvector extension (included in Docker setup)
-> - **Security**: All credentials now configurable via environment variables 
+> **Breaking Change**: PostgreSQL with pgvector is required (SQLite no longer supported) 
+
+MAESTRO is an AI-powered research platform you can host on your own hardware. It's designed to manage complex research tasks from start to finish in a collaborative research environment. Plan your research, let AI agents carry it out, and watch as they generate detailed reports based on your documents and sources from the web.
+
+## Documentation
+
+**[View Full Documentation](https://yourusername.github.io/maestro/)**
 
-MAESTRO is an AI-powered research platform you can host on your own hardware. It's designed to manage complex research tasks from start to finish in a collaborative, multi-user environment. Plan your research, let AI agents carry it out, and watch as they generate detailed reports based on your documents and sources from the web.
+### Quick Links
+- **[Quick Start Guide](https://yourusername.github.io/maestro/getting-started/quickstart/)** - Get up and running in minutes
+- **[Installation Guide](https://yourusername.github.io/maestro/getting-started/installation/)** - Detailed setup instructions
+- **[User Guide](https://yourusername.github.io/maestro/user-guide/)** - Learn all features
+- **[Configuration](https://yourusername.github.io/maestro/getting-started/configuration/overview/)** - Set up AI providers
+- **[Example Reports](https://yourusername.github.io/maestro/example-reports/)** - See what MAESTRO can do
+- **[Troubleshooting](https://yourusername.github.io/maestro/troubleshooting/)** - Common issues and solutions
 
 <p align="center">
   <img src="images/10-research-draft.png" alt="Final Draft" width="700"/>
@@ -242,7 +252,7 @@ docker compose up -d
 **⚠️ First Run:** Initial startup takes 5-10 minutes to download AI models. Monitor progress with:
 ```bash
 docker compose logs -f maestro-backend
-# Wait for: "Application startup complete"
+# Wait for: "MAESTRO Backend Started Successfully!"
 ```
 
 Access MAESTRO at **http://localhost**
@@ -291,7 +301,7 @@ Access MAESTRO at **http://localhost**
     docker compose logs -f maestro-backend
     
     # Wait for this message:
-    # "INFO:     Application startup complete."
+    # "MAESTRO Backend Started Successfully!"
     # or "Uvicorn running on http://0.0.0.0:8000"
     ```
 
@@ -343,7 +353,7 @@ Access MAESTRO at **http://localhost**
     docker compose logs -f maestro-backend
     
     # Wait for this message:
-    # "INFO:     Application startup complete."
+    # "MAESTRO Backend Started Successfully!"
     # or "Uvicorn running on http://0.0.0.0:8000"
     ```
 

README.md

@@ -71,7 +71,7 @@ docker compose -f docker-compose.cpu.yml down
 docker compose logs -f maestro-backend
 
 # Wait for this message:
-# "INFO:     Application startup complete."
+# "INFO:     MAESTRO Backend Started Successfully!."
 ```
 
 ---

TROUBLESHOOTING.md

@@ -18,7 +18,7 @@ detect_gpu() {
     # Check for NVIDIA GPU on Linux/Windows
     if command -v nvidia-smi &> /dev/null; then
         # Check if nvidia-container-toolkit is installed
-        if docker run --rm --gpus all nvidia/cuda:11.0-base nvidia-smi &> /dev/null 2>&1; then
+        if docker run --rm --gpus all nvidia/cuda:12.9.1-base-ubuntu24.04 nvidia-smi &> /dev/null 2>&1; then
             echo "nvidia"
             return
         fi

detect_gpu.sh

@@ -161,16 +161,9 @@ services:
 
 volumes:
   postgres-data:
-    driver: local
   maestro-data:
-    driver: local
-    driver_opts:
-      type: none
-      o: bind
-      device: ./maestro_data
   # Uncomment if using local LLM
   # ollama-data:
-  #   driver: local
 
 networks:
   maestro-network:

docker-compose.cpu.yml

@@ -52,11 +52,12 @@ services:
       - ./maestro_datalab_cache:/root/.cache/datalab
       - ./reports:/app/reports
       - ./maestro_backend/data:/app/data
-    # Backend is now only accessible through nginx, not directly exposed
-    # ports:
-    #   - "${BACKEND_HOST}:${BACKEND_PORT}:${BACKEND_INTERNAL_PORT}"
     working_dir: /app
     environment:
+      - DEBUG_MESSENGER=true
+      - MESSENGER_SIMPLE_PROMPT=true
+      - DEBUG_PLANNING=true
+      - DEBUG_REFLECTION=true
       - MAX_WORKER_THREADS=${MAX_WORKER_THREADS:-10}
       - TZ=${TZ:-UTC}
       - LOG_LEVEL=${LOG_LEVEL:-ERROR}
@@ -195,8 +196,6 @@ services:
 volumes:
   postgres-data:
   maestro-data:
-  maestro-models:  # New volume for models
-  # ollama-data:
 
 networks:
   maestro-network: