docker-compose.dev-remote.yml

# Development Docker Compose for Remote Upload System Testing
# This file simulates Kubernetes environment with shared volumes that simulate the Kubernetes CephFS RWX PVC behavior.
# Repos stored in /work/ (which is project root - avoiding docker volumes) and metadata are stored in /work/.codebase/repos (project root/.codebase)
# Updated to use separate PVCs for workspace and codebase to eliminate circular dependencies

version: '3.8'

services:
  # Qdrant vector database - same as base compose
  qdrant:
    image: qdrant/qdrant:latest
    container_name: qdrant-db-dev-remote
    ports:
      - "6333:6333"
      - "6334:6334"
    volumes:
      - qdrant_storage_dev_remote:/qdrant/storage
    networks:
      - dev-remote-network

  # MCP search service - same as base compose
  mcp:
    build:
      context: .
      dockerfile: Dockerfile.mcp
    container_name: mcp-search-dev-remote
    user: "1000:1000"
    depends_on:
      - qdrant
    env_file:
      - .env
    environment:
      - FASTMCP_HOST=${FASTMCP_HOST}
      - FASTMCP_PORT=${FASTMCP_PORT}
      - QDRANT_URL=${QDRANT_URL}
      - COLLECTION_NAME=${COLLECTION_NAME}
      - PATH_EMIT_MODE=container
      - HF_HOME=/work/.cache/huggingface
      - TRANSFORMERS_CACHE=/work/.cache/huggingface
      - HUGGINGFACE_HUB_CACHE=/work/.cache/huggingface
      - EMBEDDING_MODEL=${EMBEDDING_MODEL}
      - EMBEDDING_PROVIDER=${EMBEDDING_PROVIDER}
      - TOOL_STORE_DESCRIPTION=${TOOL_STORE_DESCRIPTION}
      - TOOL_FIND_DESCRIPTION=${TOOL_FIND_DESCRIPTION}
      - FASTMCP_HEALTH_PORT=18000
      - HF_HOME=/home/user/.cache
      - TRANSFORMERS_CACHE=/home/user/.cache
    ports:
      - "18000:18000"
      - "8000:8000"
    volumes:
      - workspace_pvc:/work:ro
      - huggingface_cache:/home/user/.cache
    networks:
      - dev-remote-network

  # MCP indexer service - same as base compose
  mcp_indexer:
    build:
      context: .
      dockerfile: Dockerfile.mcp-indexer
    container_name: mcp-indexer-dev-remote
    user: "1000:1000"
    # In K8s, scripts would be accessed directly at /app/scripts/ or via proper initContainer
    # For Docker Compose dev-remote simulation, create symlink so /work/scripts/ works
    # Use /tmp/huggingface for cache to avoid permission issues (universally writable)
    # Set CORRECT environment variables for HuggingFace and FastEmbed
    command: ["sh", "-c", "mkdir -p /tmp/huggingface/hub /tmp/huggingface/transformers /tmp/huggingface/fastembed && exec python /app/scripts/mcp_indexer_server.py"]
    depends_on:
      - qdrant
    env_file:
      - .env
    environment:
      - FASTMCP_HEALTH_PORT=18001
      - FASTMCP_HOST=${FASTMCP_HOST}
      - FASTMCP_INDEXER_PORT=${FASTMCP_INDEXER_PORT}
      - QDRANT_URL=${QDRANT_URL}
      - REFRAG_DECODER=${REFRAG_DECODER:-1}
      - REFRAG_RUNTIME=${REFRAG_RUNTIME:-llamacpp}
      - GLM_API_KEY=${GLM_API_KEY}
      - GLM_API_BASE=${GLM_API_BASE:-https://api.z.ai/api/paas/v4/}
      - GLM_MODEL=${GLM_MODEL:-glm-4.6}
      - LLAMACPP_URL=${LLAMACPP_URL:-http://llamacpp:8080}
      - COLLECTION_NAME=${COLLECTION_NAME}
      - PATH_EMIT_MODE=container
      - HF_HOME=/tmp/huggingface
      - HF_HUB_CACHE=/tmp/huggingface/hub
      - TRANSFORMERS_CACHE=/tmp/huggingface/transformers
      - FASTEMBED_CACHE_PATH=/tmp/huggingface/fastembed
      - EMBEDDING_MODEL=${EMBEDDING_MODEL}
      - EMBEDDING_PROVIDER=${EMBEDDING_PROVIDER}
      - QDRANT_TIMEOUT=${QDRANT_TIMEOUT:-60}
      - INDEX_SEMANTIC_CHUNKS=${INDEX_SEMANTIC_CHUNKS:-0}
      - INDEX_MICRO_CHUNKS=${INDEX_MICRO_CHUNKS:-0}
      - INDEX_UPSERT_BATCH=${INDEX_UPSERT_BATCH:-512}
      - INDEX_UPSERT_RETRIES=${INDEX_UPSERT_RETRIES:-5}
      - MAX_MICRO_CHUNKS_PER_FILE=${MAX_MICRO_CHUNKS_PER_FILE:-200}
    ports:
      - "${FASTMCP_INDEXER_PORT:-8001}:8001"
      - "18001:18001"
    volumes:
      - workspace_pvc:/work:rw
      - codebase_pvc:/work/.codebase:rw
    networks:
      - dev-remote-network

  # MCP HTTP search service - same as base compose
  mcp_http:
    build:
      context: .
      dockerfile: Dockerfile.mcp
    container_name: mcp-search-http-dev-remote
    user: "1000:1000"
    depends_on:
      - qdrant
    env_file:
      - .env
    environment:
      - FASTMCP_HOST=${FASTMCP_HOST}
      - FASTMCP_PORT=8000
      - FASTMCP_TRANSPORT=${FASTMCP_HTTP_TRANSPORT}
      - QDRANT_URL=${QDRANT_URL}
      - COLLECTION_NAME=${COLLECTION_NAME}
      - PATH_EMIT_MODE=container
      - HF_HOME=/work/.cache/huggingface
      - TRANSFORMERS_CACHE=/work/.cache/huggingface
      - HUGGINGFACE_HUB_CACHE=/work/.cache/huggingface
      - EMBEDDING_MODEL=${EMBEDDING_MODEL}
      - EMBEDDING_PROVIDER=${EMBEDDING_PROVIDER}
      - TOOL_STORE_DESCRIPTION=${TOOL_STORE_DESCRIPTION}
      - TOOL_FIND_DESCRIPTION=${TOOL_FIND_DESCRIPTION}
      - FASTMCP_HEALTH_PORT=18000
      - HF_HOME=/home/user/.cache
      - TRANSFORMERS_CACHE=/home/user/.cache
    ports:
      - "${FASTMCP_HTTP_HEALTH_PORT:-18002}:18000"
      - "${FASTMCP_HTTP_PORT:-8002}:8000"
    volumes:
      - workspace_pvc:/work:ro
      - huggingface_cache:/home/user/.cache
    networks:
      - dev-remote-network

  # MCP HTTP indexer service - same as base compose
  mcp_indexer_http:
    build:
      context: .
      dockerfile: Dockerfile.mcp-indexer
    container_name: mcp-indexer-http-dev-remote
    user: "1000:1000"
    # In K8s, scripts would be accessed directly at /app/scripts/ or via proper initContainer
    # For Docker Compose dev-remote simulation, create symlink so /work/scripts/ works
    # Use /tmp/huggingface for cache to avoid permission issues (universally writable)
    # Set CORRECT environment variables for HuggingFace and FastEmbed
    command: ["sh", "-c", "mkdir -p /tmp/huggingface/hub /tmp/huggingface/transformers /tmp/huggingface/fastembed && exec python /app/scripts/mcp_indexer_server.py"]
    depends_on:
      - qdrant
    env_file:
      - .env
    environment:
      - FASTMCP_HOST=${FASTMCP_HOST}
      - FASTMCP_INDEXER_PORT=8001
      - FASTMCP_TRANSPORT=${FASTMCP_HTTP_TRANSPORT}
      - QDRANT_URL=${QDRANT_URL}
      - REFRAG_DECODER=${REFRAG_DECODER:-1}
      - REFRAG_RUNTIME=${REFRAG_RUNTIME:-llamacpp}
      - GLM_API_KEY=${GLM_API_KEY}
      - GLM_API_BASE=${GLM_API_BASE:-https://api.z.ai/api/paas/v4/}
      - GLM_MODEL=${GLM_MODEL:-glm-4.6}
      - LLAMACPP_URL=${LLAMACPP_URL:-http://llamacpp:8080}
      - FASTMCP_HEALTH_PORT=18001
      - COLLECTION_NAME=${COLLECTION_NAME}
      - PATH_EMIT_MODE=container
      - HF_HOME=/tmp/huggingface
      - HF_HUB_CACHE=/tmp/huggingface/hub
      - TRANSFORMERS_CACHE=/tmp/huggingface/transformers
      - FASTEMBED_CACHE_PATH=/tmp/huggingface/fastembed
      - EMBEDDING_MODEL=${EMBEDDING_MODEL}
      - EMBEDDING_PROVIDER=${EMBEDDING_PROVIDER}
      - QDRANT_TIMEOUT=${QDRANT_TIMEOUT:-60}
      - INDEX_SEMANTIC_CHUNKS=${INDEX_SEMANTIC_CHUNKS:-0}
      - INDEX_MICRO_CHUNKS=${INDEX_MICRO_CHUNKS:-0}
      - INDEX_UPSERT_BATCH=${INDEX_UPSERT_BATCH:-512}
      - INDEX_UPSERT_RETRIES=${INDEX_UPSERT_RETRIES:-5}
      - MAX_MICRO_CHUNKS_PER_FILE=${MAX_MICRO_CHUNKS_PER_FILE:-200}
    ports:
      - "${FASTMCP_INDEXER_HTTP_PORT:-8003}:8001"
      - "${FASTMCP_INDEXER_HTTP_HEALTH_PORT:-18003}:18001"
    volumes:
      - workspace_pvc:/work:rw
      - codebase_pvc:/work/.codebase:rw
    networks:
      - dev-remote-network

  # Llama.cpp decoder service - same as base compose
  llamacpp:
    image: ghcr.io/ggerganov/llama.cpp:server
    container_name: llama-decoder-dev-remote
    environment:
      - LLAMA_ARG_MODEL=/models/model.gguf
      - LLAMA_ARG_CTX_SIZE=8192
      - LLAMA_ARG_HOST=0.0.0.0
      - LLAMA_ARG_PORT=8080
    ports:
      - "8080:8080"
    volumes:
      - ./models:/models:ro
    command: ["--model", "/models/model.gguf", "--host", "0.0.0.0", "--port", "8080", "--no-warmup"]
    networks:
      - dev-remote-network

  # Indexer service - modified for PVC volumes
  indexer:
    build:
      context: .
      dockerfile: Dockerfile.indexer
    container_name: indexer-dev-remote
    depends_on:
      - qdrant
    env_file:
      - .env
    environment:
      - QDRANT_URL=${QDRANT_URL}
      - COLLECTION_NAME=${COLLECTION_NAME}
      - HF_HOME=/work/.cache/huggingface
      - TRANSFORMERS_CACHE=/work/.cache/huggingface
      - HUGGINGFACE_HUB_CACHE=/work/.cache/huggingface
      - EMBEDDING_MODEL=${EMBEDDING_MODEL}
      - HF_HOME=/home/user/.cache
      - HOST_INDEX_PATH=/work
      - TRANSFORMERS_CACHE=/home/user/.cache
      - QDRANT_TIMEOUT=${QDRANT_TIMEOUT:-60}
      - INDEX_SEMANTIC_CHUNKS=${INDEX_SEMANTIC_CHUNKS:-0}
      - INDEX_MICRO_CHUNKS=${INDEX_MICRO_CHUNKS:-0}
      - INDEX_UPSERT_BATCH=${INDEX_UPSERT_BATCH:-512}
      - INDEX_UPSERT_RETRIES=${INDEX_UPSERT_RETRIES:-5}
      - MAX_MICRO_CHUNKS_PER_FILE=${MAX_MICRO_CHUNKS_PER_FILE:-200}
    volumes:
      - workspace_pvc:/work:rw
      - codebase_pvc:/work/.codebase:rw
      - huggingface_cache:/home/user/.cache
    entrypoint: ["sh", "-c", "mkdir -p /tmp/logs && /app/scripts/wait-for-qdrant.sh && cd /app && python /app/scripts/ingest_code.py --root /work"]
    restart: "no"  # Run once on startup, do not restart after completion
    networks:
      - dev-remote-network

  # Watcher service - modified for PVC volumes
  watcher:
    build:
      context: .
      dockerfile: Dockerfile.indexer
    container_name: watcher-dev-remote
    user: "1000:1000"
    depends_on:
      - qdrant
    env_file:
      - .env
    environment:
      - QDRANT_URL=${QDRANT_URL}
      - COLLECTION_NAME=${COLLECTION_NAME}
      - HF_HOME=/tmp/huggingface
      - HF_HUB_CACHE=/tmp/huggingface/hub
      - TRANSFORMERS_CACHE=/tmp/huggingface/transformers
      - FASTEMBED_CACHE_PATH=/tmp/huggingface/fastembed
      - EMBEDDING_MODEL=${EMBEDDING_MODEL}
      - EMBEDDING_PROVIDER=${EMBEDDING_PROVIDER}
      - WATCH_ROOT=${WATCH_ROOT:-/work}
      - HOST_INDEX_PATH=/work
      - QDRANT_TIMEOUT=${QDRANT_TIMEOUT:-60}
      - INDEX_SEMANTIC_CHUNKS=${INDEX_SEMANTIC_CHUNKS:-0}
      - INDEX_MICRO_CHUNKS=${INDEX_MICRO_CHUNKS:-0}
      - INDEX_UPSERT_BATCH=${INDEX_UPSERT_BATCH:-512}
      - INDEX_UPSERT_RETRIES=${INDEX_UPSERT_RETRIES:-5}
      - MAX_MICRO_CHUNKS_PER_FILE=${MAX_MICRO_CHUNKS_PER_FILE:-200}
      - WATCH_DEBOUNCE_SECS=${WATCH_DEBOUNCE_SECS:-1.5}
      - REMOTE_UPLOAD_ENABLED=${REMOTE_UPLOAD_ENABLED:-0}
    volumes:
      - workspace_pvc:/work:rw
      - codebase_pvc:/work/.codebase:rw
    command: ["sh", "-c", "mkdir -p /tmp/huggingface/hub /tmp/huggingface/transformers /tmp/huggingface/fastembed && exec python /app/scripts/watch_index.py"]
    networks:
      - dev-remote-network

  # Init payload service - modified for PVC volumes with complete bootstrap
  init_payload:
    build:
      context: .
      dockerfile: Dockerfile.indexer
    container_name: init-payload-dev-remote
    user: "1000:1000"
    depends_on:
      - qdrant
    env_file:
      - .env
    environment:
      - QDRANT_URL=${QDRANT_URL}
      - COLLECTION_NAME=${COLLECTION_NAME}
      - HF_HOME=/work/.cache/huggingface
      - TRANSFORMERS_CACHE=/work/.cache/huggingface
      - HUGGINGFACE_HUB_CACHE=/work/.cache/huggingface
      - WORKDIR=/work
      - TOKENIZER_URL=${TOKENIZER_URL:-https://huggingface.co/BAAI/bge-base-en-v1.5/resolve/main/tokenizer.json}
      - TOKENIZER_PATH=${TOKENIZER_PATH:-/work/models/tokenizer.json}
      - EMBEDDING_MODEL=${EMBEDDING_MODEL}
    volumes:
      - workspace_pvc:/work:rw
      - codebase_pvc:/work/.codebase:rw
    command: [
      "sh",
      "-c",
      "mkdir -p /tmp/logs && echo 'Starting initialization sequence...' && /app/scripts/wait-for-qdrant.sh && PYTHONPATH=/app python /app/scripts/create_indexes.py && echo 'Collections and metadata created' && python /app/scripts/warm_all_collections.py && echo 'Search caches warmed for all collections' && python /app/scripts/health_check.py && echo 'Initialization completed successfully!'"
    ]
    restart: "no"  # Run once on startup
    networks:
      - dev-remote-network

  # NEW: Upload Service for Remote Upload System
  upload_service:
    build:
      context: .
      dockerfile: Dockerfile.upload-service
    container_name: upload-service-dev-remote
    user: "0:0" # Windows bind-mount to /work requires root to create workspace dirs
    depends_on:
      - qdrant
    env_file:
      - .env
    environment:
      # Upload service configuration
      - UPLOAD_SERVICE_HOST=0.0.0.0
      - UPLOAD_SERVICE_PORT=8002
      - QDRANT_URL=${QDRANT_URL}
      - WORKDIR=/work
      - MAX_BUNDLE_SIZE_MB=100
      - UPLOAD_TIMEOUT_SECS=300
      
      # Indexing configuration
      - COLLECTION_NAME=${COLLECTION_NAME}
      - HF_HOME=/work/.cache/huggingface
      - TRANSFORMERS_CACHE=/work/.cache/huggingface
      - HUGGINGFACE_HUB_CACHE=/work/.cache/huggingface
      - EMBEDDING_MODEL=${EMBEDDING_MODEL}
      - EMBEDDING_PROVIDER=${EMBEDDING_PROVIDER}
      - USE_TREE_SITTER=${USE_TREE_SITTER}
      - INDEX_SEMANTIC_CHUNKS=${INDEX_SEMANTIC_CHUNKS}
      - INDEX_MICRO_CHUNKS=${INDEX_MICRO_CHUNKS}
      
      # Remote upload mode configuration
      - REMOTE_UPLOAD_ENABLED=1
      - REMOTE_UPLOAD_MODE=development
      - REMOTE_UPLOAD_DEBUG=1
      
      # Qdrant configuration
      - QDRANT_TIMEOUT=${QDRANT_TIMEOUT}
      - MAX_MICRO_CHUNKS_PER_FILE=${MAX_MICRO_CHUNKS_PER_FILE}
      - INDEX_UPSERT_BATCH=${INDEX_UPSERT_BATCH}
      - INDEX_UPSERT_RETRIES=${INDEX_UPSERT_RETRIES}
    ports:
      - "8004:8002"  # Map to different host port to avoid conflicts
      - "18004:18000"  # Health check port
    volumes:
      - workspace_pvc:/work:rw
      - codebase_pvc:/work/.codebase:rw
      - upload_temp:/tmp/uploads
    healthcheck:
      test: ["CMD", "curl", "-f", "http://localhost:8002/health"]
      interval: 30s
      timeout: 10s
      retries: 3
      start_period: 40s
    restart: unless-stopped
    networks:
      - dev-remote-network

  
# PVCs to simulate CephFS RWX behavior (production-like)
volumes:
  # Main workspace volume - simulates CephFS RWX for repository storage
  workspace_pvc:
    driver: local
    driver_opts:
      type: none
      o: bind
      device: ${HOST_INDEX_PATH:-./dev-workspace}
  
  # Codebase metadata volume - simulates CephFS RWX for indexing metadata
  codebase_pvc:
    driver: local
    driver_opts:
      type: none
      o: bind
      device: ./.codebase
  
  # Temporary upload storage
  upload_temp:
    driver: local
  
  # HuggingFace cache for model downloads
  huggingface_cache:
    driver: local
  
  # Indexer cache for model downloads
  indexer_cache:
    driver: local
  
  # Qdrant storage - separate from base compose to avoid conflicts
  qdrant_storage_dev_remote:
    driver: local

# Custom network for service discovery
networks:
  dev-remote-network:
    driver: bridge
    ipam:
      config:
        - subnet: 172.20.0.0/16