fix(docker): make ChromaDB optional and optimize build (#51)

* fix(docker): make ChromaDB optional and optimize build

- Add .dockerignore to exclude unnecessary files while keeping .git and README.md for build
- Remove chromadb from default dependencies to reduce image size
- Make ChromaVectorStore optional (import only if chromadb is installed)
- Add APT_MIRROR and PYPI_INDEX_URL build args for faster downloads in China
- Optimize COPY order for better layer caching
- Clean up .git after build to reduce image size

* fix(docker): correct volume mount paths

- Fix uploads path: /opt/xagent/uploads → /opt/xagent/src/xagent/web/uploads
- Fix .xagent path: /home/xagent/.xagent → /root/.xagent (container runs as root)

* fix(storage): check legacy path first, then use new path

- Check legacy location first (if has data)
- Otherwise use ~/.xagent/data/lancedb or ~/.xagent/memory_store
- Ensures backward compatibility for existing users
- Add @lru_cache to get_default_lancedb_dir() to avoid repeated filesystem checks

* test: update test for new default LanceDB path

Update test_default_lancedb_dir_when_missing_env to expect ~/.xagent/data/lancedb
as the new default path instead of project_root/data/lancedb.

* fix(docker): add WebSocket support and Playwright dependencies

- Move websockets from dev to main dependencies for WebSocket support
- Add missing Playwright Chromium system libraries
- Fix lancedb.py to use .is_dir() instead of .exists()

* fix(deps): move matplotlib to main dependencies

- Move matplotlib from dev to production dependencies

* docs(lancedb): update get_connection_from_env docstring for legacy path behavior

- Reflect current legacy-check and fallback behavior in documentation
- Clarify default path resolution logic in both class method and module function

Author: qinxuye

.dockerignore

@@ -0,0 +1,71 @@
+# Git
+# .git is needed for hatch-vcs version detection in backend build
+.github
+
+# Python
+__pycache__
+*.py[cod]
+*$py.class
+*.so
+.Python
+.pytest_cache
+.mypy_cache
+.ruff_cache
+.coverage
+htmlcov/
+
+# Virtual environments
+.venv
+venv/
+ENV/
+env/
+
+# IDE
+.vscode
+.idea
+*.swp
+*.swo
+*~
+
+# Documentation
+# README.md is needed for pyproject.toml
+docs/
+assets/
+
+# Testing
+tests/
+.pytest_cache/
+.tox/
+
+# CI/CD
+.github/
+.gitlab-ci.yml
+.travis.yml
+
+# Docker
+docker-compose*.yml
+Dockerfile*
+.dockerignore
+
+# Data directories
+data/
+memory_store/
+uploads/
+
+# Build artifacts
+dist/
+build/
+*.egg-info/
+
+# Logs
+*.log
+logs/
+
+# OS
+.DS_Store
+Thumbs.db
+
+# Misc
+*.env
+*.local
+.example.env

docker-compose.yml

@@ -41,10 +41,8 @@ services:
     env_file:
       - .env
     volumes:
-      - xagent_data:/home/xagent/.xagent
+      - xagent_data:/root/.xagent
       - xagent_uploads:/opt/xagent/src/xagent/web/uploads
-      - xagent_lancedb:/opt/xagent/data
-      - xagent_memory:/opt/xagent/memory_store
     depends_on:
       postgres:
         condition: service_healthy
@@ -84,10 +82,6 @@ volumes:
     driver: local
   xagent_uploads:
     driver: local
-  xagent_lancedb:
-    driver: local
-  xagent_memory:
-    driver: local
 
 networks:
   xagent_network:

docker/Dockerfile.backend

@@ -1,14 +1,25 @@
 # syntax=docker/dockerfile:1.6
 ARG PYTHON_VERSION=3.11
 ARG NODE_VERSION=22
+ARG PYPI_INDEX_URL=""
+ARG APT_MIRROR=""
 
 ############################
 # Backend build
 ############################
 FROM python:${PYTHON_VERSION}-bookworm AS backend-build
 
+ARG PYPI_INDEX_URL=""
+ARG APT_MIRROR=""
+
 WORKDIR /opt/xagent
 
+# Use APT mirror if provided (for faster downloads in China)
+RUN if [ -n "$APT_MIRROR" ]; then \
+        sed -i "s|http://deb.debian.org/debian|$APT_MIRROR|g" /etc/apt/sources.list.d/debian.sources \
+        || sed -i "s|http://deb.debian.org/debian|$APT_MIRROR|g" /etc/apt/sources.list; \
+    fi
+
 RUN apt-get update && apt-get install -y --no-install-recommends \
     build-essential \
     gcc \
@@ -21,43 +32,55 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
     libsm6 \
     libxrender1 \
     libxext6 \
+    && git config --global http.version HTTP/1.1 \
+    && git config --global http.postBuffer 524288000 \
     && rm -rf /var/lib/apt/lists/*
 
-# Improve git reliability for large repos / HTTP2 issues
-RUN git config --global http.version HTTP/1.1 \
-    && git config --global http.postBuffer 524288000
+# Copy dependency files first for better layer caching
+COPY pyproject.toml .
+COPY README.md .
 
+# Copy .git directory for version detection (hatch-vcs)
 COPY .git .git
-COPY pyproject.toml README.md .
-COPY src ./src
-COPY alembic.ini .
 
 # Create venv and install build tools
 RUN python -m venv /opt/venv \
-    && /opt/venv/bin/pip install --no-cache-dir --upgrade pip
+    && /opt/venv/bin/pip install --no-cache-dir ${PYPI_INDEX_URL:+--index-url $PYPI_INDEX_URL} --upgrade pip
+
+# Install torch CPU version first (to avoid CUDA version from docling dependency)
+RUN /opt/venv/bin/pip install --no-cache-dir --extra-index-url https://download.pytorch.org/whl/cpu torch
 
-# Install app + deepdoc + chromadb into venv
-# Patch deepdoc to use CPU onnxruntime on x86_64
-RUN /opt/venv/bin/pip install --no-cache-dir . \
+# Copy source code (changes frequently)
+COPY src ./src
+COPY alembic.ini .
+
+# Install app + deepdoc into venv
+RUN /opt/venv/bin/pip install --no-cache-dir ${PYPI_INDEX_URL:+--index-url $PYPI_INDEX_URL} . \
     && git clone https://github.com/xorbitsai/deepdoc-lib /opt/xagent/deepdoc-lib \
     && cd /opt/xagent/deepdoc-lib && git checkout fe75bbb676b141a1ea4257b198f6a04a89e5ac5c \
-    && /opt/venv/bin/pip install --no-cache-dir /opt/xagent/deepdoc-lib \
-    && /opt/venv/bin/pip install --no-cache-dir chromadb \
-    && rm -rf /opt/xagent/deepdoc-lib
+    && /opt/venv/bin/pip install --no-cache-dir ${PYPI_INDEX_URL:+--index-url $PYPI_INDEX_URL} /opt/xagent/deepdoc-lib \
+    && rm -rf /opt/xagent/deepdoc-lib .git
 
 ############################
 # Runtime
 ############################
 FROM python:${PYTHON_VERSION}-bookworm AS runtime
 
 ARG NODE_VERSION=22
+ARG APT_MIRROR=""
 ENV PLAYWRIGHT_BROWSERS_PATH=/ms-playwright
 ENV VIRTUAL_ENV=/opt/venv
 ENV PATH="/opt/venv/bin:$PATH"
 
 WORKDIR /opt/xagent
 
-# System deps + Node.js + LibreOffice
+# Use APT mirror if provided (for faster downloads in China)
+RUN if [ -n "$APT_MIRROR" ]; then \
+        sed -i "s|http://deb.debian.org/debian|$APT_MIRROR|g" /etc/apt/sources.list.d/debian.sources \
+        || sed -i "s|http://deb.debian.org/debian|$APT_MIRROR|g" /etc/apt/sources.list; \
+    fi
+
+# System deps + Node.js + LibreOffice + Playwright dependencies
 RUN apt-get update && apt-get install -y --no-install-recommends \
     curl \
     gnupg \
@@ -67,6 +90,25 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
     libxrender1 \
     libxext6 \
     libgl1 \
+    libatk1.0-0 \
+    libatk-bridge2.0-0 \
+    libcups2 \
+    libdbus-1-3 \
+    libdrm2 \
+    libxkbcommon0 \
+    libxcomposite1 \
+    libxdamage1 \
+    libxfixes3 \
+    libxi6 \
+    libxtst6 \
+    libpango-1.0-0 \
+    libcairo2 \
+    libgbm1 \
+    libnss3 \
+    libnspr4 \
+    libxrandr2 \
+    libasound2 \
+    libpangocairo-1.0-0 \
     && rm -rf /var/lib/apt/lists/*
 
 # Install Node.js
@@ -77,31 +119,23 @@ RUN curl -fsSL https://deb.nodesource.com/gpgkey/nodesource-repo.gpg.key | gpg -
     && apt-get install -y --no-install-recommends nodejs \
     && rm -rf /var/lib/apt/lists/*
 
-# Copy venv + app
+# Copy entrypoint script early (changes rarely)
+COPY docker/entrypoint.sh /opt/xagent/deploy/entrypoint.sh
+
+# Copy venv + app (exclude .git directory)
 COPY --from=backend-build /opt/venv /opt/venv
-COPY --from=backend-build /opt/xagent /opt/xagent
+COPY --from=backend-build /opt/xagent/src /opt/xagent/src
+COPY --from=backend-build /opt/xagent/pyproject.toml /opt/xagent/
+COPY --from=backend-build /opt/xagent/alembic.ini /opt/xagent/
 
 # Copy frontend package files for npm tasks (e.g., PPTX generation)
 COPY frontend/package.json frontend/package-lock.json /opt/xagent/frontend/
-RUN cd /opt/xagent/frontend && npm ci
-
-# Install pptxgenjs globally for JavaScript executor tool
-RUN npm install -g pptxgenjs@4.0.1
-
-# Playwright browser + deps
-RUN python -m playwright install --with-deps chromium
-
-# Download NLTK data and tiktoken models
-RUN python -c "import nltk; nltk.download('punkt_tab')" \
-    && python -c "import tiktoken; tiktoken.encoding_for_model('gpt-4')"
-
-# Copy entrypoint script
-COPY docker/entrypoint.sh /opt/xagent/deploy/entrypoint.sh
-RUN chmod +x /opt/xagent/deploy/entrypoint.sh
-
-# Create directory for uploads and data
-RUN mkdir -p /opt/xagent/uploads \
-    && mkdir -p /home/xagent/.xagent
+RUN cd /opt/xagent/frontend && npm ci \
+    && npm install -g pptxgenjs@4.0.1 \
+    && python -m playwright install chromium \
+    && python -c "import nltk; nltk.download('punkt'); nltk.download('punkt_tab'); nltk.download('wordnet'); nltk.download('averaged_perceptron_tagger')" \
+    && python -c "import tiktoken; tiktoken.encoding_for_model('gpt-4')" \
+    && chmod +x /opt/xagent/deploy/entrypoint.sh
 
 EXPOSE 8000
 ENTRYPOINT ["/opt/xagent/deploy/entrypoint.sh"]

pyproject.toml

@@ -30,6 +30,7 @@ dependencies = [
   "PyYAML >= 6.0.2",
   "fastapi >= 0.35.0",
   "uvicorn >= 0.35.0",
+  "websockets>=15.0.1",
   "httpx >= 0.27.2",
   "rich >= 14.0.0",
   "prompt_toolkit >= 3.0.0",
@@ -61,6 +62,7 @@ dependencies = [
   "jsonschema>=4.25.1",
   "playwright>=1.40.0",
   "beartype>=0.18.5",
+  "matplotlib>=3.5.0",
 ]
 
 [project.optional-dependencies]
@@ -84,13 +86,11 @@ dev = [
   "types-openpyxl>=3.1.5",
   "openpyxl>=3.1.5",
   "pytest-timeout>=2.4.0",
-  "websockets>=15.0.1",
   "types-setuptools>=69.0.0",
   "pytest-socket>=0.7.0",
   # Data science libraries for testing
   "numpy>=1.21.0",
   "pandas>=1.3.0",
-  "matplotlib>=3.5.0",
   "types-docker>=7.1.0.20251009",
   "copilotkit==0.1.69",
 ]

src/xagent/providers/vector_store/__init__.py

@@ -5,16 +5,29 @@
 the standard VectorStore interface.
 """
 
-from xagent.providers.vector_store.base import VectorStore
-from xagent.providers.vector_store.chroma import ChromaVectorStore
-from xagent.providers.vector_store.lancedb import (
+import importlib.util
+
+from .base import VectorStore
+from .lancedb import (
     LanceDBConnectionManager,
     LanceDBVectorStore,
 )
 
-__all__ = [
-    "VectorStore",
-    "ChromaVectorStore",
-    "LanceDBVectorStore",
-    "LanceDBConnectionManager",
-]
+# ChromaVectorStore is optional (requires chromadb)
+_chroma_available = importlib.util.find_spec("chromadb") is not None
+
+if _chroma_available:
+    from .chroma import ChromaVectorStore
+
+    __all__ = [
+        "VectorStore",
+        "LanceDBVectorStore",
+        "LanceDBConnectionManager",
+        "ChromaVectorStore",
+    ]
+else:
+    __all__ = [
+        "VectorStore",
+        "LanceDBVectorStore",
+        "LanceDBConnectionManager",
+    ]

src/xagent/providers/vector_store/chroma.py

@@ -3,9 +3,14 @@
 from typing import Any, ClassVar, Optional
 from uuid import uuid4
 
-from chromadb import Client
-from chromadb.config import Settings
-from chromadb.utils.embedding_functions import DefaultEmbeddingFunction
+try:
+    from chromadb import Client
+    from chromadb.config import Settings
+    from chromadb.utils.embedding_functions import DefaultEmbeddingFunction
+except ImportError as e:
+    raise ImportError(
+        "ChromaDB is not installed. Please install it with: pip install chromadb"
+    ) from e
 
 from .base import VectorStore