[docker & docs] Migration guide update (#818)

update migration guide (include link in README.md)
edit compose.base.yml to be cross os friendly (using relative path)

Author: noobHappylife

MIGRATION_GUIDE.md

@@ -1 +1,116 @@
-# Migration Guide
+# JamAIBase v1 to v2 migration guide
+
+## Overview
+
+This guide helps migrate your JamAIBase instance from v1 to v2.
+
+## Pre-requisites
+
+- During migration, you will need to have both instances of JamAIBase running concurrently
+- For the existing v1 JamAIBase, please re-launch with the new port mapping, to prevent clashing with the v2 instance.
+
+## Launching v1 with updated port mapping
+
+- Update the .env
+
+```
+API_PORT=26969
+```
+
+- Update the docker/compose.cpu.yml
+
+```
+# under the owl service
+# from
+# image: jamai/owl
+# to
+image: jamai/owl:v1
+```
+
+- Launch v1 JamAIBase with a subset of the services
+
+```
+cd /path/to/your/v1/dir
+docker compose --env-file .env -f docker/compose.cpu.yml up --scale infinity=0 --scale starling=0 --scale frontend=0 --scale docling=0
+```
+
+> if on windows (powershell)
+
+```
+cd C:\path\to\your\v1\dir
+docker compose -p jamai --env-file .env -f docker/compose.cpu.yml up --scale infinity=0 --scale starling=0 --scale frontend=0 --scale docling=0
+```
+
+---
+
+## Launching v2
+
+- Launch v2 JamAIBase
+
+```
+cd /path/to/your/v2/dir
+docker compose -p jamai --env-file .env -f docker/compose.oss.yml up
+```
+
+> if on windows (powershell)
+
+```
+cd C:\path\to\your\v2\dir
+docker compose -p jamai --env-file .env -f docker/compose.oss.yml up
+```
+
+## Run the migration
+
+- Install JamAIBase locally
+
+```
+cd /path/to/your/v2/dir
+cd services/api
+pip install -e .
+cd -
+cd clients/python
+pip install -e .
+```
+
+> if on windows (powershell)
+
+```
+cd C:\path\to\your\v2\dir
+
+cd services\api
+pip install -e .
+
+cd ..\..
+cd clients\python
+pip install -e .
+```
+
+- Run the migration scripts
+
+```
+# Check your v1 db path
+export OWL_V1_DB=/path/to/your/v1/db/main.db
+export OWL_V1_FILE=/path/to/your/v1/file # path to your v1 file directory
+export OWL_REDIS_HOST=localhost
+export OWL_DB_PATH=postgresql+psycopg://owlpguser:owlpgpassword@localhost:5432/jamaibase_owl
+
+python scripts/oss_migrate.py --db_v1_path ${OWL_V1_DB} --migrate --reset --api_base_dst http://localhost:6969/api --api_base_src http://localhost:26969/api --v1_file_path ${OWL_V1_FILE} --s3_endpoint_dst http://localhost:9000
+```
+
+> if on windows (powershell)
+
+```
+# Set environment variables for the current PowerShell session
+# Use Windows-style paths for your files and databases
+ $env:OWL_V1_DB = "C:\path\to\your\v1\db\main.db"
+ $env:OWL_V1_FILE = "C:\path\to\your\v1\file"
+ $env:OWL_REDIS_HOST = "localhost"
+ $env:OWL_DB_PATH = "postgresql+psycopg://owlpguser:owlpgpassword@localhost:5432/jamaibase_owl"
+
+# Run the migration script
+# Note the use of '.\' to execute a script in the current directory
+# and $env:VAR_NAME to reference environment variables
+python .\scripts\oss_migrate.py --db_v1_path $env:OWL_V1_DB --migrate --reset --api_base_dst http://localhost:6969/api --api_base_src http://localhost:26969/api --v1_file_path $env:OWL_V1_FILE --s3_endpoint_dst http://localhost:9000
+```
+
+- If everything is successful, you can close the v1 instance

README.md

@@ -17,6 +17,10 @@ JamAI Base is an open-source RAG (Retrieval-Augmented Generation) backend platfo
 
 ![JamAI Base Demo](jamaibase.webp)
 
+## Migration Guide from v1 to v2
+
+Refer to [Migration Guide](./MIGRATION_GUIDE.md)
+
 ## Key Features
 
 - Embedded database (SQLite) and vector database (LanceDB)

docker/compose.base.yml

@@ -14,15 +14,15 @@ services:
     # https://docs.docker.com/compose/compose-file/compose-file-v3/#network_mode
     # network_mode: "host"
     # volumes:
-    #   - ${PWD}/docker_data/dragonfly:/data
+    #   - ../docker_data/dragonfly:/data
     networks:
       - jamai
 
   otel-collector:
     image: otel/opentelemetry-collector-contrib:0.113.0
     command: ["--config=/etc/otelcol/config.yaml"]
     volumes:
-      - ${PWD}/docker/otel_configs/otel-collector-config.yaml:/etc/otelcol/config.yaml
+      - ./otel_configs/otel-collector-config.yaml:/etc/otelcol/config.yaml
     networks:
       - jamai
 
@@ -32,7 +32,7 @@ services:
       - "--selfScrapeInterval=15s"
       - "--retentionPeriod=100y"
     volumes:
-      - ${PWD}/docker_data/vm_data:/victoria-metrics-data
+      - ../docker_data/vm_data:/victoria-metrics-data
     networks:
       - jamai
     healthcheck:
@@ -47,7 +47,7 @@ services:
     command:
       - "--auth.config=/etc/config.yml"
     volumes:
-      - ${PWD}/docker/vmauth/config.yml:/etc/config.yml
+      - ./vmauth/config.yml:/etc/config.yml
     networks:
       - jamai
     healthcheck:
@@ -62,7 +62,7 @@ services:
     command:
       - "--retentionPeriod=100y"
     volumes:
-      - ${PWD}/docker_data/vl_data:/victoria-logs-data
+      - ../docker_data/vl_data:/victoria-logs-data
     networks:
       - jamai
     healthcheck:
@@ -82,8 +82,8 @@ services:
       victorialogs:
         condition: service_healthy
     volumes:
-      - ${PWD}/docker/vmagent/streamingAggregation.yaml:/etc/config/streamingAggregation.yaml
-      - ${PWD}/docker_data/vmagent:/tmp/vmagent
+      - ./vmagent/streamingAggregation.yaml:/etc/config/streamingAggregation.yaml
+      - ../docker_data/vmagent:/tmp/vmagent
     command:
       # - "--promscrape.config=/etc/prometheus/prometheus.yml"
       - "--remoteWrite.url=http://vmauth:8427/vm/api/v1/write"
@@ -99,11 +99,11 @@ services:
   clickhouse:
     image: clickhouse:24.10.2.80
     volumes:
-      - ${PWD}/docker_data/ch_data:/var/lib/clickhouse/
-      - ${PWD}/docker_data/ch_logs:/var/log/clickhouse-server/
-      - ${PWD}/docker/ch_configs/clickhouse_config.xml:/etc/clickhouse-server/config.d/custom_config.xml
-      - ${PWD}/docker/ch_configs/clickhouse_user_config.xml:/etc/clickhouse-server/users.d/custom_config.xml
-      - ${PWD}/docker/ch_configs/create_ch_prom_db.sh:/docker-entrypoint-initdb.d/create_ch_prom_db.sh
+      - ../docker_data/ch_data:/var/lib/clickhouse/
+      - ../docker_data/ch_logs:/var/log/clickhouse-server/
+      - ./ch_configs/clickhouse_config.xml:/etc/clickhouse-server/config.d/custom_config.xml
+      - ./ch_configs/clickhouse_user_config.xml:/etc/clickhouse-server/users.d/custom_config.xml
+      - ./ch_configs/create_ch_prom_db.sh:/docker-entrypoint-initdb.d/create_ch_prom_db.sh
     environment:
       - CLICKHOUSE_ALWAYS_RUN_INITDB_SCRIPTS=1
       - CLICKHOUSE_USER=${CLICKHOUSE_USER:-owluser}
@@ -146,7 +146,7 @@ services:
       PGPASSWORD: owlpgpassword
       PGDATABASE: jamaibase_owl
     volumes:
-      - ${PWD}/docker_data/postgres_db:/var/lib/postgresql/data
+      - ../docker_data/postgres_db:/var/lib/postgresql/data
     networks:
       - jamai
     healthcheck:
@@ -201,7 +201,7 @@ services:
     networks:
       - jamai
     volumes:
-      - ${PWD}/docker_data/minio:/data
+      - ../docker_data/minio:/data
 
   docling:
     image: ghcr.io/embeddedllm/docling-serve:20250528
@@ -230,9 +230,9 @@ services:
     networks:
       - jamai
     volumes:
-      - ${PWD}/docker_data/owl/db:/usr/src/app/db
-      - ${PWD}/docker_data/owl/logs:/usr/src/app/logs
-      - ${PWD}/docker_data/owl/file:/usr/src/app/file
+      - ../docker_data/owl/db:/usr/src/app/db
+      - ../docker_data/owl/logs:/usr/src/app/logs
+      - ../docker_data/owl/file:/usr/src/app/file
     depends_on:
       dragonfly:
         condition: service_healthy