diff --git a/.gitignore b/.gitignore index 9609e2106..313a4c346 100644 --- a/.gitignore +++ b/.gitignore @@ -31,6 +31,9 @@ Cargo.lock ztm_agent.db ztm_agent_db* +# Demo env variables +docker/demo/.env + # buck2 out buck-out diff --git a/docker/demo/.dockerignore b/docker/demo/.dockerignore new file mode 100644 index 000000000..7644aa3a3 --- /dev/null +++ b/docker/demo/.dockerignore @@ -0,0 +1,25 @@ +# Exclude files / directories not needed in build context for demo services + +# VCS +.git + +# IDE / editor config +.idea/ +.vscode/ +*.code-workspace + +# OS metadata +.DS_Store + +# Rust build artifacts +target/ +**/target/ + +# Node, Python, others +node_modules/ +__pycache__/ + +# Tests, docs, images not required +/tests/ +/docs/ + diff --git a/docker/demo/.env.example b/docker/demo/.env.example new file mode 100644 index 000000000..27affc403 --- /dev/null +++ b/docker/demo/.env.example @@ -0,0 +1,144 @@ +# ============================================================================ +# Mega / Orion Demo Environment Configuration File Example +# ============================================================================ +# +# Usage: +# 1. Copy this file to .env: cp .env.example .env +# 2. Modify values in .env as needed (do not commit .env to version control) +# 3. Start services with: docker compose -f docker-compose.demo.yml up -d +# +# IMPORTANT: This configuration is for local Demo / demonstration purposes only, +# NOT suitable for production environments! +# ============================================================================ + +# ---------------------------------------------------------------------------- +# PostgreSQL Database Configuration +# ---------------------------------------------------------------------------- + +POSTGRES_USER=postgres +POSTGRES_PASSWORD=postgres +POSTGRES_DB_MONO=mono +# Note: Orion Server uses the SAME mono database (no separate orion db required) +# Note: campsite database is now managed by MySQL service + +# ---------------------------------------------------------------------------- +# MySQL Database Configuration (for Campsite API) +# ---------------------------------------------------------------------------- +# Note: For demo purposes, using root user only. For production, create dedicated user. + +MYSQL_ROOT_PASSWORD=mysqladmin +MYSQL_DATABASE=campsite + +# ---------------------------------------------------------------------------- +# RustFS (S3-compatible Object Storage) Configuration +# ---------------------------------------------------------------------------- + +RUSTFS_ACCESS_KEY=rustfsadmin +RUSTFS_SECRET_KEY=rustfsadmin +RUSTFS_CONSOLE_ENABLE=true + +# ---------------------------------------------------------------------------- +# S3 Storage Configuration (used by Mega and optionally Orion S3 log store) +# ---------------------------------------------------------------------------- + +S3_ACCESS_KEY_ID=rustfsadmin +S3_SECRET_ACCESS_KEY=rustfsadmin +S3_REGION=us-east-1 + +# ---------------------------------------------------------------------------- +# Docker Image Configuration +# ---------------------------------------------------------------------------- +# All images are pulled from Amazon ECR Public: public.ecr.aws/m8q5m4u3/mega +# You can override these to use local images or different tags +# +# For local builds: +# ORION_SERVER_IMAGE=orion-server:latest +# MEGA_ENGINE_IMAGE=mega:mono-engine-latest + +MEGA_ENGINE_IMAGE=public.ecr.aws/m8q5m4u3/mega:mono-0.1.0-pre-release +MEGA_UI_IMAGE=public.ecr.aws/m8q5m4u3/mega:mega-ui-staging-0.1.0-pre-release +ORION_SERVER_IMAGE=public.ecr.aws/m8q5m4u3/mega:orion-server-0.1.0-pre-release +MEGA_DEV_IMAGE=public.ecr.aws/m8q5m4u3/mega:mega-dev-0.1.0-pre-release +CAMPSITE_API_IMAGE=campsite-api:latest + +# ---------------------------------------------------------------------------- +# Mega Backend Service Configuration +# ---------------------------------------------------------------------------- + +MEGA_DATABASE__DB_TYPE=postgres +MEGA_DATABASE__DB_URL=postgres://postgres:postgres@postgres:5432/mono +MEGA_REDIS__URL=redis://redis:6379 + +MEGA_MONOREPO__STORAGE_TYPE=s3 +MEGA_S3__ENDPOINT_URL=http://rustfs:9000 +MEGA_S3__BUCKET=mega +MEGA_S3__REGION=us-east-1 + +MEGA_LOG__LEVEL=info +MEGA_LOG__PRINT_STD=true + +MEGA_AUTHENTICATION__ENABLE_TEST_USER=true +MEGA_AUTHENTICATION__TEST_USER_NAME=mega +MEGA_AUTHENTICATION__TEST_USER_TOKEN=mega + +MEGA_BUILD__ENABLE_BUILD=true +MEGA_BUILD__ORION_SERVER=http://orion_server:8004 + +MEGA_OAUTH__CAMPSITE_API_DOMAIN=http://campsite_api:8080 +MEGA_OAUTH__UI_DOMAIN=http://localhost:3000 +MEGA_OAUTH__COOKIE_DOMAIN=localhost +MEGA_OAUTH__ALLOWED_CORS_ORIGINS=http://localhost:3000 + +MEGA_BASE_DIR=/opt/mega +MEGA_CACHE_DIR=/opt/mega/cache + +# ---------------------------------------------------------------------------- +# Mega UI (Frontend) Configuration +# ---------------------------------------------------------------------------- + +MEGA_INTERNAL_HOST=http://mega:8000 +MEGA_HOST=http://localhost:8000 +NEXT_PUBLIC_ORION_API_URL=http://orion_server:8004 + +# ---------------------------------------------------------------------------- +# Orion Server Configuration +# ---------------------------------------------------------------------------- +# IMPORTANT: Orion Server uses mono database + +ORION_DATABASE_URL=postgres://postgres:postgres@postgres:5432/mono +ORION_BUILD_LOG_DIR=/tmp/buck2ctl +ORION_ALLOWED_CORS_ORIGINS=http://localhost:3000,http://127.0.0.1:3000,http://localhost:8000 +ORION_LOGGER_STORAGE_TYPE=local +ORION_BUCKET_NAME=orion-logs + +# ---------------------------------------------------------------------------- +# Orion Build Client / Worker Configuration +# ---------------------------------------------------------------------------- + +ORION_WORKER_SERVER_WS=ws://orion_server:8004/ws +ORION_WORKER_ID= +BUCK_PROJECT_ROOT=/workspace +BUILD_TMP=/tmp/orion-builds +SCORPIO_API_BASE_URL=http://127.0.0.1:2725 +ORION_WORKER_START_SCORPIO=true +SCORPIO_BASE_URL=http://mega:8000 +SCORPIO_LFS_URL=http://mega:8000 +RUST_LOG=info + +# ---------------------------------------------------------------------------- +# Campsite API Configuration +# ---------------------------------------------------------------------------- + +CAMPSITE_DATABASE_URL=mysql2://root:mysqladmin@mysql:3306/campsite +CAMPSITE_REDIS_URL=redis://redis:6379 +CAMPSITE_RAILS_ENV=development +CAMPSITE_SERVER_COMMAND=bundle exec puma +CAMPSITE_DEV_APP_URL=http://localhost:3000 +# Generate master key (run before first build): +# VISUAL="code -n --wait" ./bin/rails credentials:edit --environment development +# Copy the generated key and paste it below. +CAMPSITE_RAILS_MASTER_KEY=your-master-key-here + +# Run migrations only on first startup; set to 0 afterwards for faster launches. +# 1 = run migrations (default first run) | 0 = skip migrations +CAMPSITE_RUN_MIGRATIONS=1 \ No newline at end of file diff --git a/docker/demo/docker-compose.demo.yml b/docker/demo/docker-compose.demo.yml new file mode 100644 index 000000000..f74ebc42b --- /dev/null +++ b/docker/demo/docker-compose.demo.yml @@ -0,0 +1,340 @@ +# version: "3.8" # Removed: version is obsolete in Docker Compose v2+ + +services: + # --------------------------------------------------------------------------- + # Core Infra: PostgreSQL + # --------------------------------------------------------------------------- + postgres: + image: postgres:15 + container_name: mega-demo-postgres + environment: + POSTGRES_USER: ${POSTGRES_USER:-postgres} + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:-postgres} + # Main DB is primarily used by mono; orion will be created by init script + # Note: campsite database is now managed by MySQL service + POSTGRES_DB: ${POSTGRES_DB_MONO:-mono} + volumes: + - postgres-data:/var/lib/postgresql/data + ports: + - "5432:5432" + healthcheck: + test: ["CMD-SHELL", "pg_isready -U ${POSTGRES_USER:-postgres}"] + interval: 5s + timeout: 5s + retries: 5 + restart: unless-stopped + + # --------------------------------------------------------------------------- + # Core Infra: Redis + # --------------------------------------------------------------------------- + redis: + image: redis:7 + container_name: mega-demo-redis + command: ["redis-server", "--appendonly", "no"] + volumes: + - redis-data:/data + ports: + - "6379:6379" + healthcheck: + test: ["CMD", "redis-cli", "ping"] + interval: 5s + timeout: 5s + retries: 5 + restart: unless-stopped + + # --------------------------------------------------------------------------- + # Core Infra: MySQL (for Campsite API) + # --------------------------------------------------------------------------- + # Note: For demo purposes, using root user only. For production, create dedicated user. + mysql: + image: mysql:8.0 + container_name: mega-demo-mysql + environment: + MYSQL_ROOT_PASSWORD: ${MYSQL_ROOT_PASSWORD:-mysqladmin} + MYSQL_DATABASE: ${MYSQL_DATABASE:-campsite} + volumes: + - mysql-data:/var/lib/mysql + ports: + - "3306:3306" + healthcheck: + test: ["CMD-SHELL", "mysqladmin ping -h localhost -u root -p${MYSQL_ROOT_PASSWORD:-mysqladmin} || exit 1"] + interval: 5s + timeout: 5s + retries: 5 + restart: unless-stopped + + # --------------------------------------------------------------------------- + # Core Infra: RustFS (S3-compatible object storage) + # --------------------------------------------------------------------------- + rustfs: + image: rustfs/rustfs:latest + container_name: mega-demo-rustfs + # Use 0.0.0.0:9000 to listen on both IPv4 and IPv6 for Docker network compatibility + # Note: RUSTFS_SERVER_DOMAINS is not set to avoid virtual-host style conflicts + # We use path-style addressing (force_path_style=true) in the S3 client + command: --address 0.0.0.0:9000 --console-enable --access-key ${RUSTFS_ACCESS_KEY:-rustfsadmin} --secret-key ${RUSTFS_SECRET_KEY:-rustfsadmin} /data + environment: + RUSTFS_ACCESS_KEY: ${RUSTFS_ACCESS_KEY:-rustfsadmin} + RUSTFS_SECRET_KEY: ${RUSTFS_SECRET_KEY:-rustfsadmin} + RUSTFS_CONSOLE_ENABLE: ${RUSTFS_CONSOLE_ENABLE:-true} + # RUSTFS_SERVER_DOMAINS removed - using path-style addressing instead of virtual-host style + volumes: + - rustfs-data:/data + ports: + # API port: 9000 (container) -> 9000 (host) + # Console port: 9001 (container) -> 9001 (host) - if RustFS console uses separate port + - "9000:9000" + - "9001:9001" + healthcheck: + test: ["CMD-SHELL", "curl -f http://localhost:9000/health || exit 1"] + interval: 10s + timeout: 5s + retries: 5 + restart: unless-stopped + + # --------------------------------------------------------------------------- + # Mega / Mono Engine (Rust backend) + # --------------------------------------------------------------------------- + # Use pre-built image from ECR Public: public.ecr.aws/m8q5m4u3/mega:mono-0.1.0-pre-release + # Or use locally built image: mega:mono-engine-latest (after building with docker build) + # Set MEGA_ENGINE_IMAGE environment variable to override + # Priority: MEGA_ENGINE_IMAGE env var > local build (mega:mono-engine-latest) > ECR image + mega: + image: ${MEGA_ENGINE_IMAGE:-public.ecr.aws/m8q5m4u3/mega:mono-0.1.0-pre-release} + container_name: mega-demo-backend + depends_on: + postgres: + condition: service_healthy + redis: + condition: service_healthy + rustfs: + condition: service_healthy + campsite_api: + condition: service_started + environment: + # Database & Redis + MEGA_DATABASE__DB_TYPE: postgres + MEGA_DATABASE__DB_URL: ${MEGA_DATABASE__DB_URL:-postgres://${POSTGRES_USER:-postgres}:${POSTGRES_PASSWORD:-postgres}@postgres:5432/${POSTGRES_DB_MONO:-mono}} + MEGA_REDIS__URL: ${MEGA_REDIS__URL:-redis://redis:6379} + + # S3 / RustFS + # To use S3 storage, set MEGA_MONOREPO__STORAGE_TYPE=s3 and ensure RustFS bucket is created + MEGA_MONOREPO__STORAGE_TYPE: ${MEGA_MONOREPO__STORAGE_TYPE:-s3} + MEGA_S3__ENDPOINT_URL: ${MEGA_S3__ENDPOINT_URL:-http://rustfs:9000} + # RustFS requires bucket names to follow S3 naming rules: 3-63 chars, lowercase, alphanumeric, hyphens + MEGA_S3__BUCKET: ${MEGA_S3__BUCKET:-mega} + MEGA_S3__REGION: ${MEGA_S3__REGION:-us-east-1} + MEGA_S3__ACCESS_KEY_ID: ${S3_ACCESS_KEY_ID:-rustfsadmin} + MEGA_S3__SECRET_ACCESS_KEY: ${S3_SECRET_ACCESS_KEY:-rustfsadmin} + + # Logging & demo user experience + MEGA_LOG__LEVEL: ${MEGA_LOG__LEVEL:-info} + MEGA_LOG__PRINT_STD: ${MEGA_LOG__PRINT_STD:-true} + MEGA_AUTHENTICATION__ENABLE_TEST_USER: ${MEGA_AUTHENTICATION__ENABLE_TEST_USER:-true} + MEGA_AUTHENTICATION__TEST_USER_NAME: ${MEGA_AUTHENTICATION__TEST_USER_NAME:-mega} + MEGA_AUTHENTICATION__TEST_USER_TOKEN: ${MEGA_AUTHENTICATION__TEST_USER_TOKEN:-mega} + + # Orion build integration + MEGA_BUILD__ENABLE_BUILD: ${MEGA_BUILD__ENABLE_BUILD:-true} + MEGA_BUILD__ORION_SERVER: ${MEGA_BUILD__ORION_SERVER:-http://orion_server:8004} + + # Campsite integration (OAuth / SSO etc.) + MEGA_OAUTH__CAMPSITE_API_DOMAIN: ${MEGA_OAUTH__CAMPSITE_API_DOMAIN:-http://campsite_api:8080} + MEGA_OAUTH__UI_DOMAIN: ${MEGA_OAUTH__UI_DOMAIN:-http://localhost:3000} + MEGA_OAUTH__COOKIE_DOMAIN: ${MEGA_OAUTH__COOKIE_DOMAIN:-localhost} + # Note: allowed_cors_origins expects an array format in TOML + # For environment variable, use JSON array format or remove this line to use config.toml default + # MEGA_OAUTH__ALLOWED_CORS_ORIGINS: ${MEGA_OAUTH__ALLOWED_CORS_ORIGINS:-["http://localhost:3000"]} + + # Directory paths (can be overridden) + MEGA_BASE_DIR: ${MEGA_BASE_DIR:-/opt/mega} + MEGA_CACHE_DIR: ${MEGA_CACHE_DIR:-/opt/mega/cache} + volumes: + - mega-data:/opt/mega + ports: + # Container listens on 8000, host port is 8000 + - "8000:8000" + healthcheck: + # Mega backend health check + # NOTE: The ECR mono image may not include curl/wget/busybox. Use bash built-in /dev/tcp. + test: ["CMD-SHELL", "bash -lc 'echo > /dev/tcp/127.0.0.1/8000' || exit 1"] + interval: 10s + timeout: 5s + retries: 10 + restart: unless-stopped + + # --------------------------------------------------------------------------- + # Mega UI (Next.js) + # --------------------------------------------------------------------------- + # Use pre-built image from ECR Public: public.ecr.aws/m8q5m4u3/mega:mega-ui-{env}-0.1.0-pre-release + # Available environments: staging, openatom, gitmono + # Set MEGA_UI_IMAGE environment variable to use ECR image, or tag the ECR image as mega:mono-ui-latest-release + # Default uses local tag (mega:mono-ui-latest-release) if ECR image is tagged locally + mega_ui: + image: ${MEGA_UI_IMAGE:-public.ecr.aws/m8q5m4u3/mega:mega-ui-staging-0.1.0-pre-release} + container_name: mega-demo-ui + depends_on: + mega: + condition: service_healthy + orion_server: + condition: service_healthy + environment: + PORT: 3000 + # Ensure Next.js listens on all interfaces (0.0.0.0) for healthcheck to work + HOSTNAME: 0.0.0.0 + MEGA_INTERNAL_HOST: ${MEGA_INTERNAL_HOST:-http://mega:8000} + MEGA_HOST: ${MEGA_HOST:-http://localhost:8000} + # Use internal Orion Server address by default to avoid browser CORS issues + NEXT_PUBLIC_ORION_API_URL: ${NEXT_PUBLIC_ORION_API_URL:-http://orion_server:8004} + ports: + # Container listens on 3000, host port is 3000 + - "3000:3000" + healthcheck: + # NOTE: mega-ui image may not include curl/wget/busybox. Use bash built-in /dev/tcp. + # With HOSTNAME=0.0.0.0, Next.js will listen on all interfaces, so 127.0.0.1 should work + test: ["CMD-SHELL", "bash -lc 'echo > /dev/tcp/127.0.0.1/3000' || exit 1"] + interval: 10s + timeout: 5s + retries: 10 + restart: unless-stopped + + # --------------------------------------------------------------------------- + # Orion Server (Rust) + # --------------------------------------------------------------------------- + orion_server: + # Use pre-built image from ECR Public instead of building locally + # To build locally, replace with: + # build: + # context: .. + # dockerfile: orion-server/Dockerfile + image: ${ORION_SERVER_IMAGE:-public.ecr.aws/m8q5m4u3/mega:orion-server-0.1.0-pre-release} + container_name: mega-demo-orion-server + depends_on: + postgres: + condition: service_healthy + environment: + DATABASE_URL: ${ORION_DATABASE_URL:-postgres://${POSTGRES_USER:-postgres}:${POSTGRES_PASSWORD:-postgres}@postgres:5432/${POSTGRES_DB_MONO:-mono}} + PORT: 8004 + BUILD_LOG_DIR: ${ORION_BUILD_LOG_DIR:-/tmp/buck2ctl} + ALLOWED_CORS_ORIGINS: ${ORION_ALLOWED_CORS_ORIGINS:-http://localhost:3000,http://127.0.0.1:3000,http://localhost:8000} + LOGGER_STORAGE_TYPE: ${ORION_LOGGER_STORAGE_TYPE:-local} + BUCKET_NAME: ${ORION_BUCKET_NAME:-orion-logs} + AWS_ACCESS_KEY_ID: ${S3_ACCESS_KEY_ID:-rustfsadmin} + AWS_SECRET_ACCESS_KEY: ${S3_SECRET_ACCESS_KEY:-rustfsadmin} + AWS_DEFAULT_REGION: ${S3_REGION:-us-east-1} + volumes: + - orion-logs:/tmp/buck2ctl + ports: + # Container listens on 8004, host port is 8004 + - "8004:8004" + healthcheck: + # Orion Server health check - use /v2/health endpoint + test: ["CMD-SHELL", "curl -f http://localhost:8004/v2/health || exit 1"] + interval: 10s + timeout: 5s + retries: 10 + restart: unless-stopped + + # --------------------------------------------------------------------------- + # Orion Build Client / Worker (based on mega-dev image) + # --------------------------------------------------------------------------- + orion_build_client: + image: ${MEGA_DEV_IMAGE:-public.ecr.aws/m8q5m4u3/mega:mega-dev-0.1.0-pre-release} + container_name: mega-demo-orion-worker + command: orion + # privileged mode allows nested container builds (Scorpio) inside the worker. + privileged: true + depends_on: + orion_server: + condition: service_started + environment: + SERVER_WS: ${ORION_WORKER_SERVER_WS:-ws://orion_server:8004/ws} + ORION_WORKER_ID: ${ORION_WORKER_ID:-} + BUCK_PROJECT_ROOT: ${BUCK_PROJECT_ROOT:-/workspace} + BUILD_TMP: ${BUILD_TMP:-/tmp/orion-builds} + SCORPIO_API_BASE_URL: ${SCORPIO_API_BASE_URL:-http://127.0.0.1:2725} + ORION_WORKER_START_SCORPIO: ${ORION_WORKER_START_SCORPIO:-true} + SCORPIO_BASE_URL: ${SCORPIO_BASE_URL:-http://mega:8000} + SCORPIO_LFS_URL: ${SCORPIO_LFS_URL:-http://mega:8000} + SCORPIO_STORE_PATH: /data/scorpio/store + SCORPIO_WORKSPACE: /workspace/mount + RUST_LOG: ${RUST_LOG:-info} + volumes: + - orion-worker-scorpio-data:/data/scorpio + - orion-worker-workspace:/workspace + restart: unless-stopped + + # --------------------------------------------------------------------------- + # Campsite API (Ruby) + # --------------------------------------------------------------------------- + # Use pre-built image from ECR Public: public.ecr.aws/m8q5m4u3/mega:campsite-0.1.0-pre-release + # Note: Campsite API requires MySQL database, not PostgreSQL + campsite_api: + image: ${CAMPSITE_API_IMAGE:-public.ecr.aws/m8q5m4u3/mega:campsite-0.1.0-pre-release} + container_name: mega-demo-campsite-api + depends_on: + mysql: + condition: service_healthy + redis: + condition: service_healthy + environment: + # MySQL connection URL format: mysql2://user:password@host:port/database + # Using root user for demo purposes + DATABASE_URL: ${CAMPSITE_DATABASE_URL:-mysql2://root:${MYSQL_ROOT_PASSWORD:-mysqladmin}@mysql:3306/${MYSQL_DATABASE:-campsite}} + REDIS_URL: ${CAMPSITE_REDIS_URL:-redis://redis:6379} + PORT: 8080 + # Allow toggling migrations from .env + CAMPSITE_RUN_MIGRATIONS: ${CAMPSITE_RUN_MIGRATIONS:-1} + + # Align with ECS style env vars (for local demo use dev) + RAILS_ENV: ${CAMPSITE_RAILS_ENV:-development} + RAILS_MASTER_KEY: ${CAMPSITE_RAILS_MASTER_KEY:-} + SERVER_COMMAND: ${CAMPSITE_SERVER_COMMAND:-bundle exec puma} + DEV_APP_URL: ${CAMPSITE_DEV_APP_URL:-http://localhost:3000} + + command: + - /bin/bash + - -lc + - | + # Wait for MySQL to accept TCP connections (service_healthy can still be a bit early) + until (echo > /dev/tcp/mysql/3306) >/dev/null 2>&1; do + echo "waiting mysql..." + sleep 1 + done + + # Run migrations by default; allow disabling for debugging via CAMPSITE_RUN_MIGRATIONS=0 + # Use $$ to avoid docker compose substituting the variable at parse time. + if [ "$${CAMPSITE_RUN_MIGRATIONS:-1}" = "1" ]; then + bin/rails db:migrate + else + echo "Skipping db:migrate (CAMPSITE_RUN_MIGRATIONS=$${CAMPSITE_RUN_MIGRATIONS})" + fi + + exec ${CAMPSITE_SERVER_COMMAND:-bundle exec puma} + ports: + # Container listens on 8080, host port is 8080 + - "8080:8080" + healthcheck: + # Campsite API health check + # NOTE: avoid curl dependency; use bash built-in /dev/tcp + test: ["CMD-SHELL", "bash -lc 'echo > /dev/tcp/127.0.0.1/8080' || exit 1"] + interval: 10s + timeout: 5s + retries: 10 + restart: unless-stopped + +networks: + default: + name: mega-demo-network + +volumes: + postgres-data: + redis-data: + mysql-data: + rustfs-data: + mega-data: + orion-logs: + orion-worker-scorpio-data: + orion-worker-workspace: + + diff --git a/docs/demo/docker-compose-guide-for-mega-and-orion-demo-setup.md b/docs/demo/docker-compose-guide-for-mega-and-orion-demo-setup.md new file mode 100644 index 000000000..60143c4d3 --- /dev/null +++ b/docs/demo/docker-compose-guide-for-mega-and-orion-demo-setup.md @@ -0,0 +1,519 @@ +# Mega / Orion Demo Environment – Docker Compose Guide + +> ⚠️ **Important Notice**: This Docker Compose setup is **for local demo/development only**. Do **NOT** run it in production. Default passwords and test users are included. + +## Table of Contents + +- [Prerequisites](#prerequisites) +- [Quick Start](#quick-start) +- [Demo Walk-Through](#demo-walk-through) +- [Service Endpoints](#service-endpoints) +- [FAQ](#faq) +- [Stopping & Cleanup](#stopping--cleanup) +- [Log Streaming](#log-streaming) +- [Architecture Overview](#architecture-overview) + +--- + +## Prerequisites + +### System Requirements + +* Linux – native Docker & Compose +* macOS – Docker Desktop +* Windows – Docker Desktop (WSL2 recommended) + +**Hardware** (minimum): + +* CPU: 4 cores +* RAM: 8 GB (16 GB recommended for source builds) +* Disk: 10 GB free (30 GB recommended if building images locally) + + +### Required Tools + +* Docker ≥ 20.10 +* Docker Compose ≥ 2 (bundled with Docker Desktop) + +```bash +docker --version +docker compose version +``` + +--- + +## Quick Start + +### 1 Clone the repo + +```bash +git clone https://github.com/web3infra-foundation/mega.git +cd mega +``` + +### 2 Configure environment *(optional)* + +The demo environment already has sensible default values and can be used as-is. To customize any settings, create a `.env` file under `docker/demo/`: + +```bash +cd docker/demo +# (Optional) copy `.env.example` to `.env` and edit as needed +``` + +The main configurable environment variables include: + +- **Database Configuration**: + - `POSTGRES_USER`: PostgreSQL username (default: `postgres`) + - `POSTGRES_PASSWORD`: PostgreSQL password (default: `postgres`) + - `POSTGRES_DB_MONO`: PostgreSQL database name (default: `mono`, shared by Mega + Orion Server) + - `MYSQL_ROOT_PASSWORD`: MySQL root password (default: `mysqladmin`) + - ⚠️ For production, create a **dedicated low-privilege user** and update the MySQL health-check accordingly (avoid embedding root password). + - `MYSQL_DATABASE`: Campsite database name (default: `campsite`, uses MySQL) + +- **Service Images**: + - `MEGA_ENGINE_IMAGE`: Mega backend image (default: `public.ecr.aws/m8q5m4u3/mega:mono-0.1.0-pre-release`) + - `MEGA_UI_IMAGE`: Mega UI image (default: `public.ecr.aws/m8q5m4u3/mega:mega-ui-staging-0.1.0-pre-release`) + - `MEGA_DEV_IMAGE`: Orion Build Client image (default: `public.ecr.aws/m8q5m4u3/mega:mega-dev-0.1.0-pre-release`) + - `CAMPSITE_API_IMAGE`: Campsite API image (default uses locally built `campsite-api:latest`, source repo: `https://github.com/web3infra-foundation/campsite`; if development environment encryption configuration (Master Key, etc.) is ready, you can directly set it to `public.ecr.aws/m8q5m4u3/mega:campsite-0.1.0-pre-release` to pull the pre-built image) + - Before building the image, you need to generate a Rails `master key` and fill it into the `CAMPSITE_RAILS_MASTER_KEY` field in the `.env` file: + 1. Navigate to the Campsite source code directory. + 2. Execute `VISUAL="code -n --wait" ./bin/rails credentials:edit --environment development`. + 3. After saving, copy the generated `master key` from the prompt in the terminal. + 4. Write this key to `CAMPSITE_RAILS_MASTER_KEY=` in `docker/demo/.env` (copied from `.env.example`). + - `CAMPSITE_RUN_MIGRATIONS`: Whether to run database migrations when the container starts; `1` (default) to run, can be changed to `0` after the first successful migration to skip and speed up subsequent starts. + +- **RustFS Configuration**: + - `RUSTFS_ACCESS_KEY`: RustFS access key (default: `rustfsadmin`) + - `RUSTFS_SECRET_KEY`: RustFS secret key (default: `rustfsadmin`) + +> **Note**: The demo environment uses default passwords and test users for demonstration purposes only. + +### 3. Start all services + +Execute in the project root directory: + +```bash +docker compose -f docker/demo/docker-compose.demo.yml up -d +``` + +This command will: + +1. Pull the required Docker images (may take a long time for the first run) +2. Create Docker networks and volumes +3. Start all services in dependency order: + - First, start infrastructure services (PostgreSQL, MySQL, Redis, RustFS) + - Then, start application services (Mega, Orion Server, Campsite API) + - Finally, start client services (Mega UI, Orion Build Client) + +### 4. Check service status + +View the status of all services: + +```bash +docker compose -f docker/demo/docker-compose.demo.yml ps +``` + +View service logs (follow): + +```bash +docker compose -f docker/demo/docker-compose.demo.yml logs -f +``` + +View logs for specific services: + +```bash +docker compose -f docker/demo/docker-compose.demo.yml logs -f mega +docker compose -f docker/demo/docker-compose.demo.yml logs -f orion_server +``` + +### 5. Wait for services to become ready + +On the first startup, services may take some time to finish: + +- **Database initialization**: PostgreSQL and MySQL need to initialize the databases +- **Service health checks**: Each service waits for its dependencies to become healthy before starting +- **Image build**: If using locally built images, the `mega` and `orion_server` services need to be built from source (slower on the first run) +- **PostgreSQL init script**: On the very first launch the container runs `docker/demo/init-db.sh` automatically (mounted into `/docker-entrypoint-initdb.d/`). The script does not create extra schemas; it simply prints helpful hints and reminds you that the `mono` database is auto-created by the `POSTGRES_DB` variable. Because the PostgreSQL data directory is persisted in the `postgres-data` volume, this script is executed only **once** unless you delete the volume. + +Typically you should wait **2–5 minutes**. You can monitor service health with the following command: + +```bash +# View the health status of all services +docker compose -f docker/demo/docker-compose.demo.yml ps +``` + +When all services show a status of `healthy` or `running`, you can start using the demo. + +--- + +## Demo walk-through + +### 1. Open Mega UI + +Open your browser and visit: + +``` +http://localhost:3000 +``` + +### 2. Sign in with the test user + +The demo environment includes a built-in test user you can use directly: + +- **Username**: `mega` (or as configured by `MEGA_AUTHENTICATION__TEST_USER_NAME`) +- **Token**: `mega` (or as configured by `MEGA_AUTHENTICATION__TEST_USER_TOKEN`) + +### 3. Trigger an Orion build + +In Mega UI: + +1. Create a new monorepo project or select an existing one +2. On the project page, find the build-related features +3. Trigger a Buck2 build task +4. The build request will be sent to Orion Server and executed by Orion Build Client + +### 4. View build results + +- **View in the UI**: Build status and logs are displayed in Mega UI +- **View build client logs**: + ```bash + docker compose -f docker/demo/docker-compose.demo.yml logs -f orion_build_client + ``` +- **View Orion Server logs**: + ```bash + docker compose -f docker/demo/docker-compose.demo.yml logs -f orion_server + ``` + +### 5. Access the RustFS console (optional) + +RustFS object storage provides a web console: + +``` +http://localhost:9001/rustfs/console/access-keys +``` + +Log in with the following credentials: +- **Access Key**: `rustfsadmin` (or the value of `RUSTFS_ACCESS_KEY`) +- **Secret Key**: `rustfsadmin` (or the value of `RUSTFS_SECRET_KEY`) + +--- + +## Service Endpoints + +| Service | URL | Description | +|------|---------|------| +| **Mega UI** | http://localhost:3000 | Web Frontend UI | +| **Mega API** | http://localhost:8000 | Mega backend API | +| **Orion Server** | http://localhost:8004 | Orion build server API | +| **Campsite API** | http://localhost:8080 | Campsite OAuth/SSO API | +| **PostgreSQL** | localhost:5432 | Database (used by Mega & Orion, mapped to host port 5432 in demo) | +| **MySQL** | localhost:3306 | Database (used by Campsite API, mapped to host port 3306 in demo) | +| **Redis** | localhost:6379 | Cache service (mapped to host port 6379 in demo) | +| **RustFS** | http://localhost:9001 | Object storage service & console | + +### API Health Check Endpoints + +- **Mega API**: `GET http://localhost:8000/api/v1/status` +- **Orion Server**: `GET http://localhost:8004/v2/health` +- **Campsite API**: `GET http://localhost:8080/health` + +--- + +## FAQ + +### Port Conflict + +**Issue**: Docker reports the port is already allocated + +**Solution**: + +1. **Update the port mapping in the compose file**: + Edit `docker/demo/docker-compose.demo.yml` and adjust the `ports` section, e.g.: + ```yaml + ports: + - "8001:8000" # Change host port to 8001 + ``` + +2. **Stop the service occupying the port**: + ```bash + # Find the process occupying the port (Linux/macOS) + lsof -i :8000 + # or use netstat (Windows) + netstat -ano | findstr :8000 + ``` + +### Slow First-Time Start + +**Issue**: First run of `docker compose up` takes a long time + +**Reason**: +- Images must be pulled from remote registries (may be large) +- If you are using locally built images, the `mega` and `orion_server` services need to be built from source +- PostgreSQL and MySQL databases need to initialize + +**Solution**: +- Be patient; the first startup usually takes **5–15 minutes** (depending on network speed and hardware) +- You can view progress in real time with `docker compose logs -f` +- Subsequent starts will be much faster (images are cached) + +### Service Start Failure or Health Check Failure + +**Issue**: Some services remain in `unhealthy` or `restarting` state + +**Troubleshooting Steps**: + +1. **View service logs**: + ```bash + docker compose -f docker/demo/docker-compose.demo.yml logs + ``` + +2. **Check dependency services**: + Ensure infrastructure services (PostgreSQL, MySQL, Redis, RustFS) are healthy: + ```bash + docker compose -f docker/demo/docker-compose.demo.yml ps + ``` + +3. **Check environment variables**: + Verify the `.env` file (if present) has correct settings + +4. **Check network connectivity**: + Ensure container-to-container network communication is normal: + ```bash + docker network inspect mega-demo-network + ``` + +5. **Restart a service**: + ```bash + docker compose -f docker/demo/docker-compose.demo.yml restart + ``` + +### Orion Build Client cannot connect to Orion Server + +**Problem**: The `orion_build_client` container cannot connect to `orion_server`. + +**Possible causes**: +- `orion_server` has not fully started yet +- Incorrect WebSocket address configuration +- Network issues + +**Solution**: + +1. Check whether `orion_server` is healthy: + ```bash + docker compose -f docker/demo/docker-compose.demo.yml ps orion_server + ``` + +2. Inspect `orion_build_client` logs: + ```bash + docker compose -f docker/demo/docker-compose.demo.yml logs orion_build_client + ``` + +3. Verify that the environment variable `ORION_WORKER_SERVER_WS` is configured correctly (default: `ws://orion_server:8004/ws`). + +### Database connection failure + +**Problem**: Mega, Orion, or Campsite cannot connect to the database. + +**Troubleshooting steps**: + +1. **Check whether PostgreSQL is healthy** (used by Mega and Orion): + ```bash + docker compose -f docker/demo/docker-compose.demo.yml ps postgres + ``` + +2. **Check whether MySQL is healthy** (used by the Campsite API): + ```bash + docker compose -f docker/demo/docker-compose.demo.yml ps mysql + ``` + +3. **Check database connection strings**: + - Mega/Orion: ensure `DATABASE_URL` or `MEGA_DATABASE__DB_URL` is correctly formatted (PostgreSQL) + - Campsite: ensure `CAMPSITE_DATABASE_URL` is correctly formatted (MySQL; format: `mysql2://user:password@host:port/database`) + +4. **Test connectivity manually**: + ```bash + # Test PostgreSQL connection (Mega/Orion) + docker compose -f docker/demo/docker-compose.demo.yml exec postgres psql -U postgres -d mono + + # Test MySQL connection (Campsite) + docker compose -f docker/demo/docker-compose.demo.yml exec mysql mysql -u root -p${MYSQL_ROOT_PASSWORD:-mysqladmin} -e "USE campsite; SELECT 1;" + ``` + +### RustFS access failure + +**Problem**: Mega or Orion cannot access RustFS object storage. + +**Troubleshooting steps**: + +1. Check whether RustFS is healthy: + ```bash + docker compose -f docker/demo/docker-compose.demo.yml ps rustfs + ``` + +2. Check S3 configuration: + Ensure `MEGA_S3__ENDPOINT_URL` points to `http://rustfs:9000` (in-container address). + +3. Check access keys: + Ensure `S3_ACCESS_KEY_ID` and `S3_SECRET_ACCESS_KEY` match the RustFS configuration. + +### Image build failure + +**Problem**: `orion_server` failed to build. + +**Possible causes**: +- Docker build context issues +- Network issues (unable to download dependencies) +- Insufficient disk space + +**Solution**: + +1. View detailed build logs: + ```bash + docker compose -f docker/demo/docker-compose.demo.yml build --no-cache orion_server + ``` + +2. Check disk space: + ```bash + df -h # Linux/macOS + ``` + +3. Prune Docker cache: + ```bash + docker system prune -a + ``` + +--- + +## Stopping and Cleanup + +### Stop services + +Stop all services (keep data): + +```bash +docker compose -f docker/demo/docker-compose.demo.yml stop +``` + +Stop and remove containers (keep volumes): + +```bash +docker compose -f docker/demo/docker-compose.demo.yml down +``` + +### Full cleanup (remove all data) + +⚠️ **Warning**: The following command will delete all volumes, including database and object storage data. Proceed with caution! + +```bash +docker compose -f docker/demo/docker-compose.demo.yml down -v +``` + +### Clean images (optional) + +To remove demo-related Docker images: + +```bash +# List images +docker images | grep mega + +# Remove specific image +docker rmi +``` + +--- + +## View logs + +### View logs for all services + +```bash +docker compose -f docker/demo/docker-compose.demo.yml logs -f +``` + +### View logs for a specific service + +```bash +# Mega backend +docker compose -f docker/demo/docker-compose.demo.yml logs -f mega + +# Mega UI +docker compose -f docker/demo/docker-compose.demo.yml logs -f mega_ui + +# Orion Server +docker compose -f docker/demo/docker-compose.demo.yml logs -f orion_server + +# Orion Build Client +docker compose -f docker/demo/docker-compose.demo.yml logs -f orion_build_client + +# Campsite API +docker compose -f docker/demo/docker-compose.demo.yml logs -f campsite_api + +# PostgreSQL +docker compose -f docker/demo/docker-compose.demo.yml logs -f postgres + +# MySQL +docker compose -f docker/demo/docker-compose.demo.yml logs -f mysql + +# Redis +docker compose -f docker/demo/docker-compose.demo.yml logs -f redis + +# RustFS +docker compose -f docker/demo/docker-compose.demo.yml logs -f rustfs +``` + +### View the last N lines of logs + +```bash +docker compose -f docker/demo/docker-compose.demo.yml logs --tail=100 +``` + +### View logs for a specific time range + +```bash +docker compose -f docker/demo/docker-compose.demo.yml logs --since 10m +``` + +--- + +## Architecture overview + +The demo environment includes the following services: + +- **Infrastructure**: + - `postgres`: PostgreSQL database (used by Mega and Orion Server) + - `mysql`: MySQL database (used by the Campsite API) + - `redis`: Redis cache + - `rustfs`: RustFS object storage (S3-compatible) + +- **Application services**: + - `mega`: Mega backend (Rust) + - `mega_ui`: Mega Web UI (Next.js) + - `orion_server`: Orion build server (Rust) + - `orion_build_client`: Orion build client (based on the mega-dev image) + - `campsite_api`: Campsite API (Ruby/Rails, built locally by default; if you have the encrypted development credentials configured you can pull the pre-built image directly via `CAMPSITE_API_IMAGE=public.ecr.aws/m8q5m4u3/mega:campsite-0.1.0-pre-release`) + +For a detailed architecture diagram and dependency list, see the [Mega / Orion Demo architecture design document](./mega-orion-demo-compose-arch.md). + +--- + +## Getting help + +If you run into issues, you can: + +1. Read the [FAQ](#faq) section of this document. +2. Check the service logs for troubleshooting. + +--- + +## Warning! + +⚠️ **This Docker Compose configuration is intended for local demo / evaluation only and is NOT suitable for production.** + +The demo environment includes the following insecure settings: +- Default passwords and test users +- HTTPS disabled +- No security policies configured +- Simple data-persistence setup + +**Do NOT use this configuration in production!** + diff --git a/orion-server/Dockerfile b/orion-server/Dockerfile index 811e75f86..0ef1de755 100644 --- a/orion-server/Dockerfile +++ b/orion-server/Dockerfile @@ -1,24 +1,32 @@ # ────── Stage 1: Build ────── FROM rust:1.92-bookworm AS builder -# 构建上下文中的父目录将作为 /app 挂载 +# Configure Cargo with increased retry attempts and timeout +RUN mkdir -p /root/.cargo && \ + cat > /root/.cargo/config.toml << 'EOF' +[net] +retry = 10 +git-fetch-with-cli = true +EOF + +# The parent directory of build context will be mounted as /app WORKDIR /app -# 复制整个项目结构(从根目录开始) +# Copy the entire project structure (from repository root) COPY . . -# 进入子项目目录 +# Enter subproject directory WORKDIR /app/orion-server -# 构建 release 二进制 +# Build release binary RUN cargo build --release # ────── Stage 2: Runtime ────── FROM debian:bookworm-slim -# 安装运行依赖 +# Install runtime dependencies (including curl for health checks) RUN apt-get update && \ - apt-get install -y --no-install-recommends ca-certificates && \ + apt-get install -y --no-install-recommends ca-certificates curl && \ rm -rf /var/lib/apt/lists/* ENV BUILD_LOG_DIR="/tmp/buck2ctl" @@ -26,10 +34,10 @@ RUN mkdir -p ${BUILD_LOG_DIR} ENV PORT=8004 -# 拷贝构建好的可执行文件 +# Copy built executable COPY --from=builder /app/target/release/orion-server /usr/local/bin/app -# 减少体积(可选) +# Reduce image size (optional) RUN strip /usr/local/bin/app || true EXPOSE 8004 diff --git a/orion-server/src/api.rs b/orion-server/src/api.rs index f54ad87d0..db406cb2f 100644 --- a/orion-server/src/api.rs +++ b/orion-server/src/api.rs @@ -22,7 +22,7 @@ use orion::ws::{TaskPhase, WSMessage}; use rand::Rng; use sea_orm::{ ActiveValue::Set, ColumnTrait, DatabaseConnection, EntityTrait, QueryFilter as _, QueryOrder, - prelude::DateTimeUtc, + QuerySelect, prelude::DateTimeUtc, }; use serde::{Deserialize, Serialize}; use serde_json::{Value, json}; @@ -157,6 +157,7 @@ pub fn routers() -> Router { get(get_orion_client_status_by_id), ) .route("/retry-build", post(build_retry_handler)) + .route("/v2/health", get(health_check_handler)) } /// Start queue management background task (event-driven + periodic cleanup) @@ -178,6 +179,30 @@ pub async fn queue_stats_handler(State(state): State) -> impl IntoResp (StatusCode::OK, Json(stats)) } +/// Health check endpoint for Orion Server +/// Returns simple health status based on database connectivity +#[utoipa::path( + get, + path = "/v2/health", + responses( + (status = 200, description = "Service is healthy", body = serde_json::Value), + (status = 503, description = "Service is unhealthy", body = serde_json::Value) + ) +)] +pub async fn health_check_handler(State(state): State) -> impl IntoResponse { + // Simple health check: verify database connectivity + match tasks::Entity::find().limit(1).all(&state.conn).await { + Ok(_) => (StatusCode::OK, Json(json!({"status": "healthy"}))), + Err(e) => { + tracing::error!("Health check failed: {}", e); + ( + StatusCode::SERVICE_UNAVAILABLE, + Json(json!({"status": "unhealthy", "error": "database connectivity check failed"})), + ) + } + } +} + /// Streams build output logs in real-time using Server-Sent Events (SSE) /// Continuously monitors the log file and streams new content as it becomes available #[utoipa::path( diff --git a/orion-server/src/server.rs b/orion-server/src/server.rs index e99ae6328..bd2f2a24a 100644 --- a/orion-server/src/server.rs +++ b/orion-server/src/server.rs @@ -31,7 +31,8 @@ use crate::{ api::task_targets_handler, api::get_orion_clients_info, api::get_orion_client_status_by_id, - api::build_retry_handler + api::build_retry_handler, + api::health_check_handler ), components( schemas(