feat: add worker mode support for horizontal scaling (#3903)

Author: ildyria

.env.example

@@ -124,8 +124,10 @@ LOG_STDOUT=false
 SESSION_DRIVER=file
 SESSION_LIFETIME=120
 
-# `sync` if jobs needs to be executed live (default) or `database` if they can be defered.
+# `sync` if jobs need to be executed live (default) or `database` if they can be deferred.
 QUEUE_CONNECTION=sync
+# Choose this mode only if you have set up a queue worker (strongly recommended though).
+# QUEUE_CONNECTION=database
 
 SECURITY_HEADER_HSTS_ENABLE=false
 SECURITY_HEADER_CSP_CONNECT_SRC=

Dockerfile

@@ -140,4 +140,8 @@ EXPOSE 8000
 ENTRYPOINT ["/usr/local/bin/entrypoint.sh"]
 
 # Default command: run Octane with FrankenPHP
+# Container mode is controlled by LYCHEE_MODE environment variable:
+# - "web" (default): Runs FrankenPHP/Octane web server (this CMD)
+# - "worker": Runs Laravel queue worker (entrypoint.sh overrides CMD)
+# See docs/specs/2-how-to/deploy-worker-mode.md for deployment guide
 CMD ["php", "artisan", "octane:start", "--server=frankenphp", "--host=0.0.0.0", "--port=8000"]

docker-compose.yaml

@@ -121,7 +121,7 @@ services:
 
       # Cache
       CACHE_STORE: "${CACHE_STORE:-file}"
-      CACHE_PREFIX: "${CACHE_PREFIX:-lychee_keygen_cache}"
+      CACHE_PREFIX: "${CACHE_PREFIX:-lychee_cache}"
       REDIS_HOST: "lychee_cache"
       REDIS_PASSWORD: "null"
       REDIS_PORT: "6379"
@@ -142,8 +142,6 @@ services:
       LOG_LEVEL: "${LOG_LEVEL:-debug}"
       LOG_STDOUT: "${LOG_STDOUT:-true}"
 
-      # QUEUE_CONNECTION:sync
-
       # SECURITY_HEADER_HSTS_ENABLE:false
       # SECURITY_HEADER_CSP_CONNECT_SRC:
       # SECURITY_HEADER_SCRIPT_SRC_ALLOW:
@@ -288,6 +286,131 @@ services:
     networks:
       - lychee
 
+  # Queue Worker Service (disabled by default)
+  # Uncomment to enable horizontal scaling of background job processing
+  lychee_worker:
+    image: lychee-frankenphp:latest
+    # build:
+    #   context: ./app
+    #   dockerfile: Dockerfile
+    #   args:
+    #     NODE_ENV: "${NODE_ENV:-production}"
+    container_name: lychee-worker
+    restart: unless-stopped  # Auto-restart at container level (outer layer)
+
+    # Security hardening (match lychee_api)
+    security_opt:
+      - no-new-privileges:true
+      - seccomp:unconfined
+    cap_drop:
+      - ALL
+    cap_add:
+      - CHOWN
+      - SETGID
+      - SETUID
+      - DAC_OVERRIDE
+    read_only: false  # Laravel needs write access to storage/cache
+    tmpfs:
+      - /tmp:noexec,nosuid,nodev,size=100m
+
+    # Resource limits (adjust based on workload)
+    # Workers typically need less CPU but more memory for image processing
+    deploy:
+      resources:
+        limits:
+          cpus: '2'
+          memory: 2G
+        reservations:
+          cpus: '0.5'
+          memory: 512M
+
+    env_file:
+      - path: ./.env
+        required: true
+    environment:
+      PUID: "${PUID:-1000}"
+      PGID: "${PGID:-1000}"
+
+      # WORKER MODE CONFIGURATION
+      # Set LYCHEE_MODE=worker to enable queue worker mode
+      LYCHEE_MODE: worker
+
+      # Queue configuration (CRITICAL for worker mode)
+      # Recommended: redis for production (faster, better concurrency)
+      # Alternative: database (no Redis dependency)
+      QUEUE_CONNECTION: "${QUEUE_CONNECTION:-database}"
+
+      # Queue priority (optional)
+      # Process high-priority jobs first, then default, then low
+      # Example: QUEUE_NAMES=high,default,low
+      # QUEUE_NAMES: "${QUEUE_NAMES:-default}"
+
+      # Worker restart interval (optional)
+      # Restart worker after N seconds to mitigate memory leaks
+      # Default: 3600 seconds (1 hour)
+      # WORKER_MAX_TIME: "${WORKER_MAX_TIME:-3600}"
+
+      # Application (inherit from lychee_api)
+      APP_NAME: "${APP_NAME:-Lychee}"
+      APP_ENV: "${APP_ENV:-production}"
+      APP_DEBUG: "${APP_DEBUG:-false}"
+      APP_TIMEZONE: "${TIMEZONE:-UTC}"
+      APP_URL: "${APP_URL:-http://localhost:8000}"
+
+      # Database (same as lychee_api - shared database required)
+      DB_CONNECTION: "${DB_CONNECTION:-mysql}"
+      DB_HOST: "${DB_HOST:-lychee_db}"
+      DB_PORT: "${DB_PORT:-3306}"
+      DB_DATABASE: "${DB_DATABASE:-lychee}"
+      DB_USERNAME: "${DB_USERNAME:-lychee}"
+      # DB_PASSWORD comes from .env file
+
+      # Redis (if using redis queue driver)
+      # Must point to same Redis instance as lychee_api
+      REDIS_HOST: "lychee_cache"
+      REDIS_PASSWORD: "null"
+      REDIS_PORT: "6379"
+
+      # Cache (same as lychee_api)
+      CACHE_STORE: "${CACHE_STORE:-file}"
+      CACHE_PREFIX: "${CACHE_PREFIX:-lychee_cache}"
+
+      # Logging (output to stdout/stderr for container log aggregation)
+      LOG_CHANNEL: "${LOG_CHANNEL:-stack}"
+      LOG_STACK: "${LOG_STACK:-single}"
+      LOG_LEVEL: "${LOG_LEVEL:-debug}"
+      LOG_STDOUT: "${LOG_STDOUT:-true}"
+
+    # Shared volumes with lychee_api (CRITICAL for photo processing)
+    # Workers need access to the same uploads and storage as web service
+    volumes:
+      - ./lychee/uploads:/app/public/uploads
+      - ./lychee/storage/app:/app/storage/app
+      - ./lychee/logs:/app/storage/logs
+      - ./lychee/tmp:/app/storage/tmp
+      - .env:/app/.env:ro
+
+    depends_on:
+      lychee_db:
+        condition: service_healthy
+
+    # Worker health check
+    # Verifies queue:work process is running
+    healthcheck:
+      test: ["CMD-SHELL", "pgrep -f 'queue:work' || exit 1"]
+      interval: 30s
+      timeout: 10s
+      retries: 3
+      start_period: 60s  # Give worker time to start up
+
+    networks:
+      - lychee
+  #
+  # SCALING WORKERS:
+  # To run multiple worker containers (horizontal scaling):
+  #   docker compose up -d --scale lychee_worker=3
+  # This will create 3 worker containers processing jobs in parallel.
+
   lychee_cache:
       image: redis:alpine
       hostname: lychee_cache

docker/scripts/entrypoint.sh

@@ -77,5 +77,78 @@ php artisan view:cache
 
 echo "✅ Application ready!"
 
-# Execute the main command
-exec "$@"
+# Detect LYCHEE_MODE and execute appropriate command
+LYCHEE_MODE=${LYCHEE_MODE:-web}
+
+case "$LYCHEE_MODE" in
+    web)
+        echo "🌐 Starting Lychee in web mode..."
+        # Execute the main command (from Dockerfile CMD: octane:start)
+        exec "$@"
+        ;;
+    worker)
+        echo "⚙️  Starting Lychee in worker mode..."
+        echo "🔄 Auto-restart enabled: worker will restart if it exits"
+
+        # Get queue configuration from environment
+        QUEUE_NAMES=${QUEUE_NAMES:-default}
+        WORKER_MAX_TIME=${WORKER_MAX_TIME:-3600}
+        QUEUE_CONNECTION=${QUEUE_CONNECTION:-sync}
+
+        echo "📋 Queue names: $QUEUE_NAMES"
+        echo "⏱️  Max time: ${WORKER_MAX_TIME}s"
+        echo "📡 Queue connection: $QUEUE_CONNECTION"
+
+        # Warn if using sync driver (not recommended for worker mode)
+        if [ "$QUEUE_CONNECTION" = "sync" ]; then
+            echo "⚠️  WARNING: QUEUE_CONNECTION=sync is not recommended for worker mode."
+            echo "   Jobs will run synchronously, defeating the purpose of a queue worker."
+            echo "   Consider using 'redis' or 'database' for persistent asynchronous queues."
+        fi
+
+        # Track if we should keep running
+        KEEP_RUNNING=true
+
+        # Handle graceful shutdown
+        trap 'echo "🛑 Received shutdown signal, stopping..."; KEEP_RUNNING=false' TERM INT        # Auto-restart loop: if queue:work exits, restart it
+
+        # This handles memory leak mitigation (max-time) and crash recovery
+        while $KEEP_RUNNING; do
+            echo "🚀 Starting queue worker ($(date '+%Y-%m-%d %H:%M:%S'))"
+
+            # Default exit code to 0
+            EXIT_CODE=0
+
+            # Run queue worker with standard options
+            # --tries=3: retry failed jobs up to 3 times
+            # --timeout=3600: kill job if it runs longer than 1 hour
+            # --sleep=3: sleep 3 seconds when queue is empty
+            # --max-time=$WORKER_MAX_TIME: restart worker after N seconds (memory leak mitigation)
+            php artisan queue:work \
+                --queue="$QUEUE_NAMES" \
+                --tries=3 \
+                --timeout=3600 \
+                --sleep=3 \
+                --max-time="$WORKER_MAX_TIME" || EXIT_CODE=$?
+
+            if [ $EXIT_CODE -eq 0 ]; then
+                echo "✅ Queue worker exited cleanly (exit code 0)"
+            else
+                echo "⚠️  Queue worker exited with code $EXIT_CODE"
+            fi
+
+            # Exit if we received shutdown signal
+            if ! $KEEP_RUNNING; then
+                echo "👋 Shutting down worker..."
+                exit $EXIT_CODE
+            fi
+
+            echo "⏳ Waiting 5 seconds before restart..."
+            sleep 5
+        done
+        ;;
+    *)
+        echo "❌ ERROR: Invalid LYCHEE_MODE: $LYCHEE_MODE. Must be 'web' or 'worker'."
+        exit 1
+        ;;
+esac