Reapply "Merge pull request #686 from AnishSarkar22/feat/replace-logs"

This reverts commit 3418c0e.

Author: MODSetter

.env.example

@@ -9,7 +9,6 @@ FRONTEND_PORT=3000
 NEXT_PUBLIC_FASTAPI_BACKEND_URL=http://localhost:8000 (Default: http://localhost:8000)
 NEXT_PUBLIC_FASTAPI_BACKEND_AUTH_TYPE=LOCAL or GOOGLE (Default: LOCAL)
 NEXT_PUBLIC_ETL_SERVICE=UNSTRUCTURED or LLAMACLOUD or DOCLING (Default: DOCLING)
-
 # Backend Configuration
 BACKEND_PORT=8000
 
@@ -19,6 +18,17 @@ POSTGRES_PASSWORD=postgres
 POSTGRES_DB=surfsense
 POSTGRES_PORT=5432
 
+# Electric-SQL Configuration
+ELECTRIC_PORT=5133
+# PostgreSQL host for Electric connection
+# - 'db' for Docker PostgreSQL (service name in docker-compose)
+# - 'host.docker.internal' for local PostgreSQL (recommended when Electric runs in Docker)
+#   Note: host.docker.internal works on Docker Desktop (Mac/Windows) and can be enabled on Linux
+POSTGRES_HOST=db
+ELECTRIC_DB_USER=electric
+ELECTRIC_DB_PASSWORD=electric_password
+NEXT_PUBLIC_ELECTRIC_URL=http://localhost:5133
+
 # pgAdmin Configuration
 PGADMIN_PORT=5050
 PGADMIN_DEFAULT_EMAIL=admin@surfsense.com

Dockerfile.allinone

@@ -1,10 +1,11 @@
 # SurfSense All-in-One Docker Image
-# This image bundles PostgreSQL+pgvector, Redis, Backend, and Frontend
-# Usage: docker run -d -p 3000:3000 -p 8000:8000 -v surfsense-data:/data --name surfsense ghcr.io/modsetter/surfsense:latest
+# This image bundles PostgreSQL+pgvector, Redis, Electric SQL, Backend, and Frontend
+# Usage: docker run -d -p 3000:3000 -p 8000:8000 -p 5133:5133 -v surfsense-data:/data --name surfsense ghcr.io/modsetter/surfsense:latest
 #
 # Included Services (all run locally by default):
 # - PostgreSQL 14 + pgvector (vector database)
 # - Redis (task queue)
+# - Electric SQL (real-time sync)
 # - Docling (document processing, CPU-only, OCR disabled)
 # - Kokoro TTS (local text-to-speech for podcasts)
 # - Faster-Whisper (local speech-to-text for audio files)
@@ -14,7 +15,12 @@
 # will be available in the future for faster AI inference.
 
 # ====================
-# Stage 1: Build Frontend
+# Stage 1: Get Electric SQL Binary
+# ====================
+FROM electricsql/electric:latest AS electric-builder
+
+# ====================
+# Stage 2: Build Frontend
 # ====================
 FROM node:20-alpine AS frontend-builder
 
@@ -42,12 +48,14 @@ RUN pnpm fumadocs-mdx
 ENV NEXT_PUBLIC_FASTAPI_BACKEND_URL=__NEXT_PUBLIC_FASTAPI_BACKEND_URL__
 ENV NEXT_PUBLIC_FASTAPI_BACKEND_AUTH_TYPE=__NEXT_PUBLIC_FASTAPI_BACKEND_AUTH_TYPE__
 ENV NEXT_PUBLIC_ETL_SERVICE=__NEXT_PUBLIC_ETL_SERVICE__
+ENV NEXT_PUBLIC_ELECTRIC_URL=__NEXT_PUBLIC_ELECTRIC_URL__
+ENV NEXT_PUBLIC_ELECTRIC_AUTH_MODE=__NEXT_PUBLIC_ELECTRIC_AUTH_MODE__
 
 # Build
 RUN pnpm run build
 
 # ====================
-# Stage 2: Runtime Image
+# Stage 3: Runtime Image
 # ====================
 FROM ubuntu:22.04 AS runtime
 
@@ -167,6 +175,11 @@ COPY --from=frontend-builder /app/public ./public
 
 COPY surfsense_web/content/docs /app/surfsense_web/content/docs
 
+# ====================
+# Copy Electric SQL Release
+# ====================
+COPY --from=electric-builder /app /app/electric-release
+
 # ====================
 # Setup Backend
 # ====================
@@ -238,11 +251,22 @@ ENV NEXT_PUBLIC_FASTAPI_BACKEND_URL=http://localhost:8000
 ENV NEXT_PUBLIC_FASTAPI_BACKEND_AUTH_TYPE=LOCAL
 ENV NEXT_PUBLIC_ETL_SERVICE=DOCLING
 
+# Electric SQL configuration (ELECTRIC_DATABASE_URL is built dynamically by entrypoint from these values)
+ENV ELECTRIC_DB_USER=electric
+ENV ELECTRIC_DB_PASSWORD=electric_password
+# Note: ELECTRIC_DATABASE_URL is NOT set here - entrypoint builds it dynamically from ELECTRIC_DB_USER/PASSWORD
+ENV ELECTRIC_INSECURE=true
+ENV ELECTRIC_WRITE_TO_PG_MODE=direct
+ENV ELECTRIC_PORT=5133
+ENV PORT=5133
+ENV NEXT_PUBLIC_ELECTRIC_URL=http://localhost:5133
+ENV NEXT_PUBLIC_ELECTRIC_AUTH_MODE=insecure
+
 # Data volume
 VOLUME ["/data"]
 
-# Expose ports
-EXPOSE 3000 8000
+# Expose ports (Frontend: 3000, Backend: 8000, Electric: 5133)
+EXPOSE 3000 8000 5133
 
 # Health check
 HEALTHCHECK --interval=30s --timeout=10s --start-period=120s --retries=3 \

docker-compose.yml

@@ -7,10 +7,15 @@ services:
       - "${POSTGRES_PORT:-5432}:5432"
     volumes:
       - postgres_data:/var/lib/postgresql/data
+      - ./scripts/docker/postgresql.conf:/etc/postgresql/postgresql.conf:ro
+      - ./scripts/docker/init-electric-user.sh:/docker-entrypoint-initdb.d/init-electric-user.sh:ro
     environment:
       - POSTGRES_USER=${POSTGRES_USER:-postgres}
       - POSTGRES_PASSWORD=${POSTGRES_PASSWORD:-postgres}
       - POSTGRES_DB=${POSTGRES_DB:-surfsense}
+      - ELECTRIC_DB_USER=${ELECTRIC_DB_USER:-electric}
+      - ELECTRIC_DB_PASSWORD=${ELECTRIC_DB_PASSWORD:-electric_password}
+    command: postgres -c config_file=/etc/postgresql/postgresql.conf
 
   pgadmin:
     image: dpage/pgadmin4
@@ -51,11 +56,14 @@ services:
       - UNSTRUCTURED_HAS_PATCHED_LOOP=1
       - LANGCHAIN_TRACING_V2=false
       - LANGSMITH_TRACING=false
+      - ELECTRIC_DB_USER=${ELECTRIC_DB_USER:-electric}
+      - ELECTRIC_DB_PASSWORD=${ELECTRIC_DB_PASSWORD:-electric_password}
+      - NEXT_FRONTEND_URL=http://frontend:3000
     depends_on:
       - db
       - redis
 
-  # Run these services seperately in production
+  # Run these services separately in production
   # celery_worker:
   #   build: ./surfsense_backend
   #   # image: ghcr.io/modsetter/surfsense_backend:latest
@@ -110,6 +118,23 @@ services:
   #     - redis
   #     - celery_worker
 
+  electric:
+    image: electricsql/electric:latest
+    ports:
+      - "${ELECTRIC_PORT:-5133}:3000"
+    environment:
+      - DATABASE_URL=${ELECTRIC_DATABASE_URL:-postgresql://${ELECTRIC_DB_USER:-electric}:${ELECTRIC_DB_PASSWORD:-electric_password}@${POSTGRES_HOST:-db}:${POSTGRES_PORT:-5432}/${POSTGRES_DB:-surfsense}?sslmode=disable}
+      - ELECTRIC_INSECURE=true
+      - ELECTRIC_WRITE_TO_PG_MODE=direct
+    restart: unless-stopped
+    # depends_on:
+    #   - db
+    healthcheck:
+      test: ["CMD", "curl", "-f", "http://localhost:3000/v1/health"]
+      interval: 10s
+      timeout: 5s
+      retries: 5
+
   frontend:
     build:
       context: ./surfsense_web
@@ -122,8 +147,12 @@ services:
       - "${FRONTEND_PORT:-3000}:3000"
     env_file:
       - ./surfsense_web/.env
+    environment:
+      - NEXT_PUBLIC_ELECTRIC_URL=${NEXT_PUBLIC_ELECTRIC_URL:-http://localhost:5133}
+      - NEXT_PUBLIC_ELECTRIC_AUTH_MODE=insecure
     depends_on:
       - backend
+      - electric
 
 volumes:
   postgres_data:

scripts/docker/entrypoint-allinone.sh

@@ -42,6 +42,31 @@ if [ -z "$STT_SERVICE" ]; then
     echo "✅ Using default STT_SERVICE: local/base"
 fi
 
+# ================================================
+# Set Electric SQL configuration
+# ================================================
+export ELECTRIC_DB_USER="${ELECTRIC_DB_USER:-electric}"
+export ELECTRIC_DB_PASSWORD="${ELECTRIC_DB_PASSWORD:-electric_password}"
+if [ -z "$ELECTRIC_DATABASE_URL" ]; then
+    export ELECTRIC_DATABASE_URL="postgresql://${ELECTRIC_DB_USER}:${ELECTRIC_DB_PASSWORD}@localhost:5432/${POSTGRES_DB:-surfsense}?sslmode=disable"
+    echo "✅ Electric SQL URL configured dynamically"
+else
+    # Ensure sslmode=disable is in the URL if not already present
+    if [[ "$ELECTRIC_DATABASE_URL" != *"sslmode="* ]]; then
+        # Add sslmode=disable (handle both cases: with or without existing query params)
+        if [[ "$ELECTRIC_DATABASE_URL" == *"?"* ]]; then
+            export ELECTRIC_DATABASE_URL="${ELECTRIC_DATABASE_URL}&sslmode=disable"
+        else
+            export ELECTRIC_DATABASE_URL="${ELECTRIC_DATABASE_URL}?sslmode=disable"
+        fi
+    fi
+    echo "✅ Electric SQL URL configured from environment"
+fi
+
+# Set Electric SQL port
+export ELECTRIC_PORT="${ELECTRIC_PORT:-5133}"
+export PORT="${ELECTRIC_PORT}"
+
 # ================================================
 # Initialize PostgreSQL if needed
 # ================================================
@@ -60,6 +85,11 @@ if [ ! -f /data/postgres/PG_VERSION ]; then
     echo "local all all trust" >> /data/postgres/pg_hba.conf
     echo "listen_addresses='*'" >> /data/postgres/postgresql.conf
 
+    # Enable logical replication for Electric SQL
+    echo "wal_level = logical" >> /data/postgres/postgresql.conf
+    echo "max_replication_slots = 10" >> /data/postgres/postgresql.conf
+    echo "max_wal_senders = 10" >> /data/postgres/postgresql.conf
+    
     # Start PostgreSQL temporarily to create database and user
     su - postgres -c "/usr/lib/postgresql/14/bin/pg_ctl -D /data/postgres -l /tmp/postgres_init.log start"
 
@@ -73,6 +103,35 @@ if [ ! -f /data/postgres/PG_VERSION ]; then
     # Enable pgvector extension
     su - postgres -c "psql -d ${POSTGRES_DB:-surfsense} -c 'CREATE EXTENSION IF NOT EXISTS vector;'"
 
+    # Create Electric SQL replication user (idempotent - uses IF NOT EXISTS)
+    echo "📡 Creating Electric SQL replication user..."
+    su - postgres -c "psql -d ${POSTGRES_DB:-surfsense} <<-EOSQL
+        DO \\\$\\\$
+        BEGIN
+            IF NOT EXISTS (SELECT FROM pg_user WHERE usename = '${ELECTRIC_DB_USER}') THEN
+                CREATE USER ${ELECTRIC_DB_USER} WITH REPLICATION PASSWORD '${ELECTRIC_DB_PASSWORD}';
+            END IF;
+        END
+        \\\$\\\$;
+
+        GRANT CONNECT ON DATABASE ${POSTGRES_DB:-surfsense} TO ${ELECTRIC_DB_USER};
+        GRANT USAGE ON SCHEMA public TO ${ELECTRIC_DB_USER};
+        GRANT SELECT ON ALL TABLES IN SCHEMA public TO ${ELECTRIC_DB_USER};
+        GRANT SELECT ON ALL SEQUENCES IN SCHEMA public TO ${ELECTRIC_DB_USER};
+        ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT SELECT ON TABLES TO ${ELECTRIC_DB_USER};
+        ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT SELECT ON SEQUENCES TO ${ELECTRIC_DB_USER};
+
+        -- Create the publication for Electric SQL (if not exists)
+        DO \\\$\\\$
+        BEGIN
+            IF NOT EXISTS (SELECT FROM pg_publication WHERE pubname = 'electric_publication_default') THEN
+                CREATE PUBLICATION electric_publication_default;
+            END IF;
+        END
+        \\\$\\\$;
+EOSQL"
+    echo "✅ Electric SQL user '${ELECTRIC_DB_USER}' created"
+    
     # Stop temporary PostgreSQL
     su - postgres -c "/usr/lib/postgresql/14/bin/pg_ctl -D /data/postgres stop"
 
@@ -107,18 +166,23 @@ echo "🔧 Applying runtime environment configuration..."
 NEXT_PUBLIC_FASTAPI_BACKEND_URL="${NEXT_PUBLIC_FASTAPI_BACKEND_URL:-http://localhost:8000}"
 NEXT_PUBLIC_FASTAPI_BACKEND_AUTH_TYPE="${NEXT_PUBLIC_FASTAPI_BACKEND_AUTH_TYPE:-LOCAL}"
 NEXT_PUBLIC_ETL_SERVICE="${NEXT_PUBLIC_ETL_SERVICE:-DOCLING}"
+NEXT_PUBLIC_ELECTRIC_URL="${NEXT_PUBLIC_ELECTRIC_URL:-http://localhost:5133}"
+NEXT_PUBLIC_ELECTRIC_AUTH_MODE="${NEXT_PUBLIC_ELECTRIC_AUTH_MODE:-insecure}"
 
 # Replace placeholders in all JS files
 find /app/frontend -type f \( -name "*.js" -o -name "*.json" \) -exec sed -i \
     -e "s|__NEXT_PUBLIC_FASTAPI_BACKEND_URL__|${NEXT_PUBLIC_FASTAPI_BACKEND_URL}|g" \
     -e "s|__NEXT_PUBLIC_FASTAPI_BACKEND_AUTH_TYPE__|${NEXT_PUBLIC_FASTAPI_BACKEND_AUTH_TYPE}|g" \
     -e "s|__NEXT_PUBLIC_ETL_SERVICE__|${NEXT_PUBLIC_ETL_SERVICE}|g" \
+    -e "s|__NEXT_PUBLIC_ELECTRIC_URL__|${NEXT_PUBLIC_ELECTRIC_URL}|g" \
+    -e "s|__NEXT_PUBLIC_ELECTRIC_AUTH_MODE__|${NEXT_PUBLIC_ELECTRIC_AUTH_MODE}|g" \
     {} +
 
 echo "✅ Environment configuration applied"
-echo "   Backend URL:  ${NEXT_PUBLIC_FASTAPI_BACKEND_URL}"
-echo "   Auth Type:    ${NEXT_PUBLIC_FASTAPI_BACKEND_AUTH_TYPE}"
-echo "   ETL Service:  ${NEXT_PUBLIC_ETL_SERVICE}"
+echo "   Backend URL:   ${NEXT_PUBLIC_FASTAPI_BACKEND_URL}"
+echo "   Auth Type:     ${NEXT_PUBLIC_FASTAPI_BACKEND_AUTH_TYPE}"
+echo "   ETL Service:   ${NEXT_PUBLIC_ETL_SERVICE}"
+echo "   Electric URL:  ${NEXT_PUBLIC_ELECTRIC_URL}"
 
 # ================================================
 # Run database migrations
@@ -161,6 +225,7 @@ echo "==========================================="
 echo "  Frontend URL:    http://localhost:3000"
 echo "  Backend API:     ${NEXT_PUBLIC_FASTAPI_BACKEND_URL}"
 echo "  API Docs:        ${NEXT_PUBLIC_FASTAPI_BACKEND_URL}/docs"
+echo "  Electric URL:    ${NEXT_PUBLIC_ELECTRIC_URL:-http://localhost:5133}"
 echo "  Auth Type:       ${NEXT_PUBLIC_FASTAPI_BACKEND_AUTH_TYPE}"
 echo "  ETL Service:     ${NEXT_PUBLIC_ETL_SERVICE}"
 echo "  TTS Service:     ${TTS_SERVICE}"

scripts/docker/init-electric-user.sh

@@ -0,0 +1,56 @@
+#!/bin/sh
+# ============================================================================
+# Electric SQL User Initialization Script (docker-compose only)
+# ============================================================================
+# This script is ONLY used when running via docker-compose.
+#
+# How it works:
+# - docker-compose.yml mounts this script into the PostgreSQL container's
+#   /docker-entrypoint-initdb.d/ directory
+# - PostgreSQL automatically executes scripts in that directory on first
+#   container initialization
+#
+# For local PostgreSQL users (non-Docker), this script is NOT used.
+# Instead, the Electric user is created by Alembic migration 66
+# (66_add_notifications_table_and_electric_replication.py).
+#
+# Both approaches are idempotent (use IF NOT EXISTS), so running both
+# will not cause conflicts.
+# ============================================================================
+
+set -e
+
+# Use environment variables with defaults
+ELECTRIC_DB_USER="${ELECTRIC_DB_USER:-electric}"
+ELECTRIC_DB_PASSWORD="${ELECTRIC_DB_PASSWORD:-electric_password}"
+
+echo "Creating Electric SQL replication user: $ELECTRIC_DB_USER"
+
+psql -v ON_ERROR_STOP=1 --username "$POSTGRES_USER" --dbname "$POSTGRES_DB" <<-EOSQL
+    DO \$\$
+    BEGIN
+        IF NOT EXISTS (SELECT FROM pg_user WHERE usename = '$ELECTRIC_DB_USER') THEN
+            CREATE USER $ELECTRIC_DB_USER WITH REPLICATION PASSWORD '$ELECTRIC_DB_PASSWORD';
+        END IF;
+    END
+    \$\$;
+
+    GRANT CONNECT ON DATABASE $POSTGRES_DB TO $ELECTRIC_DB_USER;
+    GRANT CREATE ON DATABASE $POSTGRES_DB TO $ELECTRIC_DB_USER;
+    GRANT USAGE ON SCHEMA public TO $ELECTRIC_DB_USER;
+    GRANT SELECT ON ALL TABLES IN SCHEMA public TO $ELECTRIC_DB_USER;
+    GRANT SELECT ON ALL SEQUENCES IN SCHEMA public TO $ELECTRIC_DB_USER;
+    ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT SELECT ON TABLES TO $ELECTRIC_DB_USER;
+    ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT SELECT ON SEQUENCES TO $ELECTRIC_DB_USER;
+
+    -- Create the publication for Electric SQL (if not exists)
+    DO \$\$
+    BEGIN
+        IF NOT EXISTS (SELECT FROM pg_publication WHERE pubname = 'electric_publication_default') THEN
+            CREATE PUBLICATION electric_publication_default;
+        END IF;
+    END
+    \$\$;
+EOSQL
+
+echo "Electric SQL user '$ELECTRIC_DB_USER' and publication created successfully"

scripts/docker/init-postgres.sh

@@ -9,6 +9,10 @@ POSTGRES_USER=${POSTGRES_USER:-surfsense}
 POSTGRES_PASSWORD=${POSTGRES_PASSWORD:-surfsense}
 POSTGRES_DB=${POSTGRES_DB:-surfsense}
 
+# Electric SQL user credentials (configurable)
+ELECTRIC_DB_USER=${ELECTRIC_DB_USER:-electric}
+ELECTRIC_DB_PASSWORD=${ELECTRIC_DB_PASSWORD:-electric_password}
+
 echo "Initializing PostgreSQL..."
 
 # Check if PostgreSQL is already initialized
@@ -23,8 +27,18 @@ fi
 # Configure PostgreSQL
 cat >> "$PGDATA/postgresql.conf" << EOF
 listen_addresses = '*'
-max_connections = 100
-shared_buffers = 128MB
+max_connections = 200
+shared_buffers = 256MB
+
+# Enable logical replication (required for Electric SQL)
+wal_level = logical
+max_replication_slots = 10
+max_wal_senders = 10
+
+# Performance settings
+checkpoint_timeout = 10min
+max_wal_size = 1GB
+min_wal_size = 80MB
 EOF
 
 cat >> "$PGDATA/pg_hba.conf" << EOF
@@ -45,6 +59,15 @@ CREATE USER $POSTGRES_USER WITH PASSWORD '$POSTGRES_PASSWORD' SUPERUSER;
 CREATE DATABASE $POSTGRES_DB OWNER $POSTGRES_USER;
 \c $POSTGRES_DB
 CREATE EXTENSION IF NOT EXISTS vector;
+
+-- Create Electric SQL replication user
+CREATE USER $ELECTRIC_DB_USER WITH REPLICATION PASSWORD '$ELECTRIC_DB_PASSWORD';
+GRANT CONNECT ON DATABASE $POSTGRES_DB TO $ELECTRIC_DB_USER;
+GRANT USAGE ON SCHEMA public TO $ELECTRIC_DB_USER;
+GRANT SELECT ON ALL TABLES IN SCHEMA public TO $ELECTRIC_DB_USER;
+GRANT SELECT ON ALL SEQUENCES IN SCHEMA public TO $ELECTRIC_DB_USER;
+ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT SELECT ON TABLES TO $ELECTRIC_DB_USER;
+ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT SELECT ON SEQUENCES TO $ELECTRIC_DB_USER;
 EOF
 
 echo "PostgreSQL initialized successfully."