feat: Introduce electric-sql & some fixes

docker-compose.yml

@@ -7,10 +7,13 @@ services:
       - "${POSTGRES_PORT:-5432}:5432"
     volumes:
       - postgres_data:/var/lib/postgresql/data
+      - ./scripts/docker/postgresql.conf:/etc/postgresql/postgresql.conf:ro
+      - ./scripts/docker/init-electric-user.sql:/docker-entrypoint-initdb.d/init-electric-user.sql:ro
     environment:
       - POSTGRES_USER=${POSTGRES_USER:-postgres}
       - POSTGRES_PASSWORD=${POSTGRES_PASSWORD:-postgres}
       - POSTGRES_DB=${POSTGRES_DB:-surfsense}
+    command: postgres -c config_file=/etc/postgresql/postgresql.conf
 
   pgadmin:
     image: dpage/pgadmin4
@@ -110,6 +113,23 @@ services:
   #     - redis
   #     - celery_worker
 
+  electric:
+    image: electricsql/electric:latest
+    ports:
+      - "${ELECTRIC_PORT:-5133}:3000"
+    environment:
+      - DATABASE_URL=postgresql://electric:electric_password@db:5432/${POSTGRES_DB:-surfsense}?sslmode=disable
+      - ELECTRIC_INSECURE=true
+      - ELECTRIC_WRITE_TO_PG_MODE=direct
+    depends_on:
+      - db
+    restart: unless-stopped
+    healthcheck:
+      test: ["CMD", "curl", "-f", "http://localhost:3000/v1/health"]
+      interval: 10s
+      timeout: 5s
+      retries: 5
+
   frontend:
     build:
       context: ./surfsense_web
@@ -122,8 +142,12 @@ services:
       - "${FRONTEND_PORT:-3000}:3000"
     env_file:
       - ./surfsense_web/.env
+    environment:
+      - NEXT_PUBLIC_ELECTRIC_URL=${NEXT_PUBLIC_ELECTRIC_URL:-http://localhost:5133}
+      - NEXT_PUBLIC_ELECTRIC_AUTH_MODE=insecure
     depends_on:
       - backend
+      - electric
 
 volumes:
   postgres_data:

scripts/docker/electrify-tables.sql

@@ -0,0 +1,11 @@
+-- Electrify tables for Electric SQL sync
+-- This tells Electric SQL which tables to sync
+-- Run this after running migrations
+
+-- Electrify notifications table
+ALTER TABLE notifications ENABLE ELECTRIC;
+
+-- You can electrify other tables as needed:
+-- ALTER TABLE documents ENABLE ELECTRIC;
+-- ALTER TABLE logs ENABLE ELECTRIC;
+

scripts/docker/init-electric-user.sql

@@ -0,0 +1,30 @@
+-- Create Electric SQL replication user
+-- This script is run during PostgreSQL initialization
+
+DO $$
+BEGIN
+    IF NOT EXISTS (SELECT FROM pg_user WHERE usename = 'electric') THEN
+        CREATE USER electric WITH REPLICATION PASSWORD 'electric_password';
+    END IF;
+END
+$$;
+
+-- Grant necessary permissions
+GRANT CONNECT ON DATABASE surfsense TO electric;
+GRANT USAGE ON SCHEMA public TO electric;
+GRANT SELECT ON ALL TABLES IN SCHEMA public TO electric;
+GRANT SELECT ON ALL SEQUENCES IN SCHEMA public TO electric;
+
+-- Grant permissions on future tables
+ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT SELECT ON TABLES TO electric;
+ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT SELECT ON SEQUENCES TO electric;
+
+-- Create the publication that Electric SQL expects
+-- Electric will add tables to this publication when shapes are subscribed
+DO $$
+BEGIN
+    IF NOT EXISTS (SELECT FROM pg_publication WHERE pubname = 'electric_publication_default') THEN
+        CREATE PUBLICATION electric_publication_default;
+    END IF;
+END
+$$;

scripts/docker/init-postgres.sh

@@ -23,8 +23,18 @@ fi
 # Configure PostgreSQL
 cat >> "$PGDATA/postgresql.conf" << EOF
 listen_addresses = '*'
-max_connections = 100
-shared_buffers = 128MB
+max_connections = 200
+shared_buffers = 256MB
+
+# Enable logical replication (required for Electric SQL)
+wal_level = logical
+max_replication_slots = 10
+max_wal_senders = 10
+
+# Performance settings
+checkpoint_timeout = 10min
+max_wal_size = 1GB
+min_wal_size = 80MB
 EOF
 
 cat >> "$PGDATA/pg_hba.conf" << EOF
@@ -45,6 +55,15 @@ CREATE USER $POSTGRES_USER WITH PASSWORD '$POSTGRES_PASSWORD' SUPERUSER;
 CREATE DATABASE $POSTGRES_DB OWNER $POSTGRES_USER;
 \c $POSTGRES_DB
 CREATE EXTENSION IF NOT EXISTS vector;
+
+-- Create Electric SQL replication user
+CREATE USER electric WITH REPLICATION PASSWORD 'electric_password';
+GRANT CONNECT ON DATABASE $POSTGRES_DB TO electric;
+GRANT USAGE ON SCHEMA public TO electric;
+GRANT SELECT ON ALL TABLES IN SCHEMA public TO electric;
+GRANT SELECT ON ALL SEQUENCES IN SCHEMA public TO electric;
+ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT SELECT ON TABLES TO electric;
+ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT SELECT ON SEQUENCES TO electric;
 EOF
 
 echo "PostgreSQL initialized successfully."

scripts/docker/postgresql.conf

@@ -0,0 +1,20 @@
+# PostgreSQL configuration for Electric SQL
+# This file is mounted into the PostgreSQL container
+
+listen_addresses = '*'
+max_connections = 200
+shared_buffers = 256MB
+
+# Enable logical replication (required for Electric SQL)
+wal_level = logical
+max_replication_slots = 10
+max_wal_senders = 10
+
+# Performance settings
+checkpoint_timeout = 10min
+max_wal_size = 1GB
+min_wal_size = 80MB
+
+# Logging (optional, for debugging)
+# log_statement = 'all'
+# log_replication_commands = on

surfsense_backend/alembic/versions/60_add_notifications_table.py

@@ -0,0 +1,57 @@
+"""Add notifications table
+
+Revision ID: 60
+Revises: 59
+"""
+from collections.abc import Sequence
+
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision: str = "60"
+down_revision: str | None = "59"
+branch_labels: str | Sequence[str] | None = None
+depends_on: str | Sequence[str] | None = None
+
+
+def upgrade() -> None:
+    """Upgrade schema - add notifications table."""
+    # Create notifications table
+    op.execute(
+        """
+        CREATE TABLE IF NOT EXISTS notifications (
+            id SERIAL PRIMARY KEY,
+            user_id UUID NOT NULL REFERENCES "user"(id) ON DELETE CASCADE,
+            search_space_id INTEGER REFERENCES searchspaces(id) ON DELETE CASCADE,
+            type VARCHAR(50) NOT NULL,
+            title VARCHAR(200) NOT NULL,
+            message TEXT NOT NULL,
+            read BOOLEAN NOT NULL DEFAULT FALSE,
+            metadata JSONB DEFAULT '{}',
+            created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
+            updated_at TIMESTAMPTZ
+        );
+    """
+    )
+
+    # Create indexes
+    op.create_index("ix_notifications_user_id", "notifications", ["user_id"])
+    op.create_index("ix_notifications_read", "notifications", ["read"])
+    op.create_index("ix_notifications_created_at", "notifications", ["created_at"])
+    op.create_index("ix_notifications_user_read", "notifications", ["user_id", "read"])
+
+    # Set REPLICA IDENTITY FULL (required by Electric SQL for replication)
+    # This allows Electric SQL to track all column values for updates/deletes
+    op.execute("ALTER TABLE notifications REPLICA IDENTITY FULL;")
+
+    # Note: ElectricSQL 1.x dynamically adds tables to the publication when
+    # clients subscribe to shapes. No need to manually create publications.
+
+
+def downgrade() -> None:
+    """Downgrade schema - remove notifications table."""
+    op.drop_index("ix_notifications_user_read", table_name="notifications")
+    op.drop_index("ix_notifications_created_at", table_name="notifications")
+    op.drop_index("ix_notifications_read", table_name="notifications")
+    op.drop_index("ix_notifications_user_id", table_name="notifications")
+    op.drop_table("notifications")

surfsense_backend/app/db.py

@@ -492,6 +492,12 @@ class SearchSpace(BaseModel, TimestampMixin):
         order_by="Log.id",
         cascade="all, delete-orphan",
     )
+    notifications = relationship(
+        "Notification",
+        back_populates="search_space",
+        order_by="Notification.created_at.desc()",
+        cascade="all, delete-orphan",
+    )
     search_source_connectors = relationship(
         "SearchSourceConnector",
         back_populates="search_space",
@@ -629,6 +635,25 @@ class Log(BaseModel, TimestampMixin):
     search_space = relationship("SearchSpace", back_populates="logs")
 
 
+class Notification(BaseModel, TimestampMixin):
+    __tablename__ = "notifications"
+
+    user_id = Column(
+        UUID(as_uuid=True), ForeignKey("user.id", ondelete="CASCADE"), nullable=False, index=True
+    )
+    search_space_id = Column(
+        Integer, ForeignKey("searchspaces.id", ondelete="CASCADE"), nullable=True
+    )
+    type = Column(String(50), nullable=False)  # 'document_processed', 'connector_indexed', 'user_mentioned', etc.
+    title = Column(String(200), nullable=False)
+    message = Column(Text, nullable=False)
+    read = Column(Boolean, nullable=False, default=False, server_default=text("false"), index=True)
+    notification_metadata = Column("metadata", JSONB, nullable=True, default={})
+
+    user = relationship("User", back_populates="notifications")
+    search_space = relationship("SearchSpace", back_populates="notifications")
+
+
 class SearchSpaceRole(BaseModel, TimestampMixin):
     """
     Custom roles that can be defined per search space.
@@ -773,6 +798,12 @@ class User(SQLAlchemyBaseUserTableUUID, Base):
             "OAuthAccount", lazy="joined"
         )
         search_spaces = relationship("SearchSpace", back_populates="user")
+        notifications = relationship(
+            "Notification",
+            back_populates="user",
+            order_by="Notification.created_at.desc()",
+            cascade="all, delete-orphan",
+        )
 
         # RBAC relationships
         search_space_memberships = relationship(
@@ -799,6 +830,12 @@ class User(SQLAlchemyBaseUserTableUUID, Base):
 
     class User(SQLAlchemyBaseUserTableUUID, Base):
         search_spaces = relationship("SearchSpace", back_populates="user")
+        notifications = relationship(
+            "Notification",
+            back_populates="user",
+            order_by="Notification.created_at.desc()",
+            cascade="all, delete-orphan",
+        )
 
         # RBAC relationships
         search_space_memberships = relationship(