Merge remote-tracking branch 'upstream/main' into feature/blocknote-editor

Author: AnishSarkar22

README.md

@@ -38,19 +38,24 @@ https://github.com/user-attachments/assets/a0a16566-6967-4374-ac51-9b3e07fbecd7
 ## Key Features
 
 ### 💡 **Idea**: 
-Have your own highly customizable private NotebookLM and Perplexity integrated with external sources.
+- Have your own highly customizable private NotebookLM and Perplexity integrated with external sources.
 ### 📁 **Multiple File Format Uploading Support**
-Save content from your own personal files *(Documents, images, videos and supports **50+ file extensions**)* to your own personal knowledge base .
+- Save content from your own personal files *(Documents, images, videos and supports **50+ file extensions**)* to your own personal knowledge base .
 ### 🔍 **Powerful Search**
-Quickly research or find anything in your saved content .
+- Quickly research or find anything in your saved content .
 ### 💬 **Chat with your Saved Content**
- Interact in Natural Language and get cited answers.
+- Interact in Natural Language and get cited answers.
 ### 📄 **Cited Answers**
-Get Cited answers just like Perplexity.
+- Get Cited answers just like Perplexity.
 ### 🔔 **Privacy & Local LLM Support**
-Works Flawlessly with Ollama local LLMs.
+- Works Flawlessly with Ollama local LLMs.
 ### 🏠 **Self Hostable**
-Open source and easy to deploy locally.
+- Open source and easy to deploy locally.
+### 👥 **Team Collaboration with RBAC**
+- Role-Based Access Control for Search Spaces
+- Invite team members with customizable roles (Owner, Admin, Editor, Viewer)
+- Granular permissions for documents, chats, connectors, and settings
+- Share knowledge bases securely within your organization
 ### 🎙️ Podcasts 
 - Blazingly fast podcast generation agent. (Creates a 3-minute podcast in under 20 seconds.)
 - Convert your chat conversations into engaging audio content

README.zh-CN.md

@@ -39,25 +39,31 @@ https://github.com/user-attachments/assets/a0a16566-6967-4374-ac51-9b3e07fbecd7
 ## 核心功能
 
 ### 💡 **理念**: 
-拥有您自己的高度可定制的私有 NotebookLM 和 Perplexity，并与外部数据源集成。
+- 拥有您自己的高度可定制的私有 NotebookLM 和 Perplexity，并与外部数据源集成。
 
 ### 📁 **支持多种文件格式上传**
-将您个人文件中的内容（文档、图像、视频，支持 **50+ 种文件扩展名**）保存到您自己的个人知识库。
+- 将您个人文件中的内容（文档、图像、视频，支持 **50+ 种文件扩展名**）保存到您自己的个人知识库。
 
 ### 🔍 **强大的搜索功能**
-快速研究或查找已保存内容中的任何信息。
+- 快速研究或查找已保存内容中的任何信息。
 
 ### 💬 **与已保存内容对话**
-使用自然语言交互并获得引用答案。
+- 使用自然语言交互并获得引用答案。
 
 ### 📄 **引用答案**
-像 Perplexity 一样获得带引用的答案。
+- 像 Perplexity 一样获得带引用的答案。
 
 ### 🔔 **隐私保护与本地 LLM 支持**
-完美支持 Ollama 本地大语言模型。
+- 完美支持 Ollama 本地大语言模型。
 
 ### 🏠 **可自托管**
-开源且易于本地部署。
+- 开源且易于本地部署。
+
+### 👥 **团队协作与 RBAC**
+- 搜索空间的基于角色的访问控制
+- 使用可自定义的角色（所有者、管理员、编辑者、查看者）邀请团队成员
+- 对文档、聊天、连接器和设置的细粒度权限控制
+- 在组织内安全共享知识库
 
 ### 🎙️ **播客功能**
 - 超快速播客生成代理（在 20 秒内创建 3 分钟播客）

surfsense_backend/alembic/versions/36_remove_fk_constraints_for_global_llm_configs.py

@@ -8,6 +8,8 @@
 
 from collections.abc import Sequence
 
+from sqlalchemy import text
+
 from alembic import op
 
 # revision identifiers, used by Alembic.
@@ -17,57 +19,76 @@
 depends_on: str | Sequence[str] | None = None
 
 
+def constraint_exists(connection, table_name: str, constraint_name: str) -> bool:
+    """Check if a constraint exists on the given table."""
+    result = connection.execute(
+        text(
+            """
+            SELECT 1 FROM information_schema.table_constraints
+            WHERE table_name = :table_name AND constraint_name = :constraint_name
+            """
+        ),
+        {"table_name": table_name, "constraint_name": constraint_name},
+    )
+    return result.fetchone() is not None
+
+
 def upgrade() -> None:
     """
     Remove foreign key constraints on LLM preference columns to allow global configs (negative IDs).
 
     Global LLM configs use negative IDs and don't exist in the llm_configs table,
     so we need to remove the foreign key constraints that were preventing their use.
     """
-    # Drop the foreign key constraints
-    op.drop_constraint(
+    connection = op.get_bind()
+
+    # Drop the foreign key constraints if they exist
+    constraints_to_drop = [
         "user_search_space_preferences_long_context_llm_id_fkey",
-        "user_search_space_preferences",
-        type_="foreignkey",
-    )
-    op.drop_constraint(
         "user_search_space_preferences_fast_llm_id_fkey",
-        "user_search_space_preferences",
-        type_="foreignkey",
-    )
-    op.drop_constraint(
         "user_search_space_preferences_strategic_llm_id_fkey",
-        "user_search_space_preferences",
-        type_="foreignkey",
-    )
+    ]
+
+    for constraint_name in constraints_to_drop:
+        if constraint_exists(
+            connection, "user_search_space_preferences", constraint_name
+        ):
+            op.drop_constraint(
+                constraint_name,
+                "user_search_space_preferences",
+                type_="foreignkey",
+            )
+        else:
+            print(f"Constraint '{constraint_name}' does not exist. Skipping.")
 
 
 def downgrade() -> None:
     """
     Re-add foreign key constraints (will fail if any negative IDs exist in the table).
     """
-    # Re-add the foreign key constraints
-    op.create_foreign_key(
-        "user_search_space_preferences_long_context_llm_id_fkey",
-        "user_search_space_preferences",
-        "llm_configs",
-        ["long_context_llm_id"],
-        ["id"],
-        ondelete="SET NULL",
-    )
-    op.create_foreign_key(
-        "user_search_space_preferences_fast_llm_id_fkey",
-        "user_search_space_preferences",
-        "llm_configs",
-        ["fast_llm_id"],
-        ["id"],
-        ondelete="SET NULL",
-    )
-    op.create_foreign_key(
-        "user_search_space_preferences_strategic_llm_id_fkey",
-        "user_search_space_preferences",
-        "llm_configs",
-        ["strategic_llm_id"],
-        ["id"],
-        ondelete="SET NULL",
-    )
+    connection = op.get_bind()
+
+    # Re-add the foreign key constraints if they don't exist
+    constraints_to_create = [
+        (
+            "user_search_space_preferences_long_context_llm_id_fkey",
+            "long_context_llm_id",
+        ),
+        ("user_search_space_preferences_fast_llm_id_fkey", "fast_llm_id"),
+        ("user_search_space_preferences_strategic_llm_id_fkey", "strategic_llm_id"),
+    ]
+
+    for constraint_name, column_name in constraints_to_create:
+        if not constraint_exists(
+            connection, "user_search_space_preferences", constraint_name
+        ):
+            op.create_foreign_key(
+                constraint_name,
+                "user_search_space_preferences",
+                "llm_configs",
+                [column_name],
+                ["id"],
+                ondelete="SET NULL",
+            )
+        else:
+            print(f"Constraint '{constraint_name}' already exists. Skipping.")

surfsense_backend/alembic/versions/37_add_system_prompts_to_searchspaces.py

@@ -9,6 +9,7 @@
 from collections.abc import Sequence
 
 import sqlalchemy as sa
+from sqlalchemy import text
 
 from alembic import op
 
@@ -19,24 +20,55 @@
 depends_on: str | Sequence[str] | None = None
 
 
+def column_exists(connection, table_name: str, column_name: str) -> bool:
+    """Check if a column exists on the given table."""
+    result = connection.execute(
+        text(
+            """
+            SELECT 1 FROM information_schema.columns
+            WHERE table_name = :table_name AND column_name = :column_name
+            """
+        ),
+        {"table_name": table_name, "column_name": column_name},
+    )
+    return result.fetchone() is not None
+
+
 def upgrade() -> None:
     """Add QnA configuration columns to searchspaces table."""
+    connection = op.get_bind()
+
     # Add citations_enabled boolean (default True)
-    op.add_column(
-        "searchspaces",
-        sa.Column(
-            "citations_enabled", sa.Boolean(), nullable=False, server_default="true"
-        ),
-    )
+    if not column_exists(connection, "searchspaces", "citations_enabled"):
+        op.add_column(
+            "searchspaces",
+            sa.Column(
+                "citations_enabled", sa.Boolean(), nullable=False, server_default="true"
+            ),
+        )
+    else:
+        print("Column 'citations_enabled' already exists. Skipping.")
 
     # Add custom instructions text field (nullable, defaults to empty)
-    op.add_column(
-        "searchspaces",
-        sa.Column("qna_custom_instructions", sa.Text(), nullable=True),
-    )
+    if not column_exists(connection, "searchspaces", "qna_custom_instructions"):
+        op.add_column(
+            "searchspaces",
+            sa.Column("qna_custom_instructions", sa.Text(), nullable=True),
+        )
+    else:
+        print("Column 'qna_custom_instructions' already exists. Skipping.")
 
 
 def downgrade() -> None:
     """Remove QnA configuration columns from searchspaces table."""
-    op.drop_column("searchspaces", "qna_custom_instructions")
-    op.drop_column("searchspaces", "citations_enabled")
+    connection = op.get_bind()
+
+    if column_exists(connection, "searchspaces", "qna_custom_instructions"):
+        op.drop_column("searchspaces", "qna_custom_instructions")
+    else:
+        print("Column 'qna_custom_instructions' does not exist. Skipping.")
+
+    if column_exists(connection, "searchspaces", "citations_enabled"):
+        op.drop_column("searchspaces", "citations_enabled")
+    else:
+        print("Column 'citations_enabled' does not exist. Skipping.")

surfsense_backend/alembic/versions/38_add_webcrawler_connector_enum.py

@@ -0,0 +1,59 @@
+"""Add Webcrawler connector enums
+
+Revision ID: 38
+Revises: 37
+Create Date: 2025-11-17 17:00:00.000000
+
+"""
+
+from collections.abc import Sequence
+
+from alembic import op
+
+revision: str = "38"
+down_revision: str | None = "37"
+branch_labels: str | Sequence[str] | None = None
+depends_on: str | Sequence[str] | None = None
+
+
+def upgrade() -> None:
+    """Safely add 'WEBCRAWLER_CONNECTOR' to enum types if missing."""
+
+    # Add to searchsourceconnectortype enum
+    op.execute(
+        """
+    DO $$
+    BEGIN
+        IF NOT EXISTS (
+            SELECT 1 FROM pg_type t
+            JOIN pg_enum e ON t.oid = e.enumtypid
+            WHERE t.typname = 'searchsourceconnectortype' AND e.enumlabel = 'WEBCRAWLER_CONNECTOR'
+        ) THEN
+            ALTER TYPE searchsourceconnectortype ADD VALUE 'WEBCRAWLER_CONNECTOR';
+        END IF;
+    END
+    $$;
+    """
+    )
+
+    # Add to documenttype enum
+    op.execute(
+        """
+    DO $$
+    BEGIN
+        IF NOT EXISTS (
+            SELECT 1 FROM pg_type t
+            JOIN pg_enum e ON t.oid = e.enumtypid
+            WHERE t.typname = 'documenttype' AND e.enumlabel = 'CRAWLED_URL'
+        ) THEN
+            ALTER TYPE documenttype ADD VALUE 'CRAWLED_URL';
+        END IF;
+    END
+    $$;
+    """
+    )
+
+
+def downgrade() -> None:
+    """Remove 'WEBCRAWLER_CONNECTOR' from enum types."""
+    pass