feat: add terminology management

Author: ulleo

backend/alembic/env.py

@@ -24,7 +24,8 @@
 
 # from apps.system.models.user import SQLModel  # noqa
 # from apps.settings.models.setting_models import SQLModel
-from apps.chat.models.chat_model import SQLModel
+# from apps.chat.models.chat_model import SQLModel
+from apps.terminology.models.terminology_model import SQLModel
 # from apps.dashboard.models.dashboard_model import SQLModel
 from common.core.config import settings # noqa
 

backend/alembic/versions/039_create_terminology.py

@@ -0,0 +1,41 @@
+"""039_create_terminology
+
+Revision ID: 25cbc85766fd
+Revises: fc23c4f3e755
+Create Date: 2025-08-25 11:38:32.990973
+
+"""
+from alembic import op
+import sqlalchemy as sa
+import sqlmodel.sql.sqltypes
+import pgvector
+from sqlalchemy.dialects import postgresql
+
+# revision identifiers, used by Alembic.
+revision = '25cbc85766fd'
+down_revision = 'fc23c4f3e755'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.execute("CREATE EXTENSION IF NOT EXISTS vector;")
+
+    op.create_table('terminology',
+    sa.Column('id', sa.BigInteger(), sa.Identity(always=True), nullable=False),
+    sa.Column('pid', sa.BigInteger(), nullable=True),
+    sa.Column('create_time', sa.DateTime(), nullable=True),
+    sa.Column('word', sqlmodel.sql.sqltypes.AutoString(length=255), nullable=True),
+    sa.Column('description', sa.Text(), nullable=True),
+    sa.Column('embedding', pgvector.sqlalchemy.vector.VECTOR(), nullable=True),
+    sa.PrimaryKeyConstraint('id')
+    )
+
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_table('terminology')
+    # ### end Alembic commands ###

backend/apps/ai_model/embedding.py

@@ -0,0 +1,59 @@
+import threading
+from typing import Optional
+
+from langchain_core.embeddings import Embeddings
+from langchain_huggingface import HuggingFaceEmbeddings
+from pydantic import BaseModel
+
+from common.core.config import settings
+
+
+class EmbeddingModelInfo(BaseModel):
+    folder: str
+    name: str
+    device: str = 'cpu'
+
+
+local_embedding_model = EmbeddingModelInfo(folder=settings.LOCAL_MODEL_PATH, name=settings.DEFAULT_EMBEDDING_MODEL)
+
+_lock = threading.Lock()
+locks = {}
+
+_embedding_model: dict[str, Optional[Embeddings]] = {}
+
+
+class EmbeddingModelCache:
+
+    @staticmethod
+    def _new_instance(config: EmbeddingModelInfo = local_embedding_model):
+        return HuggingFaceEmbeddings(model_name=config.name, cache_folder=config.folder,
+                                     model_kwargs={'device': config.device},
+                                     encode_kwargs={'normalize_embeddings': True}
+                                     )
+
+    @staticmethod
+    def _get_lock(key: str = settings.DEFAULT_EMBEDDING_MODEL):
+        lock = locks.get(key)
+        if lock is None:
+            with _lock:
+                lock = locks.get(key)
+                if lock is None:
+                    lock = threading.Lock()
+                    locks[key] = lock
+
+        return lock
+
+    @staticmethod
+    def get_model(key: str = settings.DEFAULT_EMBEDDING_MODEL,
+                  config: EmbeddingModelInfo = local_embedding_model) -> Embeddings:
+        global _embedding_model
+        model_instance = _embedding_model.get(key)
+        if model_instance is None:
+            lock = EmbeddingModelCache._get_lock(key)
+            with lock:
+                model_instance = _embedding_model.get(key)
+                if model_instance is None:
+                    model_instance = EmbeddingModelCache._new_instance(config)
+                    _embedding_model[key] = model_instance
+
+        return model_instance

backend/apps/api.py

@@ -1,9 +1,9 @@
 from fastapi import APIRouter
 
+from apps.terminology.api import terminology
 from apps.chat.api import chat
 from apps.dashboard.api import dashboard_api
 from apps.datasource.api import datasource
-from apps.settings.api import terminology
 from apps.system.api import login, user, aimodel, workspace, assistant
 from apps.mcp import mcp
 
@@ -19,3 +19,4 @@
 api_router.include_router(dashboard_api.router)
 api_router.include_router(mcp.router)
 
+

backend/apps/terminology/__init__.py

@@ -0,0 +1,37 @@
+from typing import Optional
+
+from fastapi import APIRouter, Query
+
+from apps.terminology.curd.terminology import page_terminology, create_terminology, update_terminology, \
+    delete_terminology
+from apps.terminology.models.terminology_model import TerminologyInfo
+from common.core.deps import SessionDep
+
+router = APIRouter(tags=["Terminology"], prefix="/system/terminology")
+
+
+@router.get("/page/{current_page}/{page_size}")
+async def pager(session: SessionDep, current_page: int, page_size: int,
+                word: Optional[str] = Query(None, description="搜索术语(可选)")):
+    current_page, page_size, total_count, total_pages, _list = page_terminology(session, current_page, page_size, word)
+
+    return {
+        "current_page": current_page,
+        "page_size": page_size,
+        "total_count": total_count,
+        "total_pages": total_pages,
+        "data": _list
+    }
+
+
+@router.put("")
+async def create_or_update(session: SessionDep, info: TerminologyInfo):
+    if info.id:
+        return update_terminology(session, info)
+    else:
+        return create_terminology(session, info)
+
+
+@router.delete("")
+async def delete(session: SessionDep, id_list: list[int]):
+    delete_terminology(session, id_list)

backend/apps/terminology/api/__init__.py

@@ -0,0 +1,196 @@
+import datetime
+from typing import List, Optional
+
+from sqlalchemy import and_, or_, select, func, delete, update
+from sqlalchemy.dialects.postgresql import JSONB
+from sqlalchemy.orm import aliased
+
+from apps.terminology.models.terminology_model import Terminology, TerminologyInfo
+from common.core.deps import SessionDep
+
+
+def page_terminology(session: SessionDep, current_page: int = 1, page_size: int = 10, name: Optional[str] = None):
+    _list: List[TerminologyInfo] = []
+
+    child = aliased(Terminology)
+
+    current_page = max(1, current_page)
+    page_size = max(10, page_size)
+
+    total_count = 0
+    total_pages = 0
+
+    if name and name.strip() != "":
+        keyword_pattern = f"%{name.strip()}%"
+        # 步骤1：先找到所有匹配的节点ID（无论是父节点还是子节点）
+        matched_ids_subquery = (
+            select(Terminology.id)
+            .where(Terminology.name.like(keyword_pattern))  # LIKE查询条件
+            .subquery()
+        )
+
+        # 步骤2：找到这些匹配节点的所有父节点（包括自身如果是父节点）
+        parent_ids_subquery = (
+            select(Terminology.id)
+            .where(
+                (Terminology.id.in_(matched_ids_subquery)) |
+                (Terminology.id.in_(
+                    select(Terminology.pid)
+                    .where(Terminology.id.in_(matched_ids_subquery))
+                    .where(Terminology.pid.isnot(None))
+                ))
+            )
+            .where(Terminology.pid.is_(None))  # 只取父节点
+        )
+
+        count_stmt = select(func.count()).select_from(parent_ids_subquery.subquery())
+        total_count = session.execute(count_stmt).scalar()
+        total_pages = (total_count + page_size - 1) // page_size
+
+        # 步骤3：获取分页后的父节点ID
+        paginated_parent_ids = (
+            parent_ids_subquery
+            .order_by(Terminology.create_time.desc())
+            .offset((current_page - 1) * page_size)
+            .limit(page_size)
+            .subquery()
+        )
+
+        # 步骤4：获取这些父节点的childrenNames
+        children_subquery = (
+            select(
+                child.pid,
+                func.jsonb_agg(child.name).order_by(child.create_time.desc()).label('children_names')
+            )
+            .where(child.pid.isnot(None))
+            .group_by(child.pid)
+            .subquery()
+        )
+
+        # 主查询
+        stmt = (
+            select(
+                Terminology.id,
+                Terminology.word,
+                Terminology.create_time,
+                Terminology.description,
+                func.coalesce(
+                    children_subquery.c.children_names,
+                    func.cast('[]', JSONB)
+                ).label('other_words')
+            )
+            .outerjoin(
+                children_subquery,
+                Terminology.id == children_subquery.c.pid
+            )
+            .where(Terminology.id.in_(paginated_parent_ids))
+            .order_by(Terminology.create_time.desc())
+        )
+        print(str(stmt))
+    else:
+        parent_ids_subquery = (
+            select(Terminology.id)
+            .where(Terminology.pid.is_(None))  # 只取父节点
+        )
+        count_stmt = select(func.count()).select_from(parent_ids_subquery.subquery())
+        total_count = session.execute(count_stmt).scalar()
+        total_pages = (total_count + page_size - 1) // page_size
+
+        paginated_parent_ids = (
+            parent_ids_subquery
+            .order_by(Terminology.create_time.desc())
+            .offset((current_page - 1) * page_size)
+            .limit(page_size)
+            .subquery()
+        )
+
+        stmt = (
+            select(
+                Terminology.id,
+                Terminology.word,
+                Terminology.create_time,
+                Terminology.description,
+                func.coalesce(
+                    func.jsonb_agg(child.word),
+                    func.cast('[]', JSONB)
+                ).label('other_words')
+            )
+            .outerjoin(child, and_(Terminology.id == child.pid))
+            .where(Terminology.id.in_(paginated_parent_ids))
+            .group_by(Terminology.id, Terminology.word)
+            .order_by(Terminology.create_time.desc())
+        )
+        print(str(stmt))
+
+    result = session.execute(stmt)
+
+    for row in result:
+        _list.append(TerminologyInfo(
+            id=row.id,
+            word=row.word,
+            create_time=row.create_time,
+            description=row.description,
+            other_words=row.other_words,
+        ))
+
+    return current_page, page_size, total_count, total_pages, _list
+
+
+def create_terminology(session: SessionDep, info: TerminologyInfo):
+    create_time = datetime.datetime.now()
+    parent = Terminology(word=info.word, create_time=create_time, description=info.description)
+
+    result = Terminology(**parent.model_dump())
+
+    session.add(parent)
+    session.flush()
+    session.refresh(parent)
+
+    result.id = parent.id
+    session.commit()
+
+    _list: List[Terminology] = []
+    if info.other_words:
+        for other_word in info.other_words:
+            _list.append(
+                Terminology(pid=result.id, word=other_word, create_time=create_time, description=info.description))
+    session.bulk_save_objects(_list)
+    session.flush()
+    session.commit()
+
+    # todo embedding
+
+    return result.id
+
+
+def update_terminology(session: SessionDep, info: TerminologyInfo):
+    stmt = update(Terminology).where(and_(Terminology.id == info.id)).values(
+        word=info.word,
+        description=info.description,
+    )
+    session.execute(stmt)
+    session.commit()
+
+    stmt = delete(Terminology).where(and_(Terminology.pid == info.id))
+    session.execute(stmt)
+    session.commit()
+
+    create_time = datetime.datetime.now()
+    _list: List[Terminology] = []
+    if info.other_words:
+        for other_word in info.other_words:
+            _list.append(
+                Terminology(pid=info.id, word=other_word, create_time=create_time, description=info.description))
+    session.bulk_save_objects(_list)
+    session.flush()
+    session.commit()
+
+    # todo embedding
+
+    return info.id
+
+
+def delete_terminology(session: SessionDep, ids: list[int]):
+    stmt = delete(Terminology).where(or_(Terminology.id.in_(ids), Terminology.pid.in_(ids)))
+    session.execute(stmt)
+    session.commit()