feat: terminology add oid

Author: ulleo

backend/alembic/versions/041_add_terminology_oid.py

@@ -0,0 +1,33 @@
+"""041_add_terminology_oid
+
+Revision ID: c4c3c36b720d
+Revises: 0fc14c2cfe41
+Create Date: 2025-08-28 16:41:33.977242
+
+"""
+from alembic import op
+import sqlalchemy as sa
+import sqlmodel.sql.sqltypes
+from sqlalchemy.dialects import postgresql
+
+# revision identifiers, used by Alembic.
+revision = 'c4c3c36b720d'
+down_revision = '0fc14c2cfe41'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+
+    op.add_column('terminology', sa.Column('oid', sa.BigInteger(), nullable=True))
+
+    op.execute('update terminology set oid=1 where oid is null')
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_column('terminology', 'oid')
+    
+    # ### end Alembic commands ###

backend/apps/chat/task/llm.py

@@ -220,7 +220,7 @@ def generate_analysis(self):
         self.chat_question.data = orjson.dumps(data.get('data')).decode()
         analysis_msg: List[Union[BaseMessage, dict[str, Any]]] = []
 
-        self.chat_question.terminologies = get_terminology_template(self.session, self.chat_question.question)
+        self.chat_question.terminologies = get_terminology_template(self.session, self.chat_question.question, self.current_user.oid)
 
         analysis_msg.append(SystemMessage(content=self.chat_question.analysis_sys_question()))
         analysis_msg.append(HumanMessage(content=self.chat_question.analysis_user_question()))
@@ -894,7 +894,7 @@ def run_task_cache(self, in_chat: bool = True):
 
     def run_task(self, in_chat: bool = True):
         try:
-            self.chat_question.terminologies = get_terminology_template(self.session, self.chat_question.question)
+            self.chat_question.terminologies = get_terminology_template(self.session, self.chat_question.question, self.current_user.oid)
             self.init_messages()
 
             # return id

backend/apps/terminology/api/terminology.py

@@ -5,15 +5,15 @@
 from apps.terminology.curd.terminology import page_terminology, create_terminology, update_terminology, \
     delete_terminology
 from apps.terminology.models.terminology_model import TerminologyInfo
-from common.core.deps import SessionDep
+from common.core.deps import SessionDep, CurrentUser, Trans
 
 router = APIRouter(tags=["Terminology"], prefix="/system/terminology")
 
 
 @router.get("/page/{current_page}/{page_size}")
-async def pager(session: SessionDep, current_page: int, page_size: int,
+async def pager(session: SessionDep, current_user: CurrentUser, current_page: int, page_size: int,
                 word: Optional[str] = Query(None, description="搜索术语(可选)")):
-    current_page, page_size, total_count, total_pages, _list = page_terminology(session, current_page, page_size, word)
+    current_page, page_size, total_count, total_pages, _list = page_terminology(session, current_page, page_size, word, current_user.oid)
 
     return {
         "current_page": current_page,
@@ -25,11 +25,12 @@ async def pager(session: SessionDep, current_page: int, page_size: int,
 
 
 @router.put("")
-async def create_or_update(session: SessionDep, info: TerminologyInfo):
+async def create_or_update(session: SessionDep, current_user: CurrentUser, trans: Trans, info: TerminologyInfo):
+    oid = current_user.oid
     if info.id:
-        return update_terminology(session, info)
+        return update_terminology(session, info, oid, trans)
     else:
-        return create_terminology(session, info)
+        return create_terminology(session, info, oid)
 
 
 @router.delete("")

backend/apps/terminology/curd/terminology.py

@@ -15,12 +15,13 @@
 from apps.template.generate_chart.generator import get_base_terminology_template
 from apps.terminology.models.terminology_model import Terminology, TerminologyInfo
 from common.core.config import settings
-from common.core.deps import SessionDep
+from common.core.deps import SessionDep, Trans
 
 executor = ThreadPoolExecutor(max_workers=200)
 
 
-def page_terminology(session: SessionDep, current_page: int = 1, page_size: int = 10, name: Optional[str] = None):
+def page_terminology(session: SessionDep, current_page: int = 1, page_size: int = 10, name: Optional[str] = None,
+                     oid: Optional[int] = 1):
     _list: List[TerminologyInfo] = []
 
     child = aliased(Terminology)
@@ -91,7 +92,7 @@ def page_terminology(session: SessionDep, current_page: int = 1, page_size: int
                 children_subquery,
                 Terminology.id == children_subquery.c.pid
             )
-            .where(Terminology.id.in_(paginated_parent_ids))
+            .where(and_(Terminology.id.in_(paginated_parent_ids), Terminology.oid == oid))
             .order_by(Terminology.create_time.desc())
         )
     else:
@@ -120,7 +121,7 @@ def page_terminology(session: SessionDep, current_page: int = 1, page_size: int
                 func.jsonb_agg(child.word).filter(child.word.isnot(None)).label('other_words')
             )
             .outerjoin(child, and_(Terminology.id == child.pid))
-            .where(Terminology.id.in_(paginated_parent_ids))
+            .where(and_(Terminology.id.in_(paginated_parent_ids), Terminology.oid == oid))
             .group_by(Terminology.id, Terminology.word)
             .order_by(Terminology.create_time.desc())
         )
@@ -139,9 +140,9 @@ def page_terminology(session: SessionDep, current_page: int = 1, page_size: int
     return current_page, page_size, total_count, total_pages, _list
 
 
-def create_terminology(session: SessionDep, info: TerminologyInfo):
+def create_terminology(session: SessionDep, info: TerminologyInfo, oid: int):
     create_time = datetime.datetime.now()
-    parent = Terminology(word=info.word, create_time=create_time, description=info.description)
+    parent = Terminology(word=info.word, create_time=create_time, description=info.description, oid=oid)
 
     result = Terminology(**parent.model_dump())
 
@@ -158,7 +159,7 @@ def create_terminology(session: SessionDep, info: TerminologyInfo):
             if other_word.strip() == "":
                 continue
             _list.append(
-                Terminology(pid=result.id, word=other_word, create_time=create_time))
+                Terminology(pid=result.id, word=other_word, create_time=create_time, oid=oid))
     session.bulk_save_objects(_list)
     session.flush()
     session.commit()
@@ -169,7 +170,14 @@ def create_terminology(session: SessionDep, info: TerminologyInfo):
     return result.id
 
 
-def update_terminology(session: SessionDep, info: TerminologyInfo):
+def update_terminology(session: SessionDep, info: TerminologyInfo, oid: int, trans: Trans):
+    count = session.query(Terminology).filter(
+        Terminology.oid == oid,
+        Terminology.id == info.id
+    ).count()
+    if count == 0:
+        raise Exception(trans('i18n_terminology.terminology_not_exists'))
+
     stmt = update(Terminology).where(and_(Terminology.id == info.id)).values(
         word=info.word,
         description=info.description,
@@ -188,14 +196,15 @@ def update_terminology(session: SessionDep, info: TerminologyInfo):
             if other_word.strip() == "":
                 continue
             _list.append(
-                Terminology(pid=info.id, word=other_word, create_time=create_time))
+                Terminology(pid=info.id, word=other_word, create_time=create_time, oid=oid))
     session.bulk_save_objects(_list)
     session.flush()
     session.commit()
 
     # embedding
     run_save_embeddings([info.id])
 
+
     return info.id
 
 
@@ -256,23 +265,19 @@ def save_embeddings(ids: List[int]):
 
 
 embedding_sql = f"""
-SELECT id, pid, word, description, similarity
+SELECT id, pid, word, similarity
 FROM
-(SELECT id, pid, word, 
-COALESCE(
-        description,
-        (SELECT description FROM terminology AS parent WHERE parent.id = child.pid)
-    ) AS description,
+(SELECT id, pid, word, oid,
 ( 1 - (embedding <=> :embedding_array) ) AS similarity
 FROM terminology AS child
 ) TEMP
-WHERE similarity > {settings.EMBEDDING_SIMILARITY}
+WHERE similarity > {settings.EMBEDDING_SIMILARITY} and oid = :oid
 ORDER BY similarity DESC
 LIMIT {settings.EMBEDDING_TOP_COUNT}
 """
 
 
-def select_terminology_by_word(session: SessionDep, word: str):
+def select_terminology_by_word(session: SessionDep, word: str, oid: int):
     if word.strip() == "":
         return []
 
@@ -283,48 +288,48 @@ def select_terminology_by_word(session: SessionDep, word: str):
             Terminology.id,
             Terminology.pid,
             Terminology.word,
-            func.coalesce(
-                Terminology.description,
-                select(Terminology.description)
-                .where(and_(Terminology.id == Terminology.pid))
-                .scalar_subquery()
-            ).label('description')
         )
         .where(
-            text(":sentence ILIKE '%' || word || '%'")
+            and_(text(":sentence ILIKE '%' || word || '%'"), Terminology.oid == oid)
         )
     )
 
     results = session.execute(stmt, {'sentence': word}).fetchall()
 
     for row in results:
-        _list.append(Terminology(id=row.id, word=row.word, pid=row.pid, description=row.description))
+        _list.append(Terminology(id=row.id, word=row.word, pid=row.pid))
 
     if settings.EMBEDDING_ENABLED:
         try:
             model = EmbeddingModelCache.get_model()
 
             embedding = model.embed_query(word)
 
-            print(embedding_sql)
-            results = session.execute(text(embedding_sql), {'embedding_array': str(embedding)})
+            results = session.execute(text(embedding_sql), {'embedding_array': str(embedding), 'oid': oid})
 
             for row in results:
-                _list.append(Terminology(id=row.id, word=row.word, pid=row.pid, description=row.description))
+                _list.append(Terminology(id=row.id, word=row.word, pid=row.pid))
 
         except Exception:
             traceback.print_exc()
 
     _map: dict = {}
-    _ids: set[int] = set()
+    _ids: list[int] = []
     for row in _list:
-        if row.id in _ids:
+        if row.id in _ids or row.pid in _ids:
             continue
-        _ids.add(row.id)
-        if row.pid:
-            pid = str(row.pid)
+        if row.pid is not None:
+            _ids.append(row.pid)
         else:
-            pid = str(row.id)
+            _ids.append(row.id)
+
+    if len(_ids) == 0:
+        return []
+
+    t_list = session.query(Terminology.id, Terminology.pid, Terminology.word, Terminology.description).filter(
+        or_(Terminology.id.in_(_ids), Terminology.pid.in_(_ids))).all()
+    for row in t_list:
+        pid = str(row.pid) if row.pid is not None else str(row.id)
         if _map.get(pid) is None:
             _map[pid] = {'words': [], 'description': row.description}
         _map[pid]['words'].append(row.word)
@@ -350,6 +355,7 @@ def to_xml_string(_dict: list[dict] | dict, root: str = 'terminologies') -> str:
     item_name_func = lambda x: 'terminology' if x == 'terminologies' else 'word' if x == 'words' else 'item'
     dicttoxml.LOG.setLevel(logging.ERROR)
     xml = dicttoxml.dicttoxml(_dict,
+                              cdata=['word', 'description'],
                               custom_root=root,
                               item_func=item_name_func,
                               xml_declaration=False,
@@ -361,11 +367,22 @@ def to_xml_string(_dict: list[dict] | dict, root: str = 'terminologies') -> str:
         end_index = pretty_xml.find('>') + 1
         pretty_xml = pretty_xml[end_index:].lstrip()
 
+    # 替换所有 XML 转义字符
+    escape_map = {
+        '&lt;': '<',
+        '&gt;': '>',
+        '&amp;': '&',
+        '&quot;': '"',
+        '&apos;': "'"
+    }
+    for escaped, original in escape_map.items():
+        pretty_xml = pretty_xml.replace(escaped, original)
+
     return pretty_xml
 
 
-def get_terminology_template(session: SessionDep, question: str) -> str:
-    _results = select_terminology_by_word(session, question)
+def get_terminology_template(session: SessionDep, question: str, oid: int) -> str:
+    _results = select_terminology_by_word(session, question, oid)
     if _results and len(_results) > 0:
         terminology = to_xml_string(_results)
         template = get_base_terminology_template().format(terminologies=terminology)

backend/apps/terminology/models/terminology_model.py

@@ -10,6 +10,7 @@
 class Terminology(SQLModel, table=True):
     __tablename__ = "terminology"
     id: Optional[int] = Field(sa_column=Column(BigInteger, Identity(always=True), primary_key=True))
+    oid: Optional[int] = Field(sa_column=Column(BigInteger, nullable=True, default=1))
     pid: Optional[int] = Field(sa_column=Column(BigInteger, nullable=True))
     create_time: Optional[datetime] = Field(sa_column=Column(DateTime(timezone=False), nullable=True))
     word: Optional[str] = Field(max_length=255)

backend/locales/en.json

@@ -1,40 +1,43 @@
 {
-    "i18n_default_workspace": "Default workspace",
-    "i18n_ds_name_exist": "Name already exists",
-    "i18n_concat_admin": "Please contact administrator!",
-    "i18n_exist": "{msg} already exists!",
-    "i18n_not_exist": "{msg} not exists",
-    "i18n_error": "{key} error!",
-    "i18n_miss_args": "Missing {key} parameter!",
-    "i18n_format_invalid": "{key} format is incorrect!",
-    "i18n_login": {
-        "account_pwd_error": "Account or password error!",
-        "no_associated_ws": "No associated workspace, {msg}",
-        "user_disable": "Account disabled, {msg}"
-    },
-    "i18n_user": {
-        "account": "Account",
-        "email": "Email",
-        "password": "Password",
-        "language_not_support": "System does not support [{key}] language!",
-        "ws_miss": "The current user is not in the workspace [{ws}]!"
-    },
-    "i18n_ws": {
-        "title": "Workspace"
-    },
-    "i18n_permission": {
-        "only_admin": "Only administrators can call this!",
-        "no_permission": "No permission to access {url}{msg}",
-        "authenticate_invalid": "Authenticate invalid [{msg}]",
-        "token_expired": "Token has expired"
-    },
-    "i18n_llm": {
-        "validate_error": "Validation failed [{msg}]",
-        "delete_default_error": "Cannot delete default model [{key}]!",
-        "miss_default": "The default large language model has not been configured"
-    },
-    "i18n_ds_invalid": "Datasource Invalid",
-    "i18n_embedded": {
-        "invalid_origin": "Domain verification failed [{origin}]"
-    }
+  "i18n_default_workspace": "Default workspace",
+  "i18n_ds_name_exist": "Name already exists",
+  "i18n_concat_admin": "Please contact administrator!",
+  "i18n_exist": "{msg} already exists!",
+  "i18n_not_exist": "{msg} not exists",
+  "i18n_error": "{key} error!",
+  "i18n_miss_args": "Missing {key} parameter!",
+  "i18n_format_invalid": "{key} format is incorrect!",
+  "i18n_login": {
+    "account_pwd_error": "Account or password error!",
+    "no_associated_ws": "No associated workspace, {msg}",
+    "user_disable": "Account disabled, {msg}"
+  },
+  "i18n_user": {
+    "account": "Account",
+    "email": "Email",
+    "password": "Password",
+    "language_not_support": "System does not support [{key}] language!",
+    "ws_miss": "The current user is not in the workspace [{ws}]!"
+  },
+  "i18n_ws": {
+    "title": "Workspace"
+  },
+  "i18n_permission": {
+    "only_admin": "Only administrators can call this!",
+    "no_permission": "No permission to access {url}{msg}",
+    "authenticate_invalid": "Authenticate invalid [{msg}]",
+    "token_expired": "Token has expired"
+  },
+  "i18n_llm": {
+    "validate_error": "Validation failed [{msg}]",
+    "delete_default_error": "Cannot delete default model [{key}]!",
+    "miss_default": "The default large language model has not been configured"
+  },
+  "i18n_ds_invalid": "Datasource Invalid",
+  "i18n_embedded": {
+    "invalid_origin": "Domain verification failed [{origin}]"
+  },
+  "i18n_terminology": {
+    "terminology_not_exists": "Terminology does not exists"
+  }
 }