Merge remote-tracking branch 'origin/main'

Author: ziyujiahao

README.md

@@ -56,6 +56,15 @@ TBD
 - Backend：[Python / Django](https://www.djangoproject.com/)
 - Database：[PostgreSQL + pgvector](https://www.postgresql.org/)
 
+## 飞致云的其他明星项目
+
+- [DataEase](https://github.com/dataease/dataease/) - 人人可用的开源 BI 工具
+- [1Panel](https://github.com/1panel-dev/1panel/) - 现代化、开源的 Linux 服务器运维管理面板
+- [MaxKB](https://github.com/1panel-dev/MaxKB/) - 基于 LLM 大语言模型的开源知识库问答系统
+- [JumpServer](https://github.com/jumpserver/jumpserver/) - 广受欢迎的开源堡垒机
+- [Halo](https://github.com/halo-dev/halo/) - 强大易用的开源建站工具
+- [MeterSphere](https://github.com/metersphere/metersphere/) - 新一代的开源持续测试工具
+
 ## Star History
 
 [![Star History Chart](https://api.star-history.com/svg?repos=1Panel-dev/SQLBot&type=Date)](https://star-history.com/#1Panel-dev/SQLBot&Date)

backend/apps/chat/task/llm.py

@@ -536,8 +536,8 @@ def generate_filter(self, sql: str, tables: List):
                     ).first()
                     if obj is not None:
                         res.append(transRecord2DTO(self.session, permission))
-            wheres = transFilterTree(self.session, res, self.ds)
-            filters.append({"table": table.table_name, "filter": wheres})
+            where_str = transFilterTree(self.session, res, self.ds)
+            filters.append({"table": table.table_name, "filter": where_str})
 
         filter = json.dumps(filters, ensure_ascii=False)
 
@@ -662,6 +662,16 @@ def check_save_chart(self, res: str) -> Dict[str, Any]:
         if data['type'] and data['type'] != 'error':
             # todo type check
             chart = data
+            if chart.get('columns'):
+                for v in chart.get('columns'):
+                    v['value'] = v.get('value').lower()
+            if chart.get('axis'):
+                if chart.get('axis').get('x'):
+                    chart.get('axis').get('x')['value'] = chart.get('axis').get('x').get('value').lower()
+                if chart.get('axis').get('y'):
+                    chart.get('axis').get('y')['value'] = chart.get('axis').get('y').get('value').lower()
+                if chart.get('axis').get('series'):
+                    chart.get('axis').get('series')['value'] = chart.get('axis').get('series').get('value').lower()
         elif data['type'] == 'error':
             message = data['reason']
             error = True

backend/apps/datasource/api/datasource.py

@@ -8,6 +8,7 @@
 
 from apps.db.engine import create_table, get_data_engine, insert_data
 from common.core.deps import SessionDep, CurrentUser, Trans
+from common.utils.utils import SQLBotLogUtil
 from ..crud.datasource import get_datasource_list, check_status, create_ds, update_ds, delete_ds, getTables, getFields, \
     execSql, update_table_and_fields, getTablesByDs, chooseTables, preview, updateTable, updateField, get_ds, fieldEnum
 from ..crud.field import get_fields_by_table_id
@@ -17,6 +18,11 @@
 router = APIRouter(tags=["datasource"], prefix="/datasource")
 path = "/opt/sqlbot/data/excel"
 
+@router.get("/ws/{oid}", include_in_schema=False)
+async def query_by_oid(session: SessionDep, user: CurrentUser, oid: int) -> List[CoreDatasource]:
+    if not user.isAdmin:
+        raise Exception("no permission to execute")
+    return get_datasource_list(session=session, user=user, oid=oid)
 
 @router.get("/list")
 async def datasource_list(session: SessionDep, user: CurrentUser):
@@ -60,7 +66,11 @@ async def get_tables(session: SessionDep, id: int):
 
 @router.post("/getTablesByConf")
 async def get_tables_by_conf(session: SessionDep, ds: CoreDatasource):
-    return getTablesByDs(session, ds)
+    try:
+        return getTablesByDs(session, ds)
+    except Exception as e:
+        SQLBotLogUtil.error(f"get table failed: {e}")
+        raise HTTPException(status_code=500, detail=f'Get table Failed: {e.args}')
 
 
 @router.post("/getFields/{id}/{table_name}")
@@ -100,7 +110,11 @@ async def edit_field(session: SessionDep, field: CoreField):
 
 @router.post("/previewData/{id}")
 async def edit_local(session: SessionDep, current_user: CurrentUser, id: int, data: TableObj):
-    return preview(session, current_user, id, data)
+    try:
+        return preview(session, current_user, id, data)
+    except Exception as e:
+        SQLBotLogUtil.error(f"Preview failed: {e}")
+        raise HTTPException(status_code=500, detail=f'Preview Failed: {e.args}')
 
 
 @router.post("/fieldEnum/{id}")

backend/apps/datasource/crud/datasource.py

@@ -1,6 +1,6 @@
 import datetime
 import json
-from typing import List
+from typing import List, Optional
 
 from fastapi import HTTPException
 from sqlalchemy import and_, text, cast, or_, func
@@ -25,10 +25,13 @@
     DatasourceConf, TableAndFields
 
 
-def get_datasource_list(session: SessionDep, user: CurrentUser):
-    oid = user.oid if user.oid is not None else 1
-    return session.query(CoreDatasource).filter(CoreDatasource.oid == oid).order_by(
-        func.convert_to(CoreDatasource.name, 'gbk')).all()
+def get_datasource_list(session: SessionDep, user: CurrentUser, oid: Optional[int] = None) -> List[CoreDatasource]:
+    current_oid = user.oid if user.oid is not None else 1
+    if user.isAdmin and oid:
+        current_oid = oid
+    return session.exec(select(CoreDatasource).where(CoreDatasource.oid == current_oid).order_by(
+        func.convert_to(CoreDatasource.name, 'gbk'))).all()
+    
 
 
 def get_ds(session: SessionDep, id: int):
@@ -134,7 +137,7 @@ def getTables(session: SessionDep, id: int):
 
 
 def getTablesByDs(session: SessionDep, ds: CoreDatasource):
-    check_status(session, ds, True)
+    # check_status(session, ds, True)
     tables = get_tables(ds)
     return tables
 
@@ -241,7 +244,7 @@ def updateField(session: SessionDep, field: CoreField):
 
 def preview(session: SessionDep, current_user: CurrentUser, id: int, data: TableObj):
     ds = session.query(CoreDatasource).filter(CoreDatasource.id == id).first()
-    check_status(session, ds, True)
+    # check_status(session, ds, True)
 
     if data.fields is None or len(data.fields) == 0:
         return {"fields": [], "data": [], "sql": ''}
@@ -278,8 +281,8 @@ def preview(session: SessionDep, current_user: CurrentUser, id: int, data: Table
                 ).first()
                 if obj is not None:
                     res.append(transRecord2DTO(session, permission))
-        wheres = transFilterTree(session, res, ds)
-        where = (' where ' + wheres) if wheres is not None and wheres != '' else ''
+        where_str = transFilterTree(session, res, ds)
+        where = (' where ' + where_str) if where_str is not None and where_str != '' else ''
 
     fields = [f.field_name for f in f_list]
     if fields is None or len(fields) == 0:

backend/apps/db/db.py

@@ -254,20 +254,17 @@ def get_fields(ds: CoreDatasource, table_name: str = None):
 
 
 def exec_sql(ds: CoreDatasource | AssistantOutDsSchema, sql: str):
-    session = get_session(ds)
-    result = session.execute(text(sql))
-    try:
-        columns = result.keys()._keys
-        res = result.fetchall()
-        result_list = [
-            {columns[i]: float(value) if isinstance(value, Decimal) else value for i, value in enumerate(tuple_item)}
-            for tuple_item in res
-        ]
-        return {"fields": columns, "data": result_list, "sql": bytes.decode(base64.b64encode(bytes(sql, 'utf-8')))}
-    except Exception as ex:
-        raise ex
-    finally:
-        if result is not None:
-            result.close()
-        if session is not None:
-            session.close()
+    with get_session(ds) as session:
+        with session.execute(text(sql)) as result:
+            try:
+                columns = result.keys()._keys
+                res = result.fetchall()
+                result_list = [
+                    {columns[i]: float(value) if isinstance(value, Decimal) else value for i, value in
+                     enumerate(tuple_item)}
+                    for tuple_item in res
+                ]
+                return {"fields": columns, "data": result_list,
+                        "sql": bytes.decode(base64.b64encode(bytes(sql, 'utf-8')))}
+            except Exception as ex:
+                raise ex

backend/apps/system/api/aimodel.py

@@ -1,15 +1,35 @@
 import json
 from typing import List, Union
+from apps.ai_model.model_factory import LLMConfig, LLMFactory
 from apps.system.schemas.ai_model_schema import AiModelConfigItem, AiModelCreator, AiModelEditor, AiModelGridItem
 from fastapi import APIRouter, Query
 from sqlmodel import func, select, update
 
 from apps.system.models.system_model import AiModelDetail
-from common.core.deps import SessionDep
+from common.core.deps import SessionDep, Trans
 from common.utils.time import get_timestamp
+from common.utils.utils import SQLBotLogUtil
 
 router = APIRouter(tags=["system/aimodel"], prefix="/system/aimodel")
 
+@router.post("/status")
+async def check_llm(info: AiModelCreator, trans: Trans):
+    try:
+        additional_params = {item.key: item.val for item in info.config_list}
+        config = LLMConfig(
+            model_type="openai",
+            model_name=info.base_model,
+            api_key=info.api_key,
+            api_base_url=info.api_domain,
+            additional_params=additional_params,
+        )
+        llm_instance = LLMFactory.create_llm(config)
+        result = llm_instance.llm.invoke("who are you?")
+        SQLBotLogUtil.info(f"check_llm result: {result}")
+    except Exception as e:
+        SQLBotLogUtil.error(f"Error checking LLM: {e}")
+        raise Exception(trans('i18n_llm.validate_error', msg = str(e)))
+
 @router.get("", response_model=list[AiModelGridItem])
 async def query(
         session: SessionDep,
@@ -74,19 +94,20 @@ async def update_model(
     data["config"] = json.dumps([item.model_dump(exclude_unset=True) for item in editor.config_list])
     data.pop("config_list", None)
     db_model = session.get(AiModelDetail, id)
-    update_data = AiModelDetail.model_validate(data)
-    db_model.sqlmodel_update(update_data)
+    #update_data = AiModelDetail.model_validate(data)
+    db_model.sqlmodel_update(data)
     session.add(db_model)
     session.commit()
 
 @router.delete("/{id}")
 async def delete_model(
         session: SessionDep,
+        trans: Trans,
         id: int
 ):
     item = session.get(AiModelDetail, id)
     if item.default_model:
-        raise RuntimeError(f"Can not delete [${item.name}], because it is default model!")
+        raise Exception(trans('i18n_llm.delete_default_error', key = item.name))
     session.delete(item)
     session.commit()
 

backend/apps/system/api/login.py

@@ -2,7 +2,7 @@
 from fastapi import APIRouter, Depends, HTTPException
 from fastapi.security import OAuth2PasswordRequestForm
 from apps.system.schemas.system_schema import BaseUserDTO
-from common.core.deps import SessionDep
+from common.core.deps import SessionDep, Trans
 from ..crud.user import authenticate
 from common.core.security import create_access_token
 from datetime import timedelta
@@ -13,14 +13,16 @@
 @router.post("/access-token")
 def local_login(
     session: SessionDep,
+    trans: Trans,
     form_data: Annotated[OAuth2PasswordRequestForm, Depends()]
 ) -> Token:
     user: BaseUserDTO = authenticate(session=session, account=form_data.username, password=form_data.password)
     if not user:
-        raise HTTPException(status_code=400, detail="Incorrect account or password")
-    
+        raise HTTPException(status_code=400, detail=trans('i18n_login.account_pwd_error'))
     if not user.oid or user.oid == 0:
-        raise HTTPException(status_code=400, detail="No associated workspace, Please contact the administrator")
+        raise HTTPException(status_code=400, detail=trans('i18n_login.no_associated_ws', msg = trans('i18n_concat_admin')))
+    if user.status != 1:
+        raise HTTPException(status_code=400, detail=trans('i18n_login.user_disable', msg = trans('i18n_concat_admin')))
     access_token_expires = timedelta(minutes=settings.ACCESS_TOKEN_EXPIRE_MINUTES)
     user_dict = user.to_dict()
     return Token(access_token=create_access_token(

backend/apps/system/api/user.py

@@ -1,11 +1,11 @@
 from typing import Optional
 from fastapi import APIRouter, HTTPException, Query
 from sqlmodel import func, or_, select, delete as sqlmodel_delete
-from apps.system.crud.user import get_db_user, single_delete, user_ws_options
-from apps.system.models.system_model import UserWsModel
+from apps.system.crud.user import check_account_exists, check_email_exists, check_email_format, check_pwd_format, get_db_user, single_delete, user_ws_options
+from apps.system.models.system_model import UserWsModel, WorkspaceModel
 from apps.system.models.user import UserModel
 from apps.system.schemas.auth import CacheName, CacheNamespace
-from apps.system.schemas.system_schema import PwdEditor, UserCreator, UserEditor, UserGrid, UserLanguage, UserWs
+from apps.system.schemas.system_schema import PwdEditor, UserCreator, UserEditor, UserGrid, UserLanguage, UserStatus, UserWs
 from common.core.deps import CurrentUser, SessionDep, Trans
 from common.core.pagination import Paginator
 from common.core.schemas import PaginatedResponse, PaginationParams
@@ -96,10 +96,13 @@ async def ws_options(session: SessionDep, current_user: CurrentUser, trans: Tran
 
 @router.put("/ws/{oid}")
 @clear_cache(namespace=CacheNamespace.AUTH_INFO, cacheName=CacheName.USER_INFO, keyExpression="current_user.id")
-async def ws_change(session: SessionDep, current_user: CurrentUser, oid: int):
+async def ws_change(session: SessionDep, current_user: CurrentUser, trans:Trans, oid: int):
     ws_list: list[UserWs] = await user_ws_options(session, current_user.id)
     if not any(x.id == oid for x in ws_list):
-        raise HTTPException(f"oid [{oid}] is invalid!")
+        db_ws = session.get(WorkspaceModel, oid)
+        if db_ws:
+            raise Exception(trans('i18n_user.ws_miss', ws = db_ws.name))
+        raise Exception(trans('i18n_not_exist', msg = f"{trans('i18n_ws.title')}[{oid}]"))
     user_model: UserModel = get_db_user(session = session, user_id = current_user.id)
     user_model.oid = oid
     session.add(user_model)
@@ -115,7 +118,13 @@ async def query(session: SessionDep, trans: Trans, id: int) -> UserEditor:
     return result
 
 @router.post("")
-async def create(session: SessionDep, creator: UserCreator):
+async def create(session: SessionDep, creator: UserCreator, trans: Trans):
+    if check_account_exists(session=session, account=creator.account):
+        raise Exception(trans('i18n_exist', msg = f"{trans('i18n_user.account')} [{creator.account}]"))
+    if check_email_exists(session=session, email=creator.email):
+        raise Exception(trans('i18n_exist', msg = f"{trans('i18n_user.email')} [{creator.email}]"))
+    if not check_email_format(creator.email):
+        raise Exception(trans('i18n_format_invalid', key = f"{trans('i18n_user.email')} [{creator.email}]"))
     data = creator.model_dump(exclude_unset=True)
     user_model = UserModel.model_validate(data)
     #user_model.create_time = get_timestamp()
@@ -138,8 +147,16 @@ async def create(session: SessionDep, creator: UserCreator):
 
 @router.put("")
 @clear_cache(namespace=CacheNamespace.AUTH_INFO, cacheName=CacheName.USER_INFO, keyExpression="editor.id")
-async def update(session: SessionDep, editor: UserEditor):
+async def update(session: SessionDep, editor: UserEditor, trans: Trans):
     user_model: UserModel = get_db_user(session = session, user_id = editor.id)
+    if not user_model:
+        raise Exception(f"User with id [{editor.id}] not found!")
+    if editor.account != user_model.account:
+        raise Exception(f"account cannot be changed!")
+    if editor.email != user_model.email and check_email_exists(session=session, account=editor.email):
+        raise Exception(trans('i18n_exist', msg = f"{trans('i18n_user.email')} [{editor.email}]"))
+    if not check_email_format(editor.email):
+        raise Exception(trans('i18n_format_invalid', key = f"{trans('i18n_user.email')} [{editor.email}]"))
     origin_oid: int = user_model.oid
     del_stmt = sqlmodel_delete(UserWsModel).where(UserWsModel.uid == editor.id)
     session.exec(del_stmt)
@@ -174,31 +191,47 @@ async def batch_del(session: SessionDep, id_list: list[int]):
 
 @router.put("/language")
 @clear_cache(namespace=CacheNamespace.AUTH_INFO, cacheName=CacheName.USER_INFO, keyExpression="current_user.id")
-async def langChange(session: SessionDep, current_user: CurrentUser, language: UserLanguage):
+async def langChange(session: SessionDep, current_user: CurrentUser, trans: Trans, language: UserLanguage):
     lang = language.language
     if lang not in ["zh-CN", "en"]:
-        return {"message": "Language not supported"}
+        raise Exception(trans('i18n_user.language_not_support', key = lang))
     db_user: UserModel = get_db_user(session=session, user_id=current_user.id)
     db_user.language = lang
     session.add(db_user)
     session.commit()
 
 @router.patch("/pwd/{id}")
 @clear_cache(namespace=CacheNamespace.AUTH_INFO, cacheName=CacheName.USER_INFO, keyExpression="id")
-async def pwdReset(session: SessionDep, current_user: CurrentUser, id: int):
+async def pwdReset(session: SessionDep, current_user: CurrentUser, trans: Trans, id: int):
     if not current_user.isAdmin:
-        raise HTTPException('only for admin')
+        raise Exception(trans('i18n_permission.no_permission', url = " patch[/user/pwd/id],", msg = trans('i18n_permission.only_admin')))
     db_user: UserModel = get_db_user(session=session, user_id=id)
     db_user.password = default_md5_pwd()
     session.add(db_user)
     session.commit()
 
 @router.put("/pwd")
 @clear_cache(namespace=CacheNamespace.AUTH_INFO, cacheName=CacheName.USER_INFO, keyExpression="current_user.id")
-async def pwdUpdate(session: SessionDep, current_user: CurrentUser, editor: PwdEditor):
+async def pwdUpdate(session: SessionDep, current_user: CurrentUser, trans: Trans, editor: PwdEditor):
+    new_pwd = editor.new_pwd
+    if not check_pwd_format(new_pwd):
+        raise Exception(trans('i18n_format_invalid', key = trans('i18n_user.password')))
     db_user: UserModel = get_db_user(session=session, user_id=current_user.id)
     if not verify_md5pwd(editor.pwd, db_user.password):
-        raise HTTPException("pwd error")
-    db_user.password = md5pwd(editor.new_pwd)
+        raise Exception(trans('i18n_error', key = trans('i18n_user.password')))
+    db_user.password = md5pwd(new_pwd)
+    session.add(db_user)
+    session.commit()
+    
+@router.patch("/status")
+@clear_cache(namespace=CacheNamespace.AUTH_INFO, cacheName=CacheName.USER_INFO, keyExpression="statusDto.id")
+async def langChange(session: SessionDep, current_user: CurrentUser, trans: Trans, statusDto: UserStatus):
+    if not current_user.isAdmin:
+        raise Exception(trans('i18n_permission.no_permission', url = ", ", msg = trans('i18n_permission.only_admin')))
+    status = statusDto.status
+    if status not in [0, 1]:
+        return {"message": "status not supported"}
+    db_user: UserModel = get_db_user(session=session, user_id=statusDto.id)
+    db_user.status = status
     session.add(db_user)
     session.commit()