feat: Datasource table structure supports field retrieval #447

Author: XiaJunjie2020

backend/apps/datasource/api/datasource.py

@@ -20,7 +20,7 @@
     check_status_by_id
 from ..crud.field import get_fields_by_table_id
 from ..crud.table import get_tables_by_ds_id
-from ..models.datasource import CoreDatasource, CreateDatasource, TableObj, CoreTable, CoreField
+from ..models.datasource import CoreDatasource, CreateDatasource, TableObj, CoreTable, CoreField, FieldObj
 
 router = APIRouter(tags=["datasource"], prefix="/datasource")
 path = settings.EXCEL_PATH
@@ -164,8 +164,8 @@ async def table_list(session: SessionDep, id: int):
 
 
 @router.post("/fieldList/{id}")
-async def field_list(session: SessionDep, id: int):
-    return get_fields_by_table_id(session, id)
+async def field_list(session: SessionDep, id: int, field: FieldObj):
+    return get_fields_by_table_id(session, id, field)
 
 
 @router.post("/editLocalComment")

backend/apps/datasource/crud/datasource.py

@@ -269,11 +269,17 @@ def preview(session: SessionDep, current_user: CurrentUser, id: int, data: Table
     ds = session.query(CoreDatasource).filter(CoreDatasource.id == id).first()
     # check_status(session, ds, True)
 
-    if data.fields is None or len(data.fields) == 0:
+    # ignore data's fields param, query fields from database
+    if not data.table.id:
+        return {"fields": [], "data": [], "sql": ''}
+
+    fields = session.query(CoreField).filter(CoreField.table_id == data.table.id).all()
+
+    if fields is None or len(fields) == 0:
         return {"fields": [], "data": [], "sql": ''}
 
     where = ''
-    f_list = [f for f in data.fields if f.checked]
+    f_list = [f for f in fields if f.checked]
     if is_normal_user(current_user):
         # column is checked, and, column permission for data.fields
         contain_rules = session.query(DsRules).all()

backend/apps/datasource/crud/field.py

@@ -1,14 +1,22 @@
 from common.core.deps import SessionDep
-from ..models.datasource import CoreField
+from ..models.datasource import CoreField, FieldObj
+from sqlalchemy import or_, and_
 
 
 def delete_field_by_ds_id(session: SessionDep, id: int):
     session.query(CoreField).filter(CoreField.ds_id == id).delete(synchronize_session=False)
     session.commit()
 
 
-def get_fields_by_table_id(session: SessionDep, id: int):
-    return session.query(CoreField).filter(CoreField.table_id == id).order_by(CoreField.field_index.asc()).all()
+def get_fields_by_table_id(session: SessionDep, id: int, field: FieldObj):
+    if field and field.fieldName:
+        return session.query(CoreField).filter(
+            and_(CoreField.table_id == id, or_(CoreField.field_name.like(f'%{field.fieldName}%'),
+                                               CoreField.field_name.like(f'%{field.fieldName.lower()}%'),
+                                               CoreField.field_name.like(f'%{field.fieldName.upper()}%')))).order_by(
+            CoreField.field_index.asc()).all()
+    else:
+        return session.query(CoreField).filter(CoreField.table_id == id).order_by(CoreField.field_index.asc()).all()
 
 
 def update_field(session: SessionDep, item: CoreField):

backend/apps/datasource/models/datasource.py

@@ -35,6 +35,7 @@ class CoreTable(SQLModel, table=True):
     custom_comment: str = Field(sa_column=Column(Text))
     embedding: str = Field(sa_column=Column(Text, nullable=True))
 
+
 class DsRecommendedProblem(SQLModel, table=True):
     __tablename__ = "ds_recommended_problem"
     id: int = Field(sa_column=Column(BigInteger, Identity(always=True), nullable=False, primary_key=True))
@@ -84,6 +85,7 @@ def __init__(self, datasource_id,recommended_config,questions):
     questions: str = None
 
 
+
 class RecommendedProblemBase(BaseModel):
     datasource_id: int = None
     recommended_config: int = None
@@ -164,3 +166,7 @@ def __init__(self, schema, table, fields):
     schema: str
     table: CoreTable
     fields: List[CoreField]
+
+
+class FieldObj(BaseModel):
+    fieldName: str | None