Merge pull request #1250 from yixinmeng/feat/persist-ragflow-dataset-id

feat: persist ragflow dataset id and route by id

Author: lyj715824

console/backend/commons/src/main/java/com/iflytek/astron/console/commons/constant/ResponseEnum.java

@@ -395,6 +395,7 @@ public enum ResponseEnum {
     REPO_KNOWLEDGE_QUERY_FAILED(8737, "repo.knowledge.query.failed"),
     REPO_DELETE_FAILED_BOT_USED(8738, "repo.delete.failed.bot.used"),
     REPO_FILE_UPLOAD_TYPE_NOT_EXIST(8739, "repo.file.upload.type.not.exist"),
+    REPO_CREATE_RAGFLOW_FAILED(8740, "repo.create.ragflow.failed"),
 
     // 8900 - 9000 (Model related)
     MODEL_NOT_COMPATIBLE_OPENAI(8900, "model.not.compatible.openai"),

console/backend/commons/src/main/resources/messages_en.properties

@@ -376,6 +376,7 @@ repo.file.disabled=Document disabled
 repo.knowledge.query.failed=Knowledge retrieval failed
 repo.delete.failed.bot.used=Knowledge base has bot association usage, cannot delete
 repo.file.upload.type.not.exist=Upload failed: File type not supported
+repo.create.ragflow.failed=Failed to create RAGFlow dataset: {0}
 
 # Model 8900+
 model.not.compatible.openai=Interface return format not compatible with OpenAI protocol

console/backend/commons/src/main/resources/messages_zh.properties

@@ -388,6 +388,7 @@ repo.file.disabled=文档已停用
 repo.knowledge.query.failed=知识检索失败
 repo.delete.failed.bot.used=知识库存在bot关联使用，不能删除
 repo.file.upload.type.not.exist=上传失败：文件类型不支持
+repo.create.ragflow.failed=RAGFlow 知识库创建失败：{0}
 
 # 模型 8900+
 model.not.compatible.openai=接口返回格式不兼容 OpenAI 协议

console/backend/hub/src/main/resources/db/migration/V1.28__add_ragflow_dataset_id_to_repo.sql

@@ -0,0 +1,5 @@
+ALTER TABLE repo
+    ADD COLUMN ragflow_dataset_id VARCHAR(64) NULL
+    COMMENT 'RAGFlow dataset.id for Ragflow-RAG repos; NULL uses the default dataset';
+
+CREATE INDEX idx_repo_ragflow_dataset_id ON repo (ragflow_dataset_id);

console/backend/toolkit/src/main/java/com/iflytek/astron/console/toolkit/entity/core/knowledge/DatasetCreateRequest.java

@@ -0,0 +1,16 @@
+package com.iflytek.astron.console.toolkit.entity.core.knowledge;
+
+import lombok.AllArgsConstructor;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+
+/** Request body for POST /v1/dataset/create. */
+@Data
+@NoArgsConstructor
+@AllArgsConstructor
+public class DatasetCreateRequest {
+
+    String name;
+
+    String description;
+}

console/backend/toolkit/src/main/java/com/iflytek/astron/console/toolkit/entity/core/knowledge/KnowledgeRequest.java

@@ -17,6 +17,9 @@ public class KnowledgeRequest {
      */
     String group;
 
+    /** RAGFlow dataset.id for Ragflow-RAG routing; blank uses the default dataset. */
+    String datasetId;
+
     /**
      * Required: No. User ID to which the document belongs, users can specify themselves
      */

console/backend/toolkit/src/main/java/com/iflytek/astron/console/toolkit/entity/core/knowledge/QueryMatchObj.java

@@ -16,6 +16,9 @@ public class QueryMatchObj {
      */
     List<String> repoId;
 
+    /** RAGFlow dataset.id values for Ragflow-RAG routing. */
+    List<String> datasetId;
+
     /**
      * Required: No. Knowledge base score threshold, default 0
      */

console/backend/toolkit/src/main/java/com/iflytek/astron/console/toolkit/entity/core/knowledge/SplitRequest.java

@@ -50,17 +50,8 @@ public class SplitRequest {
      */
     Integer resourceType;
 
-    /**
-     * RAGFlow dataset group (coreRepoId for Ragflow-RAG). Optional; null falls back to the default
-     * group.
-     */
-    String group;
-
-    /**
-     * Human-readable label written into the RAGFlow dataset description on first creation. Helps
-     * operators identify the dataset in the RAGFlow UI without resolving UUIDs.
-     */
-    String groupDescription;
+    /** RAGFlow dataset.id for Ragflow-RAG routing; blank uses the default dataset. */
+    String datasetId;
 
     // Default to AIUI value
     public SplitRequest() {

console/backend/toolkit/src/main/java/com/iflytek/astron/console/toolkit/entity/table/repo/Repo.java

@@ -121,6 +121,9 @@ public class Repo implements Serializable {
     // Knowledge base type, CBG-RAG / AIUI-RAG2
     private String tag;
 
+    /** RAGFlow dataset.id for Ragflow-RAG repos; null uses the default dataset. */
+    private String ragflowDatasetId;
+
     private Long spaceId;
 
 }

console/backend/toolkit/src/main/java/com/iflytek/astron/console/toolkit/handler/KnowledgeV2ServiceCallHandler.java

@@ -1,6 +1,9 @@
 package com.iflytek.astron.console.toolkit.handler;
 
 import com.alibaba.fastjson2.JSON;
+import com.alibaba.fastjson2.JSONObject;
+import com.iflytek.astron.console.commons.constant.ResponseEnum;
+import com.iflytek.astron.console.commons.exception.BusinessException;
 import com.iflytek.astron.console.toolkit.common.constant.ProjectContent;
 import com.iflytek.astron.console.toolkit.config.properties.RepoAuthorizedConfig;
 import com.iflytek.astron.console.toolkit.config.properties.ApiUrl;
@@ -25,25 +28,63 @@ public class KnowledgeV2ServiceCallHandler {
     @Resource
     private RepoAuthorizedConfig repoAuthorizedConfig;
 
+    private static final String DATASET_ID_FIELD = "datasetId";
+
+    /**
+     * Create or reuse a RAGFlow dataset and return its id.
+     *
+     * @param name RAGFlow dataset.name
+     * @param description RAGFlow dataset.description; nullable
+     */
+    public String createRagflowDataset(String name, String description) {
+        String url = apiUrl.getKnowledgeUrl().concat("/v1/dataset/create");
+        DatasetCreateRequest req = new DatasetCreateRequest(name, description);
+        String reqBody = JSON.toJSONString(req);
+        log.info("createRagflowDataset url = {}, name = {}", url, name);
+        String resp = postJson(url, reqBody);
+        log.info("createRagflowDataset response = {}", resp);
+        KnowledgeResponse parsed = JSON.parseObject(resp, KnowledgeResponse.class);
+        if (parsed == null || parsed.getCode() == null || parsed.getCode() != 0) {
+            String msg = (parsed == null) ? "blank response" : parsed.getMessage();
+            throw new BusinessException(ResponseEnum.REPO_CREATE_RAGFLOW_FAILED, msg);
+        }
+        return extractDatasetId(parsed.getData());
+    }
+
+    private String extractDatasetId(Object data) {
+        JSONObject dataObj = toJsonObject(data);
+        String datasetId = dataObj == null ? null : dataObj.getString(DATASET_ID_FIELD);
+        if (StringUtils.isBlank(datasetId)) {
+            throw new BusinessException(ResponseEnum.REPO_CREATE_RAGFLOW_FAILED,
+                    "RAGFlow returned blank datasetId");
+        }
+        return datasetId;
+    }
+
+    private JSONObject toJsonObject(Object data) {
+        if (data instanceof JSONObject) {
+            return (JSONObject) data;
+        }
+        if (data instanceof String) {
+            return JSON.parseObject((String) data);
+        }
+        throw new BusinessException(ResponseEnum.REPO_CREATE_RAGFLOW_FAILED,
+                "RAGFlow returned non-object data");
+    }
+
     /**
      * Document parsing and chunking
      *
      * @param request the split request describing the document
-     * @param coreRepoId Ragflow-RAG dataset name/group; forwarded as {@code group} so the upstream
-     *        service can resolve the matching dataset. Ignored for non-Ragflow-RAG sources to keep
-     *        CBG/AIUI/Spark behavior intact.
-     * @param repoName human-readable repo display name; written into RAGFlow dataset description on
-     *        first lazy creation. Pass {@code null} to skip.
+     * @param datasetId RAGFlow dataset.id; ignored when blank or non-Ragflow
      * @return knowledge response from the upstream split API
      */
-    public KnowledgeResponse documentSplit(
-            SplitRequest request, String coreRepoId, String repoName) {
-        applyGroupToSplitRequest(request, coreRepoId);
-        applyGroupDescriptionToSplitRequest(request, repoName);
+    public KnowledgeResponse documentSplit(SplitRequest request, String datasetId) {
+        applyDatasetIdToSplitRequest(request, datasetId);
         String url = apiUrl.getKnowledgeUrl().concat("/v1/document/split");
         String reqBody = JSON.toJSONString(request);
         log.info("documentSplit url = {}, request = {}", url, reqBody);
-        String post = OkHttpUtil.post(url, reqBody);
+        String post = postJson(url, reqBody);
         log.info("documentSplit response = {}", post);
         return JSON.parseObject(post, KnowledgeResponse.class);
     }
@@ -57,19 +98,14 @@ public KnowledgeResponse documentSplit(
      * @param ragType RAG type
      * @param resourceType resource type (0=file, 1=html)
      * @param oldDocId existing RAGFlow doc id for upsert; null for first slice
-     * @param coreRepoId Ragflow-RAG dataset name/group; forwarded as {@code group} so the upstream
-     *        service can resolve the matching dataset. Ignored for non-Ragflow-RAG sources to keep
-     *        CBG/AIUI/Spark behavior intact.
-     * @param repoName human-readable repo display name; written into RAGFlow dataset description on
-     *        first lazy creation. Pass {@code null} to skip.
+     * @param datasetId RAGFlow dataset.id; ignored when blank or non-Ragflow
      * @return KnowledgeResponse
      */
     public KnowledgeResponse documentUpload(MultipartFile multipartFile,
             List<Integer> lengthRange, List<String> separator,
             String ragType, Integer resourceType,
             String oldDocId,
-            String coreRepoId,
-            String repoName) {
+            String datasetId) {
         String url = apiUrl.getKnowledgeUrl().concat("/v1/document/upload");
 
         try {
@@ -91,11 +127,10 @@ public KnowledgeResponse documentUpload(MultipartFile multipartFile,
             if (StringUtils.isNotBlank(oldDocId)) {
                 params.put("documentId", oldDocId);
             }
-            applyGroupToUploadParams(params, ragType, coreRepoId);
-            applyGroupDescriptionToUploadParams(params, ragType, repoName);
+            applyDatasetIdToUploadParams(params, ragType, datasetId);
 
             log.info("documentUpload url = {}, ragType = {}, resourceType = {}", url, ragType, resourceType);
-            String post = OkHttpUtil.postMultipart(url, new HashMap<>(), null, params, null);
+            String post = OkHttpUtil.postMultipart(url, null, null, params, null);
             log.info("documentUpload response = {}", post);
             return JSON.parseObject(post, KnowledgeResponse.class);
         } catch (Exception e) {
@@ -107,68 +142,32 @@ public KnowledgeResponse documentUpload(MultipartFile multipartFile,
         }
     }
 
-    /**
-     * Set {@code group} on the split request for Ragflow-RAG when {@code coreRepoId} is non-blank.
-     * No-op otherwise so other RAG strategies stay untouched.
-     */
-    void applyGroupToSplitRequest(SplitRequest request, String coreRepoId) {
-        if (request == null) {
-            return;
-        }
-        if (ProjectContent.FILE_SOURCE_RAG_FLOW_RAG_STR.equals(request.getRagType())
-                && StringUtils.isNotBlank(coreRepoId)) {
-            request.setGroup(coreRepoId);
-        }
-    }
-
-    /**
-     * Add {@code group} to the upload params map for Ragflow-RAG when {@code coreRepoId} is non-blank.
-     * No-op otherwise.
-     */
-    void applyGroupToUploadParams(Map<String, Object> params, String ragType, String coreRepoId) {
-        if (params == null) {
-            return;
-        }
-        if (ProjectContent.FILE_SOURCE_RAG_FLOW_RAG_STR.equals(ragType)
-                && StringUtils.isNotBlank(coreRepoId)) {
-            params.put("group", coreRepoId);
-        }
-    }
-
-    /**
-     * Set {@code groupDescription} on the split request for Ragflow-RAG when {@code repoName} is
-     * non-blank. No-op otherwise.
-     */
-    void applyGroupDescriptionToSplitRequest(SplitRequest request, String repoName) {
+    void applyDatasetIdToSplitRequest(SplitRequest request, String datasetId) {
         if (request == null) {
             return;
         }
         if (ProjectContent.FILE_SOURCE_RAG_FLOW_RAG_STR.equals(request.getRagType())
-                && StringUtils.isNotBlank(repoName)) {
-            request.setGroupDescription(repoName);
+                && StringUtils.isNotBlank(datasetId)) {
+            request.setDatasetId(datasetId);
         }
     }
 
-    /**
-     * Add {@code groupDescription} to the upload params map for Ragflow-RAG when {@code repoName} is
-     * non-blank. No-op otherwise.
-     */
-    void applyGroupDescriptionToUploadParams(
-            Map<String, Object> params, String ragType, String repoName) {
+    void applyDatasetIdToUploadParams(
+            Map<String, Object> params, String ragType, String datasetId) {
         if (params == null) {
             return;
         }
         if (ProjectContent.FILE_SOURCE_RAG_FLOW_RAG_STR.equals(ragType)
-                && StringUtils.isNotBlank(repoName)) {
-            params.put("groupDescription", repoName);
+                && StringUtils.isNotBlank(datasetId)) {
+            params.put(DATASET_ID_FIELD, datasetId);
         }
     }
 
     public KnowledgeResponse saveChunk(KnowledgeRequest request) {
         String url = apiUrl.getKnowledgeUrl().concat("/v1/chunks/save");
         String reqBody = JSON.toJSONString(request);
         log.info("saveChunk url = {}, request = {}", url, reqBody);
-        String post = OkHttpUtil.post(url, reqBody);
+        String post = postJson(url, reqBody);
         log.info("saveChunk response = {}", post);
         return JSON.parseObject(post, KnowledgeResponse.class);
     }
@@ -177,7 +176,7 @@ public KnowledgeResponse updateChunk(KnowledgeRequest request) {
         String url = apiUrl.getKnowledgeUrl().concat("/v1/chunk/update");
         String reqBody = JSON.toJSONString(request);
         log.info("updateChunk url = {}, request = {}", url, reqBody);
-        String post = OkHttpUtil.post(url, reqBody);
+        String post = postJson(url, reqBody);
         log.info("updateChunk response = {}", post);
         return JSON.parseObject(post, KnowledgeResponse.class);
     }
@@ -186,7 +185,7 @@ public KnowledgeResponse deleteDocOrChunk(KnowledgeRequest request) {
         String url = apiUrl.getKnowledgeUrl().concat("/v1/chunk/delete");
         String reqBody = JSON.toJSONString(request);
         log.info("deleteDocOrChunk url = {}, request = {}", url, reqBody);
-        String post = OkHttpUtil.post(url, reqBody);
+        String post = postJson(url, reqBody);
         log.info("deleteDocOrChunk response = {}", post);
         return JSON.parseObject(post, KnowledgeResponse.class);
     }
@@ -195,8 +194,12 @@ public KnowledgeResponse knowledgeQuery(QueryRequest request) {
         String url = apiUrl.getKnowledgeUrl().concat("/v1/chunk/query");
         String reqBody = JSON.toJSONString(request);
         log.info("knowledgeQuery request url:{}\ndata:{}", url, reqBody);
-        String respData = OkHttpUtil.post(url, reqBody);
+        String respData = postJson(url, reqBody);
         log.info("knowledgeQuery response data:{}", respData);
         return JSON.parseObject(respData, KnowledgeResponse.class);
     }
+
+    private String postJson(String url, String reqBody) {
+        return OkHttpUtil.post(url, reqBody);
+    }
 }