Skip to content

Commit 04a233b

Browse files
Dallas98Dallas98
authored
fix: 修复知识库问题 (#89)
* feat: Refactor system parameter management with new data structure and update logic * feat: Enhance dataset file management with improved file copying * feat: Enhance dataset file management with improved file copying * fix: 修复知识库相关问题 * feat: Integrate Milvus service for enhanced knowledge base management and file deletion
1 parent 145c154 commit 04a233b

File tree

4 files changed

+69
-26
lines changed

4 files changed

+69
-26
lines changed

‎backend/services/rag-indexer-service/src/main/java/com/datamate/rag/indexer/application/KnowledgeBaseService.java‎

Lines changed: 19 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,10 @@
22

33
import com.baomidou.mybatisplus.core.metadata.IPage;
44
import com.baomidou.mybatisplus.extension.plugins.pagination.Page;
5+
import com.datamate.common.infrastructure.exception.BusinessException;
6+
import com.datamate.common.infrastructure.exception.KnowledgeBaseErrorCode;
7+
import com.datamate.common.interfaces.PagedResponse;
8+
import com.datamate.common.interfaces.PagingQuery;
59
import com.datamate.common.setting.domain.repository.ModelConfigRepository;
610
import com.datamate.rag.indexer.domain.model.FileStatus;
711
import com.datamate.rag.indexer.domain.model.KnowledgeBase;
@@ -10,11 +14,10 @@
1014
import com.datamate.rag.indexer.domain.repository.KnowledgeBaseRepository;
1115
import com.datamate.rag.indexer.domain.repository.RagFileRepository;
1216
import com.datamate.rag.indexer.infrastructure.event.DataInsertedEvent;
13-
import com.datamate.common.infrastructure.exception.BusinessException;
14-
import com.datamate.common.infrastructure.exception.KnowledgeBaseErrorCode;
15-
import com.datamate.common.interfaces.PagedResponse;
16-
import com.datamate.common.interfaces.PagingQuery;
1717
import com.datamate.rag.indexer.interfaces.dto.*;
18+
import io.milvus.client.MilvusClient;
19+
import io.milvus.param.collection.DropCollectionParam;
20+
import io.milvus.param.dml.DeleteParam;
1821
import lombok.RequiredArgsConstructor;
1922
import org.jetbrains.annotations.NotNull;
2023
import org.springframework.beans.BeanUtils;
@@ -39,7 +42,7 @@ public class KnowledgeBaseService {
3942
private final RagFileRepository ragFileRepository;
4043
private final ApplicationEventPublisher eventPublisher;
4144
private final ModelConfigRepository modelConfigRepository;
42-
45+
private final MilvusClient milvusClient;
4346

4447
/**
4548
* 创建知识库
@@ -72,10 +75,13 @@ public void update(String knowledgeBaseId, KnowledgeBaseUpdateReq request) {
7275
knowledgeBaseRepository.updateById(knowledgeBase);
7376
}
7477

78+
@Transactional
7579
public void delete(String knowledgeBaseId) {
80+
KnowledgeBase knowledgeBase = Optional.ofNullable(knowledgeBaseRepository.getById(knowledgeBaseId))
81+
.orElseThrow(() -> BusinessException.of(KnowledgeBaseErrorCode.KNOWLEDGE_BASE_NOT_FOUND));
7682
knowledgeBaseRepository.removeById(knowledgeBaseId);
7783
ragFileRepository.removeByKnowledgeBaseId(knowledgeBaseId);
78-
// TODO: 删除知识库关联的所有文档
84+
milvusClient.dropCollection(DropCollectionParam.newBuilder().withCollectionName(knowledgeBase.getName()).build());
7985
}
8086

8187
public KnowledgeBaseResp getById(String knowledgeBaseId) {
@@ -136,8 +142,15 @@ public PagedResponse<RagFile> listFiles(String knowledgeBaseId, RagFileReq reque
136142
return PagedResponse.of(page.getRecords(), page.getCurrent(), page.getTotal(), page.getPages());
137143
}
138144

145+
@Transactional(rollbackFor = Exception.class)
139146
public void deleteFiles(String knowledgeBaseId, DeleteFilesReq request) {
147+
KnowledgeBase knowledgeBase = Optional.ofNullable(knowledgeBaseRepository.getById(knowledgeBaseId))
148+
.orElseThrow(() -> BusinessException.of(KnowledgeBaseErrorCode.KNOWLEDGE_BASE_NOT_FOUND));
140149
ragFileRepository.removeByIds(request.getIds());
150+
milvusClient.delete(DeleteParam.newBuilder()
151+
.withCollectionName(knowledgeBase.getName())
152+
.withExpr("metadata[\"rag_file_id\"] in [" + org.apache.commons.lang3.StringUtils.join(request.getIds().stream().map(id -> "\"" + id + "\"").toArray(), ",") + "]")
153+
.build());
141154
}
142155

143156
public PagedResponse<RagChunk> getChunks(String knowledgeBaseId, String ragFileId, PagingQuery pagingQuery) {

‎backend/services/rag-indexer-service/src/main/java/com/datamate/rag/indexer/infrastructure/event/RagEtlService.java‎

Lines changed: 4 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@
88
import com.datamate.rag.indexer.domain.model.FileStatus;
99
import com.datamate.rag.indexer.domain.model.RagFile;
1010
import com.datamate.rag.indexer.domain.repository.RagFileRepository;
11+
import com.datamate.rag.indexer.infrastructure.milvus.MilvusService;
1112
import com.datamate.rag.indexer.interfaces.dto.ProcessType;
1213
import com.google.common.collect.Lists;
1314
import dev.langchain4j.data.document.Document;
@@ -27,11 +28,8 @@
2728
import dev.langchain4j.data.embedding.Embedding;
2829
import dev.langchain4j.data.segment.TextSegment;
2930
import dev.langchain4j.model.embedding.EmbeddingModel;
30-
import dev.langchain4j.store.embedding.EmbeddingStore;
31-
import dev.langchain4j.store.embedding.milvus.MilvusEmbeddingStore;
3231
import lombok.RequiredArgsConstructor;
3332
import lombok.extern.slf4j.Slf4j;
34-
import org.springframework.beans.factory.annotation.Value;
3533
import org.springframework.scheduling.annotation.Async;
3634
import org.springframework.stereotype.Service;
3735
import org.springframework.transaction.event.TransactionPhase;
@@ -55,10 +53,7 @@
5553
public class RagEtlService {
5654
private static final Semaphore SEMAPHORE = new Semaphore(10);
5755

58-
@Value("${datamate.rag.milvus-host:milvus-standalone}")
59-
private String milvusHost;
60-
@Value("${datamate.rag.milvus-port:19530}")
61-
private int milvusPort;
56+
private final MilvusService milvusService;
6257

6358
private final RagFileRepository ragFileRepository;
6459

@@ -112,7 +107,7 @@ private void processRagFile(RagFile ragFile, DataInsertedEvent event) {
112107
if (Arrays.asList("html", "htm").contains(file.getFileType().toLowerCase())) {
113108
document = new HtmlToTextDocumentTransformer().transform(document);
114109
}
115-
document.metadata().put("fileId", ragFile.getFileId());
110+
document.metadata().put("rag_file_id", ragFile.getId());
116111
// 使用文档分块器对文档进行分块
117112
DocumentSplitter splitter = documentSplitter(event.addFilesReq().getProcessType());
118113
List<TextSegment> split = splitter.split(document);
@@ -129,7 +124,7 @@ private void processRagFile(RagFile ragFile, DataInsertedEvent event) {
129124
Lists.partition(split, 20).forEach(partition -> {
130125
List<Embedding> content = embeddingModel.embedAll(partition).content();
131126
// 存储嵌入向量到 Milvus
132-
embeddingStore(embeddingModel, event.knowledgeBase().getName()).addAll(content, partition);
127+
milvusService.embeddingStore(embeddingModel, event.knowledgeBase().getName()).addAll(content, partition);
133128
});
134129
}
135130

@@ -159,13 +154,4 @@ public DocumentSplitter documentSplitter(ProcessType processType) {
159154
case DEFAULT_CHUNK -> new DocumentByLineSplitter(1000, 100);
160155
};
161156
}
162-
163-
public EmbeddingStore<TextSegment> embeddingStore(EmbeddingModel embeddingModel, String knowledgeBaseName) {
164-
return MilvusEmbeddingStore.builder()
165-
.host(milvusHost)
166-
.port(milvusPort)
167-
.collectionName(knowledgeBaseName)
168-
.dimension(embeddingModel.dimension())
169-
.build();
170-
}
171157
}

‎backend/services/rag-indexer-service/src/main/java/com/datamate/rag/indexer/infrastructure/milvus/MilvusService.java‎

Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,44 @@
1+
package com.datamate.rag.indexer.infrastructure.milvus;
2+
3+
import dev.langchain4j.data.segment.TextSegment;
4+
import dev.langchain4j.model.embedding.EmbeddingModel;
5+
import dev.langchain4j.store.embedding.EmbeddingStore;
6+
import dev.langchain4j.store.embedding.milvus.MilvusEmbeddingStore;
7+
import io.milvus.client.MilvusClient;
8+
import io.milvus.client.MilvusServiceClient;
9+
import io.milvus.param.ConnectParam;
10+
import org.springframework.beans.factory.annotation.Value;
11+
import org.springframework.context.annotation.Bean;
12+
import org.springframework.stereotype.Component;
13+
14+
/**
15+
* Milvus 服务类
16+
*
17+
* @author dallas
18+
* @since 2025-11-17
19+
*/
20+
@Component
21+
public class MilvusService {
22+
@Value("${datamate.rag.milvus-host:milvus-standalone}")
23+
private String milvusHost;
24+
@Value("${datamate.rag.milvus-port:19530}")
25+
private int milvusPort;
26+
27+
public EmbeddingStore<TextSegment> embeddingStore(EmbeddingModel embeddingModel, String knowledgeBaseName) {
28+
return MilvusEmbeddingStore.builder()
29+
.host(milvusHost)
30+
.port(milvusPort)
31+
.collectionName(knowledgeBaseName)
32+
.dimension(embeddingModel.dimension())
33+
.build();
34+
}
35+
36+
@Bean
37+
public MilvusClient milvusClient() {
38+
ConnectParam connectParam = ConnectParam.newBuilder()
39+
.withHost(milvusHost)
40+
.withPort(milvusPort)
41+
.build();
42+
return new MilvusServiceClient(connectParam);
43+
}
44+
}

‎frontend/src/pages/KnowledgeBase/Detail/KnowledgeBaseDetail.tsx‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -48,7 +48,7 @@ const KnowledgeBaseDetailPage: React.FC = () => {
4848
setSearchParams,
4949
handleFiltersChange,
5050
} = useFetchData<KBFile>(
51-
(params) => queryKnowledgeBaseFilesUsingGet(knowledgeBase?.id, params),
51+
(params) => id ? queryKnowledgeBaseFilesUsingGet(id, params) : Promise.resolve({ data: [] }),
5252
mapFileData
5353
);
5454

@@ -237,4 +237,4 @@ const KnowledgeBaseDetailPage: React.FC = () => {
237237
);
238238
};
239239

240-
export default KnowledgeBaseDetailPage;
240+
export default KnowledgeBaseDetailPage;

0 commit comments

Comments
 (0)