Skip to content

Commit e4d4c64

Browse files
RonnyChan96RonnyChan96
committed
[app-platform] Add open source knowledge repo impl.
1 parent 6b64683 commit e4d4c64

File tree

14 files changed

+987
-0
lines changed

14 files changed

+987
-0
lines changed

‎app-builder/plugins/data-mate-knowledge/pom.xml‎

Lines changed: 142 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,142 @@
1+
<?xml version="1.0" encoding="UTF-8"?>
2+
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
3+
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
4+
<modelVersion>4.0.0</modelVersion>
5+
<parent>
6+
<groupId>modelengine.fit.jade</groupId>
7+
<artifactId>app-builder-plugin-parent</artifactId>
8+
<version>1.0.0-SNAPSHOT</version>
9+
</parent>
10+
11+
<groupId>modelengine.jade.plugin</groupId>
12+
<artifactId>data-mate-knowledge</artifactId>
13+
14+
<properties>
15+
<maven.compiler.source>17</maven.compiler.source>
16+
<maven.compiler.target>17</maven.compiler.target>
17+
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
18+
</properties>
19+
20+
<dependencies>
21+
<!-- FIT -->
22+
<dependency>
23+
<groupId>org.fitframework</groupId>
24+
<artifactId>fit-api</artifactId>
25+
</dependency>
26+
27+
<!-- Service -->
28+
<dependency>
29+
<groupId>modelengine.fit.jade.service</groupId>
30+
<artifactId>knowledge-service</artifactId>
31+
</dependency>
32+
33+
<!-- lombok -->
34+
<dependency>
35+
<groupId>org.projectlombok</groupId>
36+
<artifactId>lombok</artifactId>
37+
</dependency>
38+
39+
<!-- Mapstruct -->
40+
<dependency>
41+
<groupId>org.mapstruct</groupId>
42+
<artifactId>mapstruct</artifactId>
43+
</dependency>
44+
<dependency>
45+
<groupId>org.mapstruct</groupId>
46+
<artifactId>mapstruct-processor</artifactId>
47+
</dependency>
48+
49+
<!-- Jackson -->
50+
<dependency>
51+
<groupId>com.fasterxml.jackson.core</groupId>
52+
<artifactId>jackson-databind</artifactId>
53+
</dependency>
54+
55+
<!-- Test -->
56+
<dependency>
57+
<groupId>org.fitframework</groupId>
58+
<artifactId>fit-test-framework</artifactId>
59+
</dependency>
60+
<dependency>
61+
<groupId>org.junit.jupiter</groupId>
62+
<artifactId>junit-jupiter</artifactId>
63+
</dependency>
64+
<dependency>
65+
<groupId>org.mockito</groupId>
66+
<artifactId>mockito-core</artifactId>
67+
</dependency>
68+
<dependency>
69+
<groupId>org.assertj</groupId>
70+
<artifactId>assertj-core</artifactId>
71+
</dependency>
72+
</dependencies>
73+
74+
<build>
75+
<plugins>
76+
<plugin>
77+
<groupId>org.fitframework</groupId>
78+
<artifactId>fit-build-maven-plugin</artifactId>
79+
<version>${fit.version}</version>
80+
<executions>
81+
<execution>
82+
<id>build-plugin</id>
83+
<goals>
84+
<goal>build-plugin</goal>
85+
</goals>
86+
</execution>
87+
<execution>
88+
<id>package-plugin</id>
89+
<goals>
90+
<goal>package-plugin</goal>
91+
</goals>
92+
</execution>
93+
</executions>
94+
</plugin>
95+
<plugin>
96+
<groupId>org.fitframework</groupId>
97+
<artifactId>fit-dependency-maven-plugin</artifactId>
98+
<version>${fit.version}</version>
99+
<executions>
100+
<execution>
101+
<id>dependency</id>
102+
<phase>compile</phase>
103+
<goals>
104+
<goal>dependency</goal>
105+
</goals>
106+
</execution>
107+
</executions>
108+
</plugin>
109+
<plugin>
110+
<groupId>org.apache.maven.plugins</groupId>
111+
<artifactId>maven-jar-plugin</artifactId>
112+
<version>${maven.jar.version}</version>
113+
<configuration>
114+
<archive>
115+
<manifestEntries>
116+
<Created-By>FIT Lab</Created-By>
117+
</manifestEntries>
118+
</archive>
119+
</configuration>
120+
</plugin>
121+
<plugin>
122+
<groupId>org.apache.maven.plugins</groupId>
123+
<artifactId>maven-antrun-plugin</artifactId>
124+
<version>${maven.antrun.version}</version>
125+
<executions>
126+
<execution>
127+
<phase>install</phase>
128+
<configuration>
129+
<target>
130+
<copy file="${project.build.directory}/${project.build.finalName}.jar"
131+
todir="../../../build/plugins"/>
132+
</target>
133+
</configuration>
134+
<goals>
135+
<goal>run</goal>
136+
</goals>
137+
</execution>
138+
</executions>
139+
</plugin>
140+
</plugins>
141+
</build>
142+
</project>

‎app-builder/plugins/data-mate-knowledge/src/main/java/modelengine/fit/jade/datamate/knowledge/knowledge/convertor/ParamConvertor.java‎

Lines changed: 140 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,140 @@
1+
/*---------------------------------------------------------------------------------------------
2+
* Copyright (c) 2025 Huawei Technologies Co., Ltd. All rights reserved.
3+
* This file is a part of the ModelEngine Project.
4+
* Licensed under the MIT License. See License.txt in the project root for license information.
5+
*--------------------------------------------------------------------------------------------*/
6+
7+
package modelengine.fit.jade.datamate.knowledge.knowledge.convertor;
8+
9+
import com.fasterxml.jackson.core.JsonProcessingException;
10+
import com.fasterxml.jackson.databind.ObjectMapper;
11+
import modelengine.fitframework.util.StringUtils;
12+
import modelengine.jade.knowledge.KnowledgeRepo;
13+
import modelengine.jade.knowledge.ReferenceLimit;
14+
import modelengine.jade.knowledge.document.KnowledgeDocument;
15+
import modelengine.fit.jade.datamate.knowledge.knowledge.dto.DataMateRetrievalParam;
16+
import modelengine.fit.jade.datamate.knowledge.knowledge.entity.DataMateKnowledgeEntity;
17+
import modelengine.fit.jade.datamate.knowledge.knowledge.entity.DataMateRetrievalChunksEntity;
18+
import modelengine.jade.knowledge.support.FlatKnowledgeOption;
19+
20+
import org.mapstruct.Mapper;
21+
import org.mapstruct.Mapping;
22+
import org.mapstruct.Named;
23+
import org.mapstruct.factory.Mappers;
24+
25+
import java.time.LocalDateTime;
26+
import java.time.format.DateTimeFormatter;
27+
import java.util.HashMap;
28+
import java.util.Map;
29+
30+
/**
31+
* DataMate 内部数据的转换器接口。
32+
*
33+
* @author 陈镕希
34+
* @since 2025-12-15
35+
*/
36+
@Mapper
37+
public interface ParamConvertor {
38+
ParamConvertor INSTANCE = Mappers.getMapper(ParamConvertor.class);
39+
int TOP = 400;
40+
41+
/**
42+
* 将 {@link DataMateKnowledgeEntity} 转换为 {@link KnowledgeRepo}。
43+
*
44+
* @param entity 表示待转换的 {@link DataMateKnowledgeEntity}。
45+
* @return 转换完成的 {@link KnowledgeRepo}。
46+
*/
47+
@Mapping(target = "type", source = "entity", qualifiedByName = "mapIndexTypeToType")
48+
@Mapping(target = "createdAt", source = "entity", qualifiedByName = "stringToLocalDateTime")
49+
KnowledgeRepo convertToKnowledgeRepo(DataMateKnowledgeEntity entity);
50+
51+
/**
52+
* 将 DataMate 知识库的检索 type 映射为 平台知识库元数据 type。
53+
*
54+
* @param entity 表示待转换的 {@link DataMateKnowledgeEntity}。
55+
* @return 表示转换完成的 {@link String}。
56+
*/
57+
@Named("mapIndexTypeToType")
58+
default String mapIndexTypeToType(DataMateKnowledgeEntity entity) {
59+
return entity == null ? null : entity.getEmbeddingModel();
60+
}
61+
62+
/**
63+
* 将 DataMate 知识库的 createdAt 映射为 平台知识库元数据 createdAt。
64+
*
65+
* @param entity 表示待转换的 {@link DataMateKnowledgeEntity}。
66+
* @return 表示转换完成的 {@link LocalDateTime}。
67+
*/
68+
@Named("stringToLocalDateTime")
69+
default LocalDateTime stringToLocalDateTime(DataMateKnowledgeEntity entity) {
70+
String dateStr = entity.getCreatedAt();
71+
if (dateStr == null || StringUtils.isEmpty(dateStr)) {
72+
return null;
73+
}
74+
return LocalDateTime.parse(dateStr, DateTimeFormatter.ISO_DATE_TIME);
75+
}
76+
77+
/**
78+
* 将 {@link FlatKnowledgeOption} 转换为 {@link DataMateRetrievalParam}。
79+
*
80+
* @param option 表示待转换的 {@link FlatKnowledgeOption}。
81+
* @return 转换完成的 {@link DataMateRetrievalParam}。
82+
*/
83+
@Mapping(target = "knowledgeBaseIds", source = "repoIds")
84+
@Mapping(target = "query", source = "query")
85+
@Mapping(target = "topK", source = "referenceLimit", qualifiedByName = "mapReferenceLimitToTop")
86+
@Mapping(target = "threshold", source = "similarityThreshold")
87+
DataMateRetrievalParam convertToRetrievalParam(FlatKnowledgeOption option);
88+
89+
/**
90+
* 将平台检索请求 ReferenceLimit 映射为 DataMate 检索请求 top。
91+
*
92+
* @param limit 表示待转换的 {@link ReferenceLimit}。
93+
* @return 转换完成的 {@link int}。
94+
*/
95+
@Named("mapReferenceLimitToTop")
96+
default int mapReferenceLimitToTop(ReferenceLimit limit) {
97+
if (limit == null) {
98+
return TOP;
99+
}
100+
return limit.getValue();
101+
}
102+
103+
/**
104+
* 将 {@link DataMateRetrievalChunksEntity} 转换为 {@link KnowledgeDocument}。
105+
*
106+
* @param entity 表示待转换的 {@link DataMateRetrievalChunksEntity}。
107+
* @return 转换完成的 {@link KnowledgeDocument}。
108+
*/
109+
@Mapping(target = "id", expression = "java(entity.chunkId())")
110+
@Mapping(target = "text", expression = "java(entity.content())")
111+
@Mapping(target = "score", expression = "java(entity.retrievalScore())")
112+
@Mapping(target = "metadata", source = ".", qualifiedByName = "mapChunksEntityToMetadata")
113+
KnowledgeDocument convertToKnowledgeDocument(DataMateRetrievalChunksEntity entity);
114+
115+
/**
116+
* 将 DataMate 检索结果 entity 映射为 平台检索结果 metadata。
117+
*
118+
* @param entity 表示待转换的 {@link DataMateRetrievalChunksEntity}。
119+
* @return 转换完成的 {@link Map}{@code <}{@link String}{@code , }{@link Object}{@code >}。
120+
*/
121+
@Named("mapChunksEntityToMetadata")
122+
default Map<String, Object> mapChunksEntityToMetadata(DataMateRetrievalChunksEntity entity) {
123+
Map<String, Object> metadata = new HashMap<>();
124+
if (entity == null || entity.getEntity() == null) {
125+
return metadata;
126+
}
127+
metadata.put("primaryKey", entity.getPrimaryKey());
128+
String rawMetadata = entity.getEntity().getMetadata();
129+
if (!StringUtils.isEmpty(rawMetadata)) {
130+
try {
131+
Map<String, Object> parsed = new ObjectMapper().readValue(rawMetadata, Map.class);
132+
metadata.putAll(parsed);
133+
} catch (JsonProcessingException ex) {
134+
metadata.put("metadata", rawMetadata);
135+
}
136+
}
137+
return metadata;
138+
}
139+
}
140+

‎app-builder/plugins/data-mate-knowledge/src/main/java/modelengine/fit/jade/datamate/knowledge/knowledge/dto/DataMateKnowledgeListQueryParam.java‎

Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,43 @@
1+
/*---------------------------------------------------------------------------------------------
2+
* Copyright (c) 2025 Huawei Technologies Co., Ltd. All rights reserved.
3+
* This file is a part of the ModelEngine Project.
4+
* Licensed under the MIT License. See License.txt in the project root for license information.
5+
*--------------------------------------------------------------------------------------------*/
6+
7+
package modelengine.fit.jade.datamate.knowledge.knowledge.dto;
8+
9+
import lombok.Builder;
10+
import lombok.Data;
11+
import modelengine.fitframework.serialization.annotation.SerializeStrategy;
12+
13+
/**
14+
* DataMate 知识库列表查询参数。
15+
*
16+
* @author 陈镕希
17+
* @since 2025-12-15
18+
*/
19+
@Data
20+
@Builder
21+
@SerializeStrategy(include = SerializeStrategy.Include.NON_NULL)
22+
public class DataMateKnowledgeListQueryParam {
23+
/**
24+
* 页码，从0开始。
25+
*/
26+
private Integer page;
27+
28+
/**
29+
* 每页大小。
30+
*/
31+
private Integer size;
32+
33+
/**
34+
* 知识库名称过滤。
35+
*/
36+
private String name;
37+
38+
/**
39+
* 知识库描述过滤。
40+
*/
41+
private String description;
42+
}
43+

‎app-builder/plugins/data-mate-knowledge/src/main/java/modelengine/fit/jade/datamate/knowledge/knowledge/dto/DataMateRetrievalParam.java‎

Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,44 @@
1+
/*---------------------------------------------------------------------------------------------
2+
* Copyright (c) 2025 Huawei Technologies Co., Ltd. All rights reserved.
3+
* This file is a part of the ModelEngine Project.
4+
* Licensed under the MIT License. See License.txt in the project root for license information.
5+
*--------------------------------------------------------------------------------------------*/
6+
7+
package modelengine.fit.jade.datamate.knowledge.knowledge.dto;
8+
9+
import lombok.Builder;
10+
import lombok.Data;
11+
import modelengine.fitframework.annotation.Property;
12+
13+
import java.util.List;
14+
15+
/**
16+
* DataMate 知识库检索查询参数。
17+
*
18+
* @author 陈镕希
19+
* @since 2025-12-15
20+
*/
21+
@Data
22+
@Builder
23+
public class DataMateRetrievalParam {
24+
/**
25+
* 检索 query。
26+
*/
27+
private String query;
28+
/**
29+
* 返回前多少的条目。
30+
*/
31+
@Property(description = "topK", name = "topK")
32+
private Integer topK;
33+
/**
34+
* 相关性阈值。
35+
*/
36+
@Property(description = "threshold", name = "threshold")
37+
private Double threshold;
38+
/**
39+
* 指定知识库的id集合。
40+
*/
41+
@Property(description = "knowledgeBaseIds", name = "knowledgeBaseIds")
42+
private List<String> knowledgeBaseIds;
43+
}
44+

0 commit comments

Comments
 (0)