Skip to content

Commit 2f3244b

Browse files
committed
Merge branch 'main' into dev
# Conflicts: # frontend/src/pages/KnowledgeBase/components/DatasetFileTransfer.tsx
2 parents f1a4457 + 536ef9f commit 2f3244b

File tree

11 files changed

+376
-376
lines changed

11 files changed

+376
-376
lines changed

backend/services/rag-indexer-service/src/main/java/com/datamate/rag/indexer/infrastructure/milvus/MilvusService.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -38,7 +38,7 @@
3838
@Slf4j
3939
@Component
4040
public class MilvusService {
41-
@Value("${datamate.rag.milvus-host:milvus-standalone}")
41+
@Value("${datamate.rag.milvus-host:milvus}")
4242
private String milvusHost;
4343
@Value("${datamate.rag.milvus-port:19530}")
4444
private int milvusPort;

deployment/docker/milvus/docker-compose.yml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -38,7 +38,7 @@ services:
3838
timeout: 20s
3939
retries: 3
4040

41-
standalone:
41+
milvus:
4242
container_name: milvus-standalone
4343
image: milvusdb/milvus:v2.6.2
4444
command: ["milvus", "run", "standalone"]

deployment/helm/datamate/values.yaml

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,16 @@ global:
2424
name: "datamate-database"
2525
tag: "latest"
2626

27+
public:
28+
persistentVolumeClaim:
29+
storageClass: ""
30+
size:
31+
dataset: 10Gi
32+
flow: 1Gi
33+
log: 1Gi
34+
database: 1Gi
35+
operator: 1Gi
36+
2737
datasetVolume: &datasetVolume
2838
name: dataset-volume
2939
persistentVolumeClaim:

deployment/helm/milvus/templates/service.yaml

Lines changed: 0 additions & 62 deletions
Original file line numberDiff line numberDiff line change
@@ -58,65 +58,3 @@ spec:
5858
component: "proxy"
5959
{{- end }}
6060
{{- end }}
61-
62-
---
63-
{{- if or .Values.proxy.enabled (not .Values.cluster.enabled) }}
64-
apiVersion: v1
65-
kind: Service
66-
metadata:
67-
name: {{ template "milvus.standalone.fullname" . }}
68-
namespace: {{ .Release.Namespace }}
69-
labels:
70-
{{ include "milvus.labels" . | indent 4 }}
71-
{{- if .Values.service.labels }}
72-
{{ toYaml .Values.service.labels | indent 4 }}
73-
{{- end }}
74-
{{- if not .Values.cluster.enabled }}
75-
component: "standalone"
76-
{{- else if and .Values.proxy.enabled .Values.cluster.enabled }}
77-
component: "proxy"
78-
{{- end }}
79-
{{- with .Values.service.annotations }}
80-
annotations:
81-
{{ toYaml . | indent 4 }}
82-
{{- end }}
83-
spec:
84-
{{- if (or (eq .Values.service.type "ClusterIP") (empty .Values.service.type)) }}
85-
type: ClusterIP
86-
{{- if .Values.service.clusterIP }}
87-
clusterIP: {{ .Values.service.clusterIP }}
88-
{{- end }}
89-
{{- else if eq .Values.service.type "LoadBalancer" }}
90-
type: LoadBalancer
91-
{{- if .Values.service.loadBalancerIP }}
92-
loadBalancerIP: {{ .Values.service.loadBalancerIP }}
93-
{{- end }}
94-
{{- if .Values.service.loadBalancerSourceRanges }}
95-
loadBalancerSourceRanges:
96-
{{ toYaml .Values.service.loadBalancerSourceRanges | indent 4 }}
97-
{{- end -}}
98-
{{- else }}
99-
type: {{ .Values.service.type }}
100-
{{- end }}
101-
ports:
102-
- name: {{ .Values.service.portName }}
103-
port: {{ .Values.service.port }}
104-
protocol: TCP
105-
targetPort: milvus
106-
{{- if (and (eq .Values.service.type "NodePort") (not (empty .Values.service.nodePort))) }}
107-
nodePort: {{.Values.service.nodePort}}
108-
{{- end }}
109-
{{- if .Values.metrics.enabled }}
110-
- name: metrics
111-
protocol: TCP
112-
port: 9091
113-
targetPort: metrics
114-
{{- end }}
115-
selector:
116-
{{ include "milvus.matchLabels" . | indent 4 }}
117-
{{- if not .Values.cluster.enabled }}
118-
component: "standalone"
119-
{{- else if and .Values.proxy.enabled .Values.cluster.enabled }}
120-
component: "proxy"
121-
{{- end }}
122-
{{- end }}
Lines changed: 248 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,248 @@
1+
import React, { useEffect } from "react";
2+
import { Button, Input, Table } from "antd";
3+
import { RightOutlined } from "@ant-design/icons";
4+
import { mapDataset } from "@/pages/DataManagement/dataset.const";
5+
import {
6+
Dataset,
7+
DatasetFile,
8+
DatasetType,
9+
} from "@/pages/DataManagement/dataset.model";
10+
import {
11+
queryDatasetFilesUsingGet,
12+
queryDatasetsUsingGet,
13+
} from "@/pages/DataManagement/dataset.api";
14+
import { formatBytes } from "@/utils/unit";
15+
import { useDebouncedEffect } from "@/hooks/useDebouncedEffect";
16+
import { DatasetFileCols as fileCols } from "../pages/KnowledgeBase/knowledge-base.const";
17+
18+
interface DatasetFileTransferProps
19+
extends React.HTMLAttributes<HTMLDivElement> {
20+
open: boolean;
21+
selectedFilesMap: { [key: string]: DatasetFile };
22+
onSelectedFilesChange: (filesMap: { [key: string]: DatasetFile }) => void;
23+
}
24+
25+
// Customize Table Transfer
26+
const DatasetFileTransfer: React.FC<DatasetFileTransferProps> = ({
27+
open,
28+
selectedFilesMap,
29+
onSelectedFilesChange,
30+
...props
31+
}) => {
32+
const [datasets, setDatasets] = React.useState<Dataset[]>([]);
33+
const [datasetSearch, setDatasetSearch] = React.useState<string>("");
34+
const [datasetPagination, setDatasetPagination] = React.useState<{
35+
current: number;
36+
pageSize: number;
37+
total: number;
38+
}>({ current: 1, pageSize: 10, total: 0 });
39+
40+
const [files, setFiles] = React.useState<DatasetFile[]>([]);
41+
const [filesSearch, setFilesSearch] = React.useState<string>("");
42+
const [filesPagination, setFilesPagination] = React.useState<{
43+
current: number;
44+
pageSize: number;
45+
total: number;
46+
}>({ current: 1, pageSize: 10, total: 0 });
47+
48+
const [showFiles, setShowFiles] = React.useState<boolean>(false);
49+
const [selectedDataset, setSelectedDataset] = React.useState<Dataset | null>(
50+
null
51+
);
52+
const [datasetSelections, setDatasetSelections] = React.useState<Dataset[]>(
53+
[]
54+
);
55+
56+
const fetchDatasets = async () => {
57+
const { data } = await queryDatasetsUsingGet({
58+
page: datasetPagination.current - 1,
59+
size: datasetPagination.pageSize,
60+
keyword: datasetSearch,
61+
type: DatasetType.TEXT,
62+
});
63+
setDatasets(data.content.map(mapDataset) || []);
64+
setDatasetPagination((prev) => ({
65+
...prev,
66+
total: data.totalElements,
67+
}));
68+
};
69+
70+
useDebouncedEffect(
71+
() => {
72+
fetchDatasets();
73+
},
74+
[datasetSearch, datasetPagination.pageSize, datasetPagination.current],
75+
300
76+
);
77+
78+
const fetchFiles = async () => {
79+
if (!selectedDataset) return;
80+
const { data } = await queryDatasetFilesUsingGet(selectedDataset.id, {
81+
page: filesPagination.current - 1,
82+
size: filesPagination.pageSize,
83+
keyword: filesSearch,
84+
});
85+
setFiles(
86+
data.content.map((item) => ({
87+
...item,
88+
key: item.id,
89+
datasetName: selectedDataset.name,
90+
})) || []
91+
);
92+
setFilesPagination((prev) => ({
93+
...prev,
94+
total: data.totalElements,
95+
}));
96+
};
97+
98+
useEffect(() => {
99+
if (selectedDataset) {
100+
fetchFiles();
101+
}
102+
}, [selectedDataset]);
103+
104+
const toggleSelectFile = (record: DatasetFile) => {
105+
if (!selectedFilesMap[record.id]) {
106+
onSelectedFilesChange({
107+
...selectedFilesMap,
108+
[record.id]: record,
109+
});
110+
} else {
111+
const newSelectedFiles = { ...selectedFilesMap };
112+
delete newSelectedFiles[record.id];
113+
onSelectedFilesChange(newSelectedFiles);
114+
}
115+
};
116+
117+
useEffect(() => {
118+
if (!open) {
119+
// 重置状态
120+
setDatasets([]);
121+
setDatasetSearch("");
122+
setDatasetPagination({ current: 1, pageSize: 10, total: 0 });
123+
setFiles([]);
124+
setFilesSearch("");
125+
setFilesPagination({ current: 1, pageSize: 10, total: 0 });
126+
setShowFiles(false);
127+
setSelectedDataset(null);
128+
setDatasetSelections([]);
129+
}
130+
}, [open]);
131+
132+
const datasetCols = [
133+
{
134+
title: "数据集名称",
135+
dataIndex: "name",
136+
key: "name",
137+
ellipsis: true,
138+
},
139+
{
140+
title: "文件数",
141+
dataIndex: "fileCount",
142+
key: "fileCount",
143+
ellipsis: true,
144+
},
145+
{
146+
title: "大小",
147+
dataIndex: "totalSize",
148+
key: "totalSize",
149+
ellipsis: true,
150+
render: formatBytes,
151+
},
152+
];
153+
154+
return (
155+
<div {...props}>
156+
<div className="grid grid-cols-25 gap-4 w-full">
157+
<div className="border-card flex flex-col col-span-12">
158+
<div className="border-bottom p-2 font-bold">选择数据集</div>
159+
<div className="p-2">
160+
<Input
161+
placeholder="搜索数据集名称..."
162+
value={datasetSearch}
163+
allowClear
164+
onChange={(e) => setDatasetSearch(e.target.value)}
165+
/>
166+
</div>
167+
<Table
168+
scroll={{ y: 400 }}
169+
rowKey="id"
170+
size="small"
171+
rowClassName={(record) =>
172+
selectedDataset?.id === record.id ? "bg-blue-100" : ""
173+
}
174+
onRow={(record: Dataset) => ({
175+
onClick: () => {
176+
setSelectedDataset(record);
177+
if (!datasetSelections.find((d) => d.id === record.id)) {
178+
setDatasetSelections([...datasetSelections, record]);
179+
} else {
180+
setDatasetSelections(
181+
datasetSelections.filter((d) => d.id !== record.id)
182+
);
183+
}
184+
},
185+
})}
186+
dataSource={datasets}
187+
columns={datasetCols}
188+
pagination={datasetPagination}
189+
/>
190+
</div>
191+
<RightOutlined />
192+
<div className="border-card flex flex-col col-span-12">
193+
<div className="border-bottom p-2 font-bold">选择文件</div>
194+
<div className="p-2">
195+
<Input
196+
placeholder="搜索文件名称..."
197+
value={filesSearch}
198+
onChange={(e) => setFilesSearch(e.target.value)}
199+
/>
200+
</div>
201+
<Table
202+
scroll={{ y: 400 }}
203+
rowKey="id"
204+
size="small"
205+
dataSource={files}
206+
columns={fileCols.slice(1, fileCols.length)}
207+
pagination={filesPagination}
208+
onRow={(record: DatasetFile) => ({
209+
onClick: () => toggleSelectFile(record),
210+
})}
211+
rowSelection={{
212+
type: "checkbox",
213+
onSelectAll: (selected, _, changeRows) => {
214+
const newSelectedFiles = { ...selectedFilesMap };
215+
if (selected) {
216+
changeRows.forEach((row) => {
217+
newSelectedFiles[row.id] = row;
218+
});
219+
} else {
220+
changeRows.forEach((row) => {
221+
delete newSelectedFiles[row.id];
222+
});
223+
}
224+
onSelectedFilesChange(newSelectedFiles);
225+
},
226+
selectedRowKeys: Object.keys(selectedFilesMap),
227+
onSelect: toggleSelectFile,
228+
}}
229+
/>
230+
</div>
231+
</div>
232+
<Button className="mt-4" onClick={() => setShowFiles(!showFiles)}>
233+
{showFiles ? "取消预览" : "预览"}
234+
</Button>
235+
<div hidden={!showFiles}>
236+
<Table
237+
scroll={{ y: 400 }}
238+
rowKey="id"
239+
size="small"
240+
dataSource={Object.values(selectedFilesMap)}
241+
columns={fileCols}
242+
/>
243+
</div>
244+
</div>
245+
);
246+
};
247+
248+
export default DatasetFileTransfer;

frontend/src/pages/DataManagement/Detail/components/Overview.tsx

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@ import { datasetTypeMap } from "../../dataset.const";
66
export default function Overview({ dataset, filesOperation }) {
77
const {
88
fileList,
9+
pagination,
910
selectedFiles,
1011
setSelectedFiles,
1112
previewVisible,
@@ -179,7 +180,10 @@ export default function Overview({ dataset, filesOperation }) {
179180
dataSource={fileList}
180181
// rowSelection={rowSelection}
181182
scroll={{ x: "max-content", y: 600 }}
182-
pagination={{ showTotal: (total) => `共 ${total} 条` }}
183+
pagination={{
184+
...pagination,
185+
showTotal: (total) => `共 ${total} 条`,
186+
}}
183187
/>
184188
</div>
185189
</div>

0 commit comments

Comments
 (0)