Skip to content

Commit 0c1543a

Browse files
hhhhsc701hhhhsc701
committed
Merge branch 'main' into develop_hsc
2 parents 7bf8edf + 7012a9a commit 0c1543a

File tree

69 files changed

+3894
-2567
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

69 files changed

+3894
-2567
lines changed

‎.github/workflows/docker-image-backend-python.yml‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,12 +5,14 @@ on:
55
branches: [ "main" ]
66
paths:
77
- 'scripts/images/datamate-python/**'
8+
- 'runtime/datamate-python/**'
89
- '.github/workflows/docker-image-backend-python.yml'
910
- '.github/workflows/docker-images-reusable.yml'
1011
pull_request:
1112
branches: [ "main" ]
1213
paths:
1314
- 'scripts/images/datamate-python/**'
15+
- 'runtime/datamate-python/**'
1416
- '.github/workflows/docker-image-backend-python.yml'
1517
- '.github/workflows/docker-images-reusable.yml'
1618
workflow_dispatch:

‎backend/services/data-management-service/src/main/java/com/datamate/datamanagement/application/DatasetApplicationService.java‎

Lines changed: 11 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@
2121
import lombok.extern.slf4j.Slf4j;
2222
import org.apache.commons.collections4.CollectionUtils;
2323
import org.springframework.beans.factory.annotation.Value;
24+
import org.springframework.data.redis.core.StringRedisTemplate;
2425
import org.springframework.scheduling.annotation.Async;
2526
import org.springframework.stereotype.Service;
2627
import org.springframework.transaction.annotation.Transactional;
@@ -42,11 +43,13 @@
4243
@Transactional
4344
@RequiredArgsConstructor
4445
public class DatasetApplicationService {
46+
private static final String DATASET_PVC_NAME = "sys.management.dataset.pvc.name";
4547
private final DatasetRepository datasetRepository;
4648
private final TagMapper tagMapper;
4749
private final DatasetFileRepository datasetFileRepository;
4850
private final CollectionTaskClient collectionTaskClient;
4951
private final DatasetFileApplicationService datasetFileApplicationService;
52+
private final StringRedisTemplate redisTemplate;
5053

5154
@Value("${datamate.data-management.base-path:/dataset}")
5255
private String datasetBasePath;
@@ -76,6 +79,10 @@ public Dataset createDataset(CreateDatasetRequest createDatasetRequest) {
7679
return dataset;
7780
}
7881

82+
public String getDatasetPvcName() {
83+
return redisTemplate.opsForValue().get(DATASET_PVC_NAME);
84+
}
85+
7986
public Dataset updateDataset(String datasetId, UpdateDatasetRequest updateDatasetRequest) {
8087
Dataset dataset = datasetRepository.getById(datasetId);
8188
BusinessAssert.notNull(dataset, DataManagementErrorCode.DATASET_NOT_FOUND);
@@ -130,7 +137,10 @@ public Dataset getDataset(String datasetId) {
130137
public PagedResponse<DatasetResponse> getDatasets(DatasetPagingQuery query) {
131138
IPage<Dataset> page = new Page<>(query.getPage(), query.getSize());
132139
page = datasetRepository.findByCriteria(page, query);
133-
return PagedResponse.of(DatasetConverter.INSTANCE.convertToResponse(page.getRecords()), page.getCurrent(), page.getTotal(), page.getPages());
140+
String datasetPvcName = getDatasetPvcName();
141+
List<DatasetResponse> datasetResponses = DatasetConverter.INSTANCE.convertToResponse(page.getRecords());
142+
datasetResponses.forEach(dataset -> dataset.setPvcName(datasetPvcName));
143+
return PagedResponse.of(datasetResponses, page.getCurrent(), page.getTotal(), page.getPages());
134144
}
135145

136146
/**

‎backend/services/data-management-service/src/main/java/com/datamate/datamanagement/domain/model/dataset/FileTag.java‎

Lines changed: 1 addition & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,7 @@ public class FileTag {
2828

2929
private String fromName;
3030

31-
public List<String> getTags() {
31+
public List<String> getTagValues() {
3232
List<String> tags = new ArrayList<>();
3333
Object tagValues = values.get(type);
3434
if (tagValues instanceof List) {
@@ -40,9 +40,6 @@ public List<String> getTags() {
4040
} else if (tagValues instanceof String) {
4141
tags.add((String) tagValues);
4242
}
43-
if(StringUtils.isNotEmpty(fromName)) {
44-
return tags.stream().map(tag -> fromName + " " + tag).toList();
45-
}
4643
return tags;
4744
}
4845
}

‎backend/services/data-management-service/src/main/java/com/datamate/datamanagement/interfaces/converter/DatasetConverter.java‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -70,8 +70,8 @@ public interface DatasetConverter {
7070
* @return 标签分布
7171
*/
7272
@Named("getDistribution")
73-
default Map<String, Long> getDistribution(List<DatasetFile> datasetFiles) {
74-
Map<String, Long> distribution = new HashMap<>();
73+
default Map<String, Map<String, Long>> getDistribution(List<DatasetFile> datasetFiles) {
74+
Map<String, Map<String, Long>> distribution = new HashMap<>();
7575
if (CollectionUtils.isEmpty(datasetFiles)) {
7676
return distribution;
7777
}
@@ -81,7 +81,9 @@ default Map<String, Long> getDistribution(List<DatasetFile> datasetFiles) {
8181
return distribution;
8282
}
8383
for (FileTag tag : tags) {
84-
tag.getTags().forEach(tagName -> distribution.put(tagName, distribution.getOrDefault(tagName, 0L) + 1));
84+
Map<String, Long> tagValueMap = distribution.getOrDefault(tag.getFromName(), new HashMap<>());
85+
tag.getTagValues().forEach(tagValue -> tagValueMap.put(tagValue, tagValueMap.getOrDefault(tagValue, 0L) + 1));
86+
distribution.put(tag.getFromName(), tagValueMap);
8587
}
8688
}
8789
return distribution;

‎backend/services/data-management-service/src/main/java/com/datamate/datamanagement/interfaces/dto/DatasetResponse.java‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -46,5 +46,7 @@ public class DatasetResponse {
4646
/** 更新者 */
4747
private String updatedBy;
4848
/** 分布 */
49-
private Map<String, Long> distribution ;
49+
private Map<String, Map<String, Long>> distribution;
50+
/** 数据集pvc名称 */
51+
private String pvcName;
5052
}

‎backend/services/data-management-service/src/main/java/com/datamate/datamanagement/interfaces/rest/DatasetController.java‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -58,8 +58,9 @@ public DatasetResponse createDataset(@RequestBody @Valid CreateDatasetRequest cr
5858
*/
5959
@GetMapping("/{datasetId}")
6060
public DatasetResponse getDatasetById(@PathVariable("datasetId") String datasetId) {
61-
Dataset dataset = datasetApplicationService.getDataset(datasetId);
62-
return DatasetConverter.INSTANCE.convertToResponse(dataset);
61+
DatasetResponse dataset = DatasetConverter.INSTANCE.convertToResponse(datasetApplicationService.getDataset(datasetId));
62+
dataset.setPvcName(datasetApplicationService.getDatasetPvcName());
63+
return dataset;
6364
}
6465

6566
/**

‎deployment/docker/datamate/backend.conf‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,13 @@ server {
1414
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
1515
}
1616

17+
location /api/evaluation/ {
18+
proxy_pass http://datamate-backend-python:18000/api/evaluation/;
19+
proxy_set_header Host $host;
20+
proxy_set_header X-Real-IP $remote_addr;
21+
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
22+
}
23+
1724
location /api/annotation/ {
1825
proxy_pass http://datamate-backend-python:18000/api/annotation/;
1926
proxy_set_header Host $host;

‎deployment/docker/milvus/docker-compose.yml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@ services:
1010
volumes:
1111
- ${DOCKER_VOLUME_DIRECTORY:-.}/volumes/etcd:/etcd
1212
command: etcd -advertise-client-urls=http://etcd:2379 -listen-client-urls http://0.0.0.0:2379 --data-dir /etcd
13+
restart: always
1314
networks:
1415
- datamate
1516
healthcheck:

‎frontend/src/components/business/DatasetFileTransfer.tsx‎

Lines changed: 27 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
import React, { useEffect } from "react";
1+
import React, { useCallback, useEffect } from "react";
22
import { Button, Input, Table } from "antd";
33
import { RightOutlined } from "@ant-design/icons";
44
import { mapDataset } from "@/pages/DataManagement/dataset.const";
@@ -19,6 +19,7 @@ interface DatasetFileTransferProps
1919
open: boolean;
2020
selectedFilesMap: { [key: string]: DatasetFile };
2121
onSelectedFilesChange: (filesMap: { [key: string]: DatasetFile }) => void;
22+
onDatasetSelect?: (dataset: Dataset | null) => void;
2223
}
2324

2425
const fileCols = [
@@ -48,6 +49,7 @@ const DatasetFileTransfer: React.FC<DatasetFileTransferProps> = ({
4849
open,
4950
selectedFilesMap,
5051
onSelectedFilesChange,
52+
onDatasetSelect,
5153
...props
5254
}) => {
5355
const [datasets, setDatasets] = React.useState<Dataset[]>([]);
@@ -96,31 +98,33 @@ const DatasetFileTransfer: React.FC<DatasetFileTransferProps> = ({
9698
300
9799
);
98100

99-
const fetchFiles = async () => {
101+
const fetchFiles = useCallback(async () => {
100102
if (!selectedDataset) return;
101103
const { data } = await queryDatasetFilesUsingGet(selectedDataset.id, {
102104
page: filesPagination.current - 1,
103105
size: filesPagination.pageSize,
104106
keyword: filesSearch,
105107
});
106108
setFiles(
107-
data.content.map((item) => ({
109+
(data.content || []).map((item: DatasetFile) => ({
108110
...item,
109111
key: item.id,
110112
datasetName: selectedDataset.name,
111-
})) || []
113+
}))
112114
);
113115
setFilesPagination((prev) => ({
114116
...prev,
115117
total: data.totalElements,
116118
}));
117-
};
119+
}, [filesPagination.current, filesPagination.pageSize, filesSearch, selectedDataset]);
118120

119121
useEffect(() => {
120-
if (selectedDataset) {
121-
fetchFiles();
122-
}
123-
}, [selectedDataset]);
122+
fetchFiles().catch(() => {});
123+
}, [fetchFiles]);
124+
125+
useEffect(() => {
126+
onDatasetSelect?.(selectedDataset);
127+
}, [selectedDataset, onDatasetSelect]);
124128

125129
const toggleSelectFile = (record: DatasetFile) => {
126130
if (!selectedFilesMap[record.id]) {
@@ -147,8 +151,9 @@ const DatasetFileTransfer: React.FC<DatasetFileTransferProps> = ({
147151
setShowFiles(false);
148152
setSelectedDataset(null);
149153
setDatasetSelections([]);
154+
onDatasetSelect?.(null);
150155
}
151-
}, [open]);
156+
}, [open, onDatasetSelect]);
152157

153158
const datasetCols = [
154159
{
@@ -206,7 +211,15 @@ const DatasetFileTransfer: React.FC<DatasetFileTransferProps> = ({
206211
})}
207212
dataSource={datasets}
208213
columns={datasetCols}
209-
pagination={datasetPagination}
214+
pagination={{
215+
...datasetPagination,
216+
onChange: (page, pageSize) =>
217+
setDatasetPagination({
218+
current: page,
219+
pageSize: pageSize || datasetPagination.pageSize,
220+
total: datasetPagination.total,
221+
}),
222+
}}
210223
/>
211224
</div>
212225
<RightOutlined />
@@ -231,21 +244,11 @@ const DatasetFileTransfer: React.FC<DatasetFileTransferProps> = ({
231244
})}
232245
rowSelection={{
233246
type: "checkbox",
234-
onSelectAll: (selected, _, changeRows) => {
235-
const newSelectedFiles = { ...selectedFilesMap };
236-
if (selected) {
237-
changeRows.forEach((row) => {
238-
newSelectedFiles[row.id] = row;
239-
});
240-
} else {
241-
changeRows.forEach((row) => {
242-
delete newSelectedFiles[row.id];
243-
});
244-
}
245-
onSelectedFilesChange(newSelectedFiles);
246-
},
247247
selectedRowKeys: Object.keys(selectedFilesMap),
248248
onSelect: toggleSelectFile,
249+
getCheckboxProps: (record: DatasetFile) => ({
250+
name: record.fileName,
251+
}),
249252
}}
250253
/>
251254
</div>

0 commit comments

Comments
 (0)