diff --git a/backend/services/data-collection-service/src/main/java/com/datamate/collection/interfaces/rest/CollectionTaskController.java b/backend/services/data-collection-service/src/main/java/com/datamate/collection/interfaces/rest/CollectionTaskController.java index 38deb639..172d9b02 100644 --- a/backend/services/data-collection-service/src/main/java/com/datamate/collection/interfaces/rest/CollectionTaskController.java +++ b/backend/services/data-collection-service/src/main/java/com/datamate/collection/interfaces/rest/CollectionTaskController.java @@ -10,6 +10,7 @@ import com.datamate.datamanagement.application.DatasetApplicationService; import com.datamate.datamanagement.domain.model.dataset.Dataset; import com.datamate.datamanagement.interfaces.converter.DatasetConverter; +import com.datamate.datamanagement.interfaces.dto.DatasetResponse; import jakarta.validation.Valid; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; @@ -35,11 +36,13 @@ public class CollectionTaskController{ public ResponseEntity createTask(@Valid @RequestBody CreateCollectionTaskRequest request) { CollectionTask task = CollectionTaskConverter.INSTANCE.toCollectionTask(request); String datasetId = null; + DatasetResponse dataset = null; if (Objects.nonNull(request.getDataset())) { - datasetId = datasetService.createDataset(request.getDataset()).getId(); + dataset = DatasetConverter.INSTANCE.convertToResponse(datasetService.createDataset(request.getDataset())); + datasetId = dataset.getId(); } CollectionTaskResponse response = CollectionTaskConverter.INSTANCE.toResponse(taskService.create(task, datasetId)); - response.setDataset(DatasetConverter.INSTANCE.convertToResponse(datasetService.getDataset(datasetId))); + response.setDataset(dataset); return ResponseEntity.ok().body(response); } diff --git a/backend/services/data-management-service/src/main/java/com/datamate/datamanagement/application/DatasetApplicationService.java b/backend/services/data-management-service/src/main/java/com/datamate/datamanagement/application/DatasetApplicationService.java index e39443e6..8c9bca54 100644 --- a/backend/services/data-management-service/src/main/java/com/datamate/datamanagement/application/DatasetApplicationService.java +++ b/backend/services/data-management-service/src/main/java/com/datamate/datamanagement/application/DatasetApplicationService.java @@ -119,6 +119,8 @@ public void deleteDataset(String datasetId) { public Dataset getDataset(String datasetId) { Dataset dataset = datasetRepository.getById(datasetId); BusinessAssert.notNull(dataset, DataManagementErrorCode.DATASET_NOT_FOUND); + List datasetFiles = datasetFileRepository.findAllByDatasetId(datasetId); + dataset.setFiles(datasetFiles); return dataset; } diff --git a/backend/services/data-management-service/src/main/java/com/datamate/datamanagement/application/DatasetFileApplicationService.java b/backend/services/data-management-service/src/main/java/com/datamate/datamanagement/application/DatasetFileApplicationService.java index 0fe61942..9f23cd87 100644 --- a/backend/services/data-management-service/src/main/java/com/datamate/datamanagement/application/DatasetFileApplicationService.java +++ b/backend/services/data-management-service/src/main/java/com/datamate/datamanagement/application/DatasetFileApplicationService.java @@ -102,6 +102,10 @@ public DatasetFile getDatasetFile(String datasetId, String fileId) { public void deleteDatasetFile(String datasetId, String fileId) { DatasetFile file = getDatasetFile(datasetId, fileId); Dataset dataset = datasetRepository.getById(datasetId); + dataset.setFiles(new ArrayList<>(Collections.singleton(file))); + datasetFileRepository.removeById(fileId); + dataset.removeFile(file); + datasetRepository.updateById(dataset); // 删除文件时,上传到数据集中的文件会同时删除数据库中的记录和文件系统中的文件,归集过来的文件仅删除数据库中的记录 if (file.getFilePath().startsWith(dataset.getPath())) { try { @@ -111,9 +115,6 @@ public void deleteDatasetFile(String datasetId, String fileId) { throw BusinessException.of(SystemErrorCode.FILE_SYSTEM_ERROR); } } - datasetFileRepository.removeById(fileId); - dataset.removeFile(file); - datasetRepository.updateById(dataset); } /** diff --git a/backend/services/data-management-service/src/main/java/com/datamate/datamanagement/application/FileMetadataService.java b/backend/services/data-management-service/src/main/java/com/datamate/datamanagement/application/FileMetadataService.java index af5799c8..58dedbb0 100644 --- a/backend/services/data-management-service/src/main/java/com/datamate/datamanagement/application/FileMetadataService.java +++ b/backend/services/data-management-service/src/main/java/com/datamate/datamanagement/application/FileMetadataService.java @@ -110,7 +110,7 @@ private DatasetFile extractFileMetadata(String filePath, String datasetId) throw .fileType(fileType) .uploadTime(LocalDateTime.now()) .lastAccessTime(LocalDateTime.now()) - .status("UPLOADED") + .status("ACTIVE") .build(); } diff --git a/backend/services/data-management-service/src/main/java/com/datamate/datamanagement/domain/model/dataset/DatasetFile.java b/backend/services/data-management-service/src/main/java/com/datamate/datamanagement/domain/model/dataset/DatasetFile.java index 85fb60b7..a082b90b 100644 --- a/backend/services/data-management-service/src/main/java/com/datamate/datamanagement/domain/model/dataset/DatasetFile.java +++ b/backend/services/data-management-service/src/main/java/com/datamate/datamanagement/domain/model/dataset/DatasetFile.java @@ -2,9 +2,13 @@ import com.baomidou.mybatisplus.annotation.TableId; import com.baomidou.mybatisplus.annotation.TableName; +import com.fasterxml.jackson.databind.ObjectMapper; import lombok.*; +import org.apache.commons.lang3.StringUtils; import java.time.LocalDateTime; +import java.util.Arrays; +import java.util.Collections; import java.util.List; /** @@ -25,11 +29,25 @@ public class DatasetFile { private String fileType; // JPG/PNG/DCM/TXT private Long fileSize; // bytes private String checkSum; - private List tags; + private String tags; private String metadata; private String status; // UPLOADED, PROCESSING, COMPLETED, ERROR private LocalDateTime uploadTime; private LocalDateTime lastAccessTime; private LocalDateTime createdAt; private LocalDateTime updatedAt; + + /** + * 解析标签 + * + * @return 标签列表 + */ + public List analyzeTag() { + try { + ObjectMapper mapper = new ObjectMapper(); + return mapper.readValue(tags, List.class); + } catch (Exception e) { + return Collections.emptyList(); + } + } } diff --git a/backend/services/data-management-service/src/main/java/com/datamate/datamanagement/domain/model/dataset/StatusConstants.java b/backend/services/data-management-service/src/main/java/com/datamate/datamanagement/domain/model/dataset/StatusConstants.java deleted file mode 100644 index 05d232df..00000000 --- a/backend/services/data-management-service/src/main/java/com/datamate/datamanagement/domain/model/dataset/StatusConstants.java +++ /dev/null @@ -1,33 +0,0 @@ -package com.datamate.datamanagement.domain.model.dataset; - -/** - * 状态常量类 - 统一管理所有状态枚举值 - */ -public final class StatusConstants { - - /** - * 数据集状态 - */ - public static final class DatasetStatuses { - public static final String DRAFT = "DRAFT"; - public static final String ACTIVE = "ACTIVE"; - public static final String ARCHIVED = "ARCHIVED"; - public static final String PROCESSING = "PROCESSING"; - - private DatasetStatuses() {} - } - - /** - * 数据集文件状态 - */ - public static final class DatasetFileStatuses { - public static final String UPLOADED = "UPLOADED"; - public static final String PROCESSING = "PROCESSING"; - public static final String COMPLETED = "COMPLETED"; - public static final String ERROR = "ERROR"; - - private DatasetFileStatuses() {} - } - - private StatusConstants() {} -} diff --git a/backend/services/data-management-service/src/main/java/com/datamate/datamanagement/interfaces/converter/DatasetConverter.java b/backend/services/data-management-service/src/main/java/com/datamate/datamanagement/interfaces/converter/DatasetConverter.java index 251033e5..37280839 100644 --- a/backend/services/data-management-service/src/main/java/com/datamate/datamanagement/interfaces/converter/DatasetConverter.java +++ b/backend/services/data-management-service/src/main/java/com/datamate/datamanagement/interfaces/converter/DatasetConverter.java @@ -1,5 +1,7 @@ package com.datamate.datamanagement.interfaces.converter; +import com.datamate.common.infrastructure.exception.BusinessException; +import com.datamate.common.infrastructure.exception.SystemErrorCode; import com.datamate.datamanagement.interfaces.dto.CreateDatasetRequest; import com.datamate.datamanagement.interfaces.dto.DatasetFileResponse; import com.datamate.datamanagement.interfaces.dto.DatasetResponse; @@ -7,11 +9,16 @@ import com.datamate.common.domain.model.ChunkUploadRequest; import com.datamate.datamanagement.domain.model.dataset.Dataset; import com.datamate.datamanagement.domain.model.dataset.DatasetFile; +import com.fasterxml.jackson.databind.ObjectMapper; +import org.apache.commons.collections4.CollectionUtils; import org.mapstruct.Mapper; import org.mapstruct.Mapping; +import org.mapstruct.Named; import org.mapstruct.factory.Mappers; +import java.util.HashMap; import java.util.List; +import java.util.Map; /** * 数据集文件转换器 @@ -26,6 +33,7 @@ public interface DatasetConverter { */ @Mapping(source = "sizeBytes", target = "totalSize") @Mapping(source = "path", target = "targetLocation") + @Mapping(source = "files", target = "distribution", qualifiedByName = "getDistribution") DatasetResponse convertToResponse(Dataset dataset); /** @@ -49,4 +57,28 @@ public interface DatasetConverter { * 将数据集文件转换为响应 */ DatasetFileResponse convertToResponse(DatasetFile datasetFile); + + /** + * 获取数据文件的标签分布 + * + * @param datasetFiles 数据集文件 + * @return 标签分布 + */ + @Named("getDistribution") + default Map getDistribution(List datasetFiles) { + Map distribution = new HashMap<>(); + if (CollectionUtils.isEmpty(datasetFiles)) { + return distribution; + } + for (DatasetFile datasetFile : datasetFiles) { + List tags = datasetFile.analyzeTag(); + if (CollectionUtils.isEmpty(tags)) { + continue; + } + for (String tag : tags) { + distribution.put(tag, distribution.getOrDefault(tag, 0L) + 1); + } + } + return distribution; + } } diff --git a/backend/services/data-management-service/src/main/java/com/datamate/datamanagement/interfaces/dto/DatasetResponse.java b/backend/services/data-management-service/src/main/java/com/datamate/datamanagement/interfaces/dto/DatasetResponse.java index b09f9856..7e430923 100644 --- a/backend/services/data-management-service/src/main/java/com/datamate/datamanagement/interfaces/dto/DatasetResponse.java +++ b/backend/services/data-management-service/src/main/java/com/datamate/datamanagement/interfaces/dto/DatasetResponse.java @@ -5,6 +5,7 @@ import java.time.LocalDateTime; import java.util.List; +import java.util.Map; /** * 数据集响应DTO @@ -42,8 +43,8 @@ public class DatasetResponse { private LocalDateTime updatedAt; /** 创建者 */ private String createdBy; - /** - * 更新者 - */ + /** 更新者 */ private String updatedBy; + /** 分布 */ + private Map distribution ; } diff --git a/frontend/src/mock/ratio.tsx b/frontend/src/mock/ratio.tsx index 13918fe0..71d8b3e4 100644 --- a/frontend/src/mock/ratio.tsx +++ b/frontend/src/mock/ratio.tsx @@ -1,4 +1,4 @@ -import type { RatioTask } from "@/pages/RatioTask/ratio"; +import type { RatioTask } from "@/pages/RatioTask/ratio.model.ts"; export const mockRatioTasks: RatioTask[] = [ { diff --git a/frontend/src/pages/DataManagement/dataset.model.ts b/frontend/src/pages/DataManagement/dataset.model.ts index 411de328..a685c148 100644 --- a/frontend/src/pages/DataManagement/dataset.model.ts +++ b/frontend/src/pages/DataManagement/dataset.model.ts @@ -49,11 +49,13 @@ export interface Dataset { status: DatasetStatus; size?: string; itemCount?: number; + fileCount?: number; createdBy: string; createdAt: string; updatedAt: string; tags: string[]; targetLocation?: string; + distribution?: Record; } export interface TagItem { diff --git a/frontend/src/pages/RatioTask/Create/CreateRatioTask.tsx b/frontend/src/pages/RatioTask/Create/CreateRatioTask.tsx new file mode 100644 index 00000000..ece5ada4 --- /dev/null +++ b/frontend/src/pages/RatioTask/Create/CreateRatioTask.tsx @@ -0,0 +1,314 @@ +import { useState } from "react"; +import { Button, Card, Form, Divider, message } from "antd"; +import { ArrowLeft, Play, BarChart3, Shuffle, PieChart } from "lucide-react"; +import { createRatioTaskUsingPost } from "@/pages/RatioTask/ratio.api.ts"; +import type { Dataset } from "@/pages/DataManagement/dataset.model.ts"; +import { useNavigate } from "react-router"; +import SelectDataset from "@/pages/RatioTask/Create/components/SelectDataset.tsx"; +import BasicInformation from "@/pages/RatioTask/Create/components/BasicInformation.tsx"; +import RatioConfig from "@/pages/RatioTask/Create/components/RatioConfig.tsx"; + +export default function CreateRatioTask() { + + const navigate = useNavigate(); + const [form] = Form.useForm(); + // 配比任务相关状态 + const [ratioTaskForm, setRatioTaskForm] = useState({ + name: "", + description: "", + ratioType: "dataset" as "dataset" | "label", + selectedDatasets: [] as string[], + ratioConfigs: [] as any[], + totalTargetCount: 10000, + autoStart: true, + }); + + const [datasets, setDatasets] = useState([]); + const [creating, setCreating] = useState(false); + const [distributions, setDistributions] = useState>>({}); + + + const handleCreateRatioTask = async () => { + try { + const values = await form.validateFields(); + if (!ratioTaskForm.ratioConfigs.length) { + message.error("请配置配比项"); + return; + } + // Build request payload + const ratio_method = ratioTaskForm.ratioType === "dataset" ? "DATASET" : "TAG"; + const totals = String(values.totalTargetCount); + const config = ratioTaskForm.ratioConfigs.map((c) => { + if (ratio_method === "DATASET") { + return { + datasetId: String(c.source), + counts: String(c.quantity ?? 0), + filter_conditions: "", + }; + } + // TAG mode: source key like `${datasetId}_${label}` + const source = String(c.source || ""); + const idx = source.indexOf("_"); + const datasetId = idx > 0 ? source.slice(0, idx) : source; + const label = idx > 0 ? source.slice(idx + 1) : ""; + return { + datasetId, + counts: String(c.quantity ?? 0), + filter_conditions: label ? JSON.stringify({ label }) : "", + }; + }); + + setCreating(true); + await createRatioTaskUsingPost({ + name: values.name, + description: values.description, + totals, + ratio_method, + config, + }); + message.success("配比任务创建成功"); + navigate("/data/synthesis/ratio-task"); + } catch { + // 校验失败 + } finally { + setCreating(false); + } + }; + + // dataset selection is handled inside SelectDataset via onSelectedDatasetsChange + + const updateRatioConfig = (source: string, quantity: number) => { + setRatioTaskForm((prev) => { + const existingIndex = prev.ratioConfigs.findIndex( + (config) => config.source === source + ); + const totalOtherQuantity = prev.ratioConfigs + .filter((config) => config.source !== source) + .reduce((sum, config) => sum + config.quantity, 0); + + const newConfig = { + id: source, + name: source, + type: prev.ratioType, + quantity: Math.min( + quantity, + prev.totalTargetCount - totalOtherQuantity + ), + percentage: Math.round((quantity / prev.totalTargetCount) * 100), + source, + }; + + if (existingIndex >= 0) { + const newConfigs = [...prev.ratioConfigs]; + newConfigs[existingIndex] = newConfig; + return { ...prev, ratioConfigs: newConfigs }; + } else { + return { ...prev, ratioConfigs: [...prev.ratioConfigs, newConfig] }; + } + }); + }; + + const generateAutoRatio = () => { + const selectedCount = ratioTaskForm.selectedDatasets.length; + if (selectedCount === 0) return; + + const baseQuantity = Math.floor( + ratioTaskForm.totalTargetCount / selectedCount + ); + const remainder = ratioTaskForm.totalTargetCount % selectedCount; + + const newConfigs = ratioTaskForm.selectedDatasets.map( + (datasetId, index) => { + const quantity = baseQuantity + (index < remainder ? 1 : 0); + return { + id: datasetId, + name: datasetId, + type: ratioTaskForm.ratioType, + quantity, + percentage: Math.round( + (quantity / ratioTaskForm.totalTargetCount) * 100 + ), + source: datasetId, + }; + } + ); + + setRatioTaskForm((prev) => ({ ...prev, ratioConfigs: newConfigs })); + }; + + // 标签模式下,更新某数据集的某个标签的数量 + const updateLabelRatioConfig = (datasetId: string, label: string, quantity: number) => { + const sourceKey = `${datasetId}_${label}`; + setRatioTaskForm((prev) => { + const existingIndex = prev.ratioConfigs.findIndex((c) => c.source === sourceKey); + const totalOtherQuantity = prev.ratioConfigs + .filter((c) => c.source !== sourceKey) + .reduce((sum, c) => sum + c.quantity, 0); + + const dist = distributions[datasetId] || {}; + const labelMax = dist[label] ?? Infinity; + const cappedQuantity = Math.max( + 0, + Math.min(quantity, prev.totalTargetCount - totalOtherQuantity, labelMax) + ); + + const newConfig = { + id: sourceKey, + name: label, + type: "label", + quantity: cappedQuantity, + percentage: Math.round((cappedQuantity / prev.totalTargetCount) * 100), + source: sourceKey, + }; + + if (existingIndex >= 0) { + const newConfigs = [...prev.ratioConfigs]; + newConfigs[existingIndex] = newConfig; + return { ...prev, ratioConfigs: newConfigs }; + } else { + return { ...prev, ratioConfigs: [...prev.ratioConfigs, newConfig] }; + } + }); + }; + + const handleValuesChange = (_, allValues) => { + setRatioTaskForm({ ...ratioTaskForm, ...allValues }); + }; + + return ( +
+ {/* Header */} +
+
+ +

创建配比任务

+
+
+ +
+
+ {/* 左侧:数据集选择 */} + setRatioTaskForm({ ...ratioTaskForm, ratioType: value, ratioConfigs: [] })} + onSelectedDatasetsChange={(next) => { + setRatioTaskForm((prev) => ({ + ...prev, + selectedDatasets: next, + ratioConfigs: prev.ratioConfigs.filter((c) => { + const id = String(c.source); + // keep only items whose dataset id remains selected + const dsId = id.includes("_") ? id.split("_")[0] : id; + return next.includes(dsId); + }), + })); + }} + onDistributionsChange={(next) => setDistributions(next)} + onDatasetsChange={(list) => setDatasets(list)} + /> + {/* 右侧:配比配置 */} +
+

+ + 配比配置 +

+ +
+
+ + + 配比设置 + +
+ 设置每个数据集的配比数量 +
+
+ +
+ + updateRatioConfig(datasetId, quantity)} + onUpdateLabelQuantity={(datasetId, label, quantity) => updateLabelRatioConfig(datasetId, label, quantity)} + /> + {/* 配比预览 */} + {ratioTaskForm.ratioConfigs.length > 0 && ( +
+ 配比预览 +
+
+
+ 总配比数量: + + {ratioTaskForm.ratioConfigs + .reduce((sum, config) => sum + config.quantity, 0) + .toLocaleString()} + +
+
+ 目标数量: + + {ratioTaskForm.totalTargetCount.toLocaleString()} + +
+
+ 配比项目: + + {ratioTaskForm.ratioConfigs.length}个 + +
+
+
+
+ )} + +
+ + +
+
+
+
+
+
+
+ ); +} diff --git a/frontend/src/pages/RatioTask/Create/components/BasicInformation.tsx b/frontend/src/pages/RatioTask/Create/components/BasicInformation.tsx new file mode 100644 index 00000000..67a3c5a5 --- /dev/null +++ b/frontend/src/pages/RatioTask/Create/components/BasicInformation.tsx @@ -0,0 +1,34 @@ +import React from "react"; +import { Form, Input } from "antd"; + +const { TextArea } = Input; + +interface BasicInformationProps { + totalTargetCount: number; +} + +const BasicInformation: React.FC = ({ totalTargetCount }) => { + return ( +
+ + + + + + + +