Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
35 commits
Select commit Hold shift + click to select a range
12dc3a6
refactor: 修改调整数据归集实现,删除无用代码,优化代码结构
hefanli Oct 23, 2025
8809997
Merge remote-tracking branch 'origin/main'
hefanli Oct 23, 2025
008a1e4
Merge branch 'ModelEngine-Group:main' into main
hefanli Oct 24, 2025
8d7bb8a
feature: 每天凌晨00:00扫描所有数据集,检查数据集是否超过了预设的保留天数,超出保留天数的数据集调用删除接口进行删除
hefanli Oct 24, 2025
738127e
fix: 修改删除数据集文件的逻辑,上传到数据集中的文件会同时删除数据库中的记录和文件系统中的文件,归集过来的文件仅删除数据库中的记录
hefanli Oct 24, 2025
23a821a
Merge remote-tracking branch 'origin/main'
hefanli Oct 24, 2025
98ee5b3
fix: 增加参数校验和接口定义,删除不使用的接口
hefanli Oct 24, 2025
825bd4b
fix: 数据集统计数据默认为0
hefanli Oct 24, 2025
3b1a043
feature: 数据集状态增加流转,创建时为草稿状态,上传文件或者归集文件后修改为活动状态
hefanli Oct 24, 2025
dc3706d
refactor: 修改分页查询归集任务的代码
hefanli Oct 24, 2025
732b4e4
Merge branch 'ModelEngine-Group:main' into main
hefanli Oct 25, 2025
96c2247
Merge remote-tracking branch 'origin/main'
hefanli Oct 25, 2025
4398e56
fix: 更新后重新执行;归集任务执行增加事务控制
hefanli Oct 25, 2025
bb22e89
feature: 创建归集任务时能够同步创建数据集,更新归集任务时能更新到指定数据集
hefanli Oct 25, 2025
7edfacd
Merge branch 'ModelEngine-Group:main' into main
hefanli Oct 25, 2025
b42fa8f
fix: 创建归集任务不需要创建数据集时不应该报错
hefanli Oct 27, 2025
a9623f8
Merge branch 'ModelEngine-Group:main' into main
hefanli Oct 28, 2025
a177724
Merge remote-tracking branch 'origin/main'
hefanli Oct 28, 2025
8ad8980
Merge branch 'ModelEngine-Group:main' into main
hefanli Oct 28, 2025
5fb153e
Merge branch 'ModelEngine-Group:main' into main
hefanli Oct 28, 2025
d18b62d
fix: 修复删除文件时数据集的统计数据不变动
hefanli Oct 29, 2025
98b81a4
feature: 查询数据集详情时能够获取到文件标签分布
hefanli Oct 29, 2025
109aba9
Merge branch 'ModelEngine-Group:main' into main
hefanli Oct 30, 2025
5756ecd
Merge branch 'ModelEngine-Group:main' into main
hefanli Oct 30, 2025
e445455
fix: tags为空时不进行分析
hefanli Oct 30, 2025
ee0e93c
fix: 状态修改为ACTIVE
hefanli Oct 30, 2025
b77b0dc
fix: 修改解析tag的方法
hefanli Oct 31, 2025
9b986b5
Merge branch 'ModelEngine-Group:main' into main
hefanli Oct 31, 2025
834f213
Merge remote-tracking branch 'origin/main'
hefanli Oct 31, 2025
3d7cd24
Merge branch 'ModelEngine-Group:main' into main
hefanli Oct 31, 2025
37e30ed
feature: 实现创建、分页查询、删除配比任务
hefanli Oct 31, 2025
6276552
feature: 实现创建、分页查询、删除配比任务的前端交互
hefanli Oct 31, 2025
3f4e26e
fix: 修复进度计算异常导致的页面报错
hefanli Oct 31, 2025
6c88a30
Merge remote-tracking branch 'origin/main'
hefanli Oct 31, 2025
33ac4f8
Merge branch 'ModelEngine-Group:main' into main
hefanli Oct 31, 2025
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@
import com.datamate.datamanagement.application.DatasetApplicationService;
import com.datamate.datamanagement.domain.model.dataset.Dataset;
import com.datamate.datamanagement.interfaces.converter.DatasetConverter;
import com.datamate.datamanagement.interfaces.dto.DatasetResponse;
import jakarta.validation.Valid;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
Expand All @@ -35,11 +36,13 @@ public class CollectionTaskController{
public ResponseEntity<CollectionTaskResponse> createTask(@Valid @RequestBody CreateCollectionTaskRequest request) {
CollectionTask task = CollectionTaskConverter.INSTANCE.toCollectionTask(request);
String datasetId = null;
DatasetResponse dataset = null;
if (Objects.nonNull(request.getDataset())) {
datasetId = datasetService.createDataset(request.getDataset()).getId();
dataset = DatasetConverter.INSTANCE.convertToResponse(datasetService.createDataset(request.getDataset()));
datasetId = dataset.getId();
}
CollectionTaskResponse response = CollectionTaskConverter.INSTANCE.toResponse(taskService.create(task, datasetId));
response.setDataset(DatasetConverter.INSTANCE.convertToResponse(datasetService.getDataset(datasetId)));
response.setDataset(dataset);
return ResponseEntity.ok().body(response);
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -119,6 +119,8 @@ public void deleteDataset(String datasetId) {
public Dataset getDataset(String datasetId) {
Dataset dataset = datasetRepository.getById(datasetId);
BusinessAssert.notNull(dataset, DataManagementErrorCode.DATASET_NOT_FOUND);
List<DatasetFile> datasetFiles = datasetFileRepository.findAllByDatasetId(datasetId);
dataset.setFiles(datasetFiles);
return dataset;
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -102,6 +102,10 @@ public DatasetFile getDatasetFile(String datasetId, String fileId) {
public void deleteDatasetFile(String datasetId, String fileId) {
DatasetFile file = getDatasetFile(datasetId, fileId);
Dataset dataset = datasetRepository.getById(datasetId);
dataset.setFiles(new ArrayList<>(Collections.singleton(file)));
datasetFileRepository.removeById(fileId);
dataset.removeFile(file);
datasetRepository.updateById(dataset);
// 删除文件时,上传到数据集中的文件会同时删除数据库中的记录和文件系统中的文件,归集过来的文件仅删除数据库中的记录
if (file.getFilePath().startsWith(dataset.getPath())) {
try {
Expand All @@ -111,9 +115,6 @@ public void deleteDatasetFile(String datasetId, String fileId) {
throw BusinessException.of(SystemErrorCode.FILE_SYSTEM_ERROR);
}
}
datasetFileRepository.removeById(fileId);
dataset.removeFile(file);
datasetRepository.updateById(dataset);
}

/**
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -110,7 +110,7 @@ private DatasetFile extractFileMetadata(String filePath, String datasetId) throw
.fileType(fileType)
.uploadTime(LocalDateTime.now())
.lastAccessTime(LocalDateTime.now())
.status("UPLOADED")
.status("ACTIVE")
.build();
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2,9 +2,13 @@

import com.baomidou.mybatisplus.annotation.TableId;
import com.baomidou.mybatisplus.annotation.TableName;
import com.fasterxml.jackson.databind.ObjectMapper;
import lombok.*;
import org.apache.commons.lang3.StringUtils;

import java.time.LocalDateTime;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;

/**
Expand All @@ -25,11 +29,25 @@ public class DatasetFile {
private String fileType; // JPG/PNG/DCM/TXT
private Long fileSize; // bytes
private String checkSum;
private List<String> tags;
private String tags;
private String metadata;
private String status; // UPLOADED, PROCESSING, COMPLETED, ERROR
private LocalDateTime uploadTime;
private LocalDateTime lastAccessTime;
private LocalDateTime createdAt;
private LocalDateTime updatedAt;

/**
* 解析标签
*
* @return 标签列表
*/
public List<String> analyzeTag() {
try {
ObjectMapper mapper = new ObjectMapper();
return mapper.readValue(tags, List.class);
} catch (Exception e) {
return Collections.emptyList();
}
}
}

This file was deleted.

Original file line number Diff line number Diff line change
@@ -1,17 +1,24 @@
package com.datamate.datamanagement.interfaces.converter;

import com.datamate.common.infrastructure.exception.BusinessException;
import com.datamate.common.infrastructure.exception.SystemErrorCode;
import com.datamate.datamanagement.interfaces.dto.CreateDatasetRequest;
import com.datamate.datamanagement.interfaces.dto.DatasetFileResponse;
import com.datamate.datamanagement.interfaces.dto.DatasetResponse;
import com.datamate.datamanagement.interfaces.dto.UploadFileRequest;
import com.datamate.common.domain.model.ChunkUploadRequest;
import com.datamate.datamanagement.domain.model.dataset.Dataset;
import com.datamate.datamanagement.domain.model.dataset.DatasetFile;
import com.fasterxml.jackson.databind.ObjectMapper;
import org.apache.commons.collections4.CollectionUtils;
import org.mapstruct.Mapper;
import org.mapstruct.Mapping;
import org.mapstruct.Named;
import org.mapstruct.factory.Mappers;

import java.util.HashMap;
import java.util.List;
import java.util.Map;

/**
* 数据集文件转换器
Expand All @@ -26,6 +33,7 @@ public interface DatasetConverter {
*/
@Mapping(source = "sizeBytes", target = "totalSize")
@Mapping(source = "path", target = "targetLocation")
@Mapping(source = "files", target = "distribution", qualifiedByName = "getDistribution")
DatasetResponse convertToResponse(Dataset dataset);

/**
Expand All @@ -49,4 +57,28 @@ public interface DatasetConverter {
* 将数据集文件转换为响应
*/
DatasetFileResponse convertToResponse(DatasetFile datasetFile);

/**
* 获取数据文件的标签分布
*
* @param datasetFiles 数据集文件
* @return 标签分布
*/
@Named("getDistribution")
default Map<String, Long> getDistribution(List<DatasetFile> datasetFiles) {
Map<String, Long> distribution = new HashMap<>();
if (CollectionUtils.isEmpty(datasetFiles)) {
return distribution;
}
for (DatasetFile datasetFile : datasetFiles) {
List<String> tags = datasetFile.analyzeTag();
if (CollectionUtils.isEmpty(tags)) {
continue;
}
for (String tag : tags) {
distribution.put(tag, distribution.getOrDefault(tag, 0L) + 1);
}
}
return distribution;
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@

import java.time.LocalDateTime;
import java.util.List;
import java.util.Map;

/**
* 数据集响应DTO
Expand Down Expand Up @@ -42,8 +43,8 @@ public class DatasetResponse {
private LocalDateTime updatedAt;
/** 创建者 */
private String createdBy;
/**
* 更新者
*/
/** 更新者 */
private String updatedBy;
/** 分布 */
private Map<String, Long> distribution ;
}
2 changes: 1 addition & 1 deletion frontend/src/mock/ratio.tsx
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
import type { RatioTask } from "@/pages/RatioTask/ratio";
import type { RatioTask } from "@/pages/RatioTask/ratio.model.ts";

export const mockRatioTasks: RatioTask[] = [
{
Expand Down
2 changes: 2 additions & 0 deletions frontend/src/pages/DataManagement/dataset.model.ts
Original file line number Diff line number Diff line change
Expand Up @@ -49,11 +49,13 @@ export interface Dataset {
status: DatasetStatus;
size?: string;
itemCount?: number;
fileCount?: number;
createdBy: string;
createdAt: string;
updatedAt: string;
tags: string[];
targetLocation?: string;
distribution?: Record<string, number>;
}

export interface TagItem {
Expand Down
Loading