Skip to content

Commit 5363927

Browse files
hefanlihefanli
authored
Merge pull request #8 from ModelEngine-Group/develop_930_gfl
feature：增加切片上传接口
2 parents 03f89fa + 940e5de commit 5363927

File tree

34 files changed

+1051
-587
lines changed

34 files changed

+1051
-587
lines changed

‎backend/pom.xml‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,7 @@
3434
<mapstruct.version>1.6.3</mapstruct.version>
3535
<lombok.version>1.18.32</lombok.version>
3636
<lombok-mapstruct-binding.version>0.2.0</lombok-mapstruct-binding.version>
37+
<poi.version>5.4.0</poi.version>
3738
</properties>
3839

3940
<modules>
@@ -132,6 +133,12 @@
132133
<artifactId>mysql-connector-java</artifactId>
133134
<version>${mysql.version}</version>
134135
</dependency>
136+
137+
<dependency>
138+
<groupId>org.apache.poi</groupId>
139+
<artifactId>poi</artifactId>
140+
<version>${poi.version}</version>
141+
</dependency>
135142
</dependencies>
136143
</dependencyManagement>
137144

@@ -165,6 +172,12 @@
165172
<artifactId>mapstruct</artifactId>
166173
<version>${mapstruct.version}</version>
167174
</dependency>
175+
176+
<dependency>
177+
<groupId>org.apache.poi</groupId>
178+
<artifactId>poi</artifactId>
179+
<version>${poi.version}</version>
180+
</dependency>
168181
</dependencies>
169182

170183
<build>

‎backend/services/data-management-service/src/main/java/com/dataengine/datamanagement/application/service/DatasetApplicationService.java‎

Lines changed: 25 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -1,23 +1,26 @@
11
package com.dataengine.datamanagement.application.service;
22

33
import com.dataengine.datamanagement.domain.model.dataset.Dataset;
4-
import com.dataengine.datamanagement.domain.model.dataset.StatusConstants;
54
import com.dataengine.datamanagement.domain.model.dataset.Tag;
65
import com.dataengine.datamanagement.infrastructure.persistence.mapper.DatasetFileMapper;
76
import com.dataengine.datamanagement.infrastructure.persistence.mapper.DatasetMapper;
87
import com.dataengine.datamanagement.infrastructure.persistence.mapper.TagMapper;
8+
import com.dataengine.datamanagement.interfaces.converter.DatasetConverter;
99
import com.dataengine.datamanagement.interfaces.dto.AllDatasetStatisticsResponse;
10+
import com.dataengine.datamanagement.interfaces.dto.CreateDatasetRequest;
11+
import com.dataengine.datamanagement.interfaces.dto.DatasetPagingQuery;
12+
import org.apache.commons.collections4.CollectionUtils;
1013
import org.apache.ibatis.session.RowBounds;
1114
import org.springframework.beans.factory.annotation.Autowired;
15+
import org.springframework.beans.factory.annotation.Value;
1216
import org.springframework.data.domain.Page;
1317
import org.springframework.data.domain.PageImpl;
14-
import org.springframework.data.domain.Pageable;
18+
import org.springframework.data.domain.PageRequest;
1519
import org.springframework.stereotype.Service;
1620
import org.springframework.transaction.annotation.Transactional;
1721

1822
import java.time.LocalDateTime;
1923
import java.util.*;
20-
import java.util.stream.Collectors;
2124

2225
/**
2326
* 数据集应用服务（对齐 DB schema，使用 UUID 字符串主键）
@@ -30,6 +33,9 @@ public class DatasetApplicationService {
3033
private final TagMapper tagMapper;
3134
private final DatasetFileMapper datasetFileMapper;
3235

36+
@Value("${dataset.base.path:/dataset}")
37+
private String datasetBasePath;
38+
3339
@Autowired
3440
public DatasetApplicationService(DatasetMapper datasetMapper, TagMapper tagMapper, DatasetFileMapper datasetFileMapper) {
3541
this.datasetMapper = datasetMapper;
@@ -40,32 +46,21 @@ public DatasetApplicationService(DatasetMapper datasetMapper, TagMapper tagMappe
4046
/**
4147
* 创建数据集
4248
*/
43-
public Dataset createDataset(String name, String description, String datasetType,
44-
List<String> tagNames, Long dataSourceId,
45-
String path, String format, String createdBy) {
46-
if (datasetMapper.findByName(name) != null) {
47-
throw new IllegalArgumentException("Dataset with name '" + name + "' already exists");
49+
@Transactional
50+
public Dataset createDataset(CreateDatasetRequest createDatasetRequest) {
51+
if (datasetMapper.findByName(createDatasetRequest.getName()) != null) {
52+
throw new IllegalArgumentException("Dataset with name '" + createDatasetRequest.getName() + "' already exists");
4853
}
4954

50-
Dataset dataset = new Dataset();
51-
dataset.setId(UUID.randomUUID().toString());
52-
dataset.setName(name);
53-
dataset.setDescription(description);
54-
dataset.setDatasetType(datasetType);
55-
dataset.setDataSourceId(dataSourceId);
56-
dataset.setPath(path);
57-
dataset.setFormat(format);
58-
dataset.setStatus(StatusConstants.DatasetStatuses.ACTIVE);
59-
dataset.setCreatedBy(createdBy);
60-
dataset.setUpdatedBy(createdBy);
61-
dataset.setCreatedAt(LocalDateTime.now());
62-
dataset.setUpdatedAt(LocalDateTime.now());
63-
datasetMapper.insert(dataset); // 手动设定 UUID 主键
55+
// 创建数据集对象
56+
Dataset dataset = DatasetConverter.INSTANCE.convertToDataset(createDatasetRequest);
57+
dataset.initCreateParam(datasetBasePath);
58+
datasetMapper.insert(dataset);
6459

6560
// 处理标签
6661
Set<Tag> processedTags = new HashSet<>();
67-
if (tagNames != null && !tagNames.isEmpty()) {
68-
processedTags = processTagNames(tagNames);
62+
if (CollectionUtils.isNotEmpty(createDatasetRequest.getTags())) {
63+
processedTags = processTagNames(createDatasetRequest.getTags());
6964
for (Tag t : processedTags) {
7065
tagMapper.insertDatasetTag(dataset.getId(), t.getId());
7166
}
@@ -150,23 +145,21 @@ public Dataset getDataset(String datasetId) {
150145
* 分页查询数据集
151146
*/
152147
@Transactional(readOnly = true)
153-
public Page<Dataset> getDatasets(String typeCode, String status, String keyword,
154-
List<String> tagNames, Pageable pageable) {
155-
RowBounds bounds = new RowBounds(pageable.getPageNumber() * pageable.getPageSize(), pageable.getPageSize());
156-
List<Dataset> content = datasetMapper.findByCriteria(typeCode, status, keyword, tagNames, bounds);
148+
public Page<Dataset> getDatasets(DatasetPagingQuery query) {
149+
RowBounds bounds = new RowBounds(query.getPage() * query.getSize(), query.getSize());
150+
List<Dataset> content = datasetMapper.findByCriteria(query.getType(), query.getStatus(), query.getKeyword(), query.getTagList(), bounds);
151+
long total = datasetMapper.countByCriteria(query.getType(), query.getStatus(), query.getKeyword(), query.getTagList());
157152

158153
// 为每个数据集填充标签信息
159-
if (content != null && !content.isEmpty()) {
154+
if (CollectionUtils.isNotEmpty(content)) {
160155
for (Dataset dataset : content) {
161156
List<Tag> tags = tagMapper.findByDatasetId(dataset.getId());
162157
if (tags != null) {
163158
dataset.getTags().addAll(tags);
164159
}
165160
}
166161
}
167-
168-
long total = datasetMapper.countByCriteria(typeCode, status, keyword, tagNames);
169-
return new PageImpl<>(content, pageable, total);
162+
return new PageImpl<>(content, PageRequest.of(query.getPage(), query.getSize()), total);
170163
}
171164

172165
/**

‎backend/services/data-management-service/src/main/java/com/dataengine/datamanagement/application/service/DatasetFileApplicationService.java‎

Lines changed: 80 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,21 @@
11
package com.dataengine.datamanagement.application.service;
22

3+
import com.dataengine.common.domain.model.ChunkUploadPreRequest;
4+
import com.dataengine.common.domain.model.FileUploadResult;
5+
import com.dataengine.common.domain.service.FileService;
6+
import com.dataengine.common.domain.utils.AnalyzerUtils;
7+
import com.dataengine.datamanagement.domain.contants.DatasetConstant;
38
import com.dataengine.datamanagement.domain.model.dataset.Dataset;
49
import com.dataengine.datamanagement.domain.model.dataset.DatasetFile;
10+
import com.dataengine.datamanagement.domain.model.dataset.DatasetFileUploadCheckInfo;
511
import com.dataengine.datamanagement.domain.model.dataset.StatusConstants;
612
import com.dataengine.datamanagement.infrastructure.persistence.mapper.DatasetFileMapper;
713
import com.dataengine.datamanagement.infrastructure.persistence.mapper.DatasetMapper;
14+
import com.dataengine.datamanagement.interfaces.converter.DatasetConverter;
15+
import com.dataengine.datamanagement.interfaces.dto.UploadFileRequest;
16+
import com.dataengine.datamanagement.interfaces.dto.UploadFilesPreRequest;
17+
import com.fasterxml.jackson.core.JsonProcessingException;
18+
import com.fasterxml.jackson.databind.ObjectMapper;
819
import org.apache.ibatis.session.RowBounds;
920
import org.springframework.beans.factory.annotation.Autowired;
1021
import org.springframework.beans.factory.annotation.Value;
@@ -17,6 +28,7 @@
1728
import org.springframework.transaction.annotation.Transactional;
1829
import org.springframework.web.multipart.MultipartFile;
1930

31+
import java.io.File;
2032
import java.io.IOException;
2133
import java.net.MalformedURLException;
2234
import java.nio.file.Files;
@@ -25,6 +37,7 @@
2537
import java.nio.file.StandardCopyOption;
2638
import java.time.LocalDateTime;
2739
import java.util.List;
40+
import java.util.Objects;
2841
import java.util.UUID;
2942

3043
/**
@@ -37,14 +50,19 @@ public class DatasetFileApplicationService {
3750
private final DatasetFileMapper datasetFileMapper;
3851
private final DatasetMapper datasetMapper;
3952
private final Path fileStorageLocation;
53+
private final FileService fileService;
54+
55+
@Value("${dataset.base.path:/dataset}")
56+
private String datasetBasePath;
4057

4158
@Autowired
4259
public DatasetFileApplicationService(DatasetFileMapper datasetFileMapper,
43-
DatasetMapper datasetMapper,
60+
DatasetMapper datasetMapper, FileService fileService,
4461
@Value("${app.file.upload-dir:./uploads}") String uploadDir) {
4562
this.datasetFileMapper = datasetFileMapper;
4663
this.datasetMapper = datasetMapper;
4764
this.fileStorageLocation = Paths.get(uploadDir).toAbsolutePath().normalize();
65+
this.fileService = fileService;
4866
try {
4967
Files.createDirectories(this.fileStorageLocation);
5068
} catch (Exception ex) {
@@ -170,4 +188,65 @@ private String getFileExtension(String fileName) {
170188
}
171189
return fileName.substring(lastDotIndex + 1);
172190
}
191+
192+
/**
193+
* 预上传
194+
*
195+
* @param chunkUploadRequest 上传请求
196+
* @param datasetId 数据集id
197+
* @return 请求id
198+
*/
199+
@Transactional
200+
public String preUpload(UploadFilesPreRequest chunkUploadRequest, String datasetId) {
201+
ChunkUploadPreRequest request = ChunkUploadPreRequest.builder().build();
202+
request.setUploadPath(datasetBasePath + File.separator + datasetId);
203+
request.setTotalFileNum(chunkUploadRequest.getTotalFileNum());
204+
request.setServiceId(DatasetConstant.SERVICE_ID);
205+
DatasetFileUploadCheckInfo checkInfo = new DatasetFileUploadCheckInfo();
206+
checkInfo.setDatasetId(datasetId);
207+
checkInfo.setHasArchive(chunkUploadRequest.isHasArchive());
208+
try {
209+
ObjectMapper objectMapper = new ObjectMapper();
210+
String checkInfoJson = objectMapper.writeValueAsString(checkInfo);
211+
request.setCheckInfo(checkInfoJson);
212+
} catch (JsonProcessingException e) {
213+
throw new IllegalArgumentException("Failed to serialize checkInfo to JSON", e);
214+
}
215+
return fileService.preUpload(request);
216+
}
217+
218+
/**
219+
* 切片上传
220+
*
221+
* @param uploadFileRequest 上传请求
222+
*/
223+
public void chunkUpload(String datasetId, UploadFileRequest uploadFileRequest) {
224+
FileUploadResult uploadResult = fileService.chunkUpload(DatasetConverter.INSTANCE.toChunkUploadRequest(uploadFileRequest));
225+
saveFileInfoToDb(uploadResult, uploadFileRequest, datasetId);
226+
if (uploadResult.isAllFilesUploaded()) {
227+
// 解析文件，后续依据需求看是否添加校验文件元数据和解析半结构化文件的逻辑，
228+
}
229+
}
230+
231+
private void saveFileInfoToDb(FileUploadResult fileUploadResult, UploadFileRequest uploadFile, String datasetId) {
232+
if (Objects.isNull(fileUploadResult.getSavedFile())) {
233+
// 文件切片上传没有完成
234+
return;
235+
}
236+
File savedFile = fileUploadResult.getSavedFile();
237+
LocalDateTime currentTime = LocalDateTime.now();
238+
DatasetFile datasetFile = DatasetFile.builder()
239+
.id(UUID.randomUUID().toString())
240+
.datasetId(datasetId)
241+
.fileSize(savedFile.length())
242+
.uploadTime(currentTime)
243+
.lastAccessTime(currentTime)
244+
.lastAccessTime(currentTime)
245+
.fileName(uploadFile.getFileName())
246+
.filePath(savedFile.getPath())
247+
.fileType(AnalyzerUtils.getExtension(uploadFile.getFileName()))
248+
.build();
249+
250+
datasetFileMapper.insert(datasetFile);
251+
}
173252
}

‎backend/services/data-management-service/src/main/java/com/dataengine/datamanagement/domain/contants/DatasetConstant.java‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
package com.dataengine.datamanagement.domain.contants;
2+
3+
/**
4+
* 数据集常量
5+
*/
6+
public interface DatasetConstant {
7+
/**
8+
* 服务ID
9+
*/
10+
String SERVICE_ID = "DATA_MANAGEMENT";
11+
}

0 commit comments

Comments
 (0)