Skip to content

Commit a1199ed

Browse files
jsbjfkbsjkjsbjfkbsjk
authored
优化Excel读取内存占用 (#409)
* 把提取分析excel文件的功能做了插件化处理 * 更正注释，格式和目录 * 更正格式 * 增加序列化能力和修改格式 * 修改格式和健壮性处理 * Update FileExtractor.java 格式化 * 修改格式 * Update FileExtractorContainer.java
1 parent 4deb1f6 commit a1199ed

File tree

14 files changed

+591
-70
lines changed

14 files changed

+591
-70
lines changed

‎app-builder/plugins/aipp-file-extract-excel/pom.xml‎

Lines changed: 102 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,102 @@
1+
<?xml version="1.0" encoding="UTF-8"?>
2+
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
3+
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
4+
<modelVersion>4.0.0</modelVersion>
5+
6+
<parent>
7+
<groupId>modelengine.fit.jade</groupId>
8+
<artifactId>app-builder-plugin-parent</artifactId>
9+
<version>1.0.0-SNAPSHOT</version>
10+
</parent>
11+
12+
<groupId>modelengine.fit.jade.plugin</groupId>
13+
<artifactId>aipp-file-extract-excel</artifactId>
14+
15+
<dependencies>
16+
<!-- FIT -->
17+
<dependency>
18+
<groupId>org.fitframework</groupId>
19+
<artifactId>fit-api</artifactId>
20+
</dependency>
21+
<dependency>
22+
<groupId>org.fitframework</groupId>
23+
<artifactId>fit-util</artifactId>
24+
</dependency>
25+
26+
<!-- fast excel -->
27+
<dependency>
28+
<groupId>cn.idev.excel</groupId>
29+
<artifactId>fastexcel</artifactId>
30+
</dependency>
31+
32+
<!-- Services -->
33+
<dependency>
34+
<groupId>modelengine.fit.jade</groupId>
35+
<artifactId>aipp-file-extract-service</artifactId>
36+
</dependency>
37+
<dependency>
38+
<groupId>modelengine.fit.jade</groupId>
39+
<artifactId>aipp-service</artifactId>
40+
</dependency>
41+
42+
<!-- Tests -->
43+
<dependency>
44+
<groupId>org.junit.jupiter</groupId>
45+
<artifactId>junit-jupiter</artifactId>
46+
<scope>test</scope>
47+
</dependency>
48+
<dependency>
49+
<groupId>org.fitframework</groupId>
50+
<artifactId>fit-test-framework</artifactId>
51+
<scope>test</scope>
52+
</dependency>
53+
<dependency>
54+
<groupId>org.assertj</groupId>
55+
<artifactId>assertj-core</artifactId>
56+
<scope>test</scope>
57+
</dependency>
58+
</dependencies>
59+
60+
<build>
61+
<plugins>
62+
<plugin>
63+
<groupId>org.fitframework</groupId>
64+
<artifactId>fit-build-maven-plugin</artifactId>
65+
<version>${fit.version}</version>
66+
<executions>
67+
<execution>
68+
<id>build-plugin</id>
69+
<goals>
70+
<goal>build-plugin</goal>
71+
</goals>
72+
</execution>
73+
<execution>
74+
<id>package-plugin</id>
75+
<goals>
76+
<goal>package-plugin</goal>
77+
</goals>
78+
</execution>
79+
</executions>
80+
</plugin>
81+
<plugin>
82+
<groupId>org.apache.maven.plugins</groupId>
83+
<artifactId>maven-antrun-plugin</artifactId>
84+
<version>${maven.antrun.version}</version>
85+
<executions>
86+
<execution>
87+
<phase>install</phase>
88+
<configuration>
89+
<target>
90+
<copy file="${project.build.directory}/${project.build.finalName}.jar"
91+
todir="../../../build/plugins"/>
92+
</target>
93+
</configuration>
94+
<goals>
95+
<goal>run</goal>
96+
</goals>
97+
</execution>
98+
</executions>
99+
</plugin>
100+
</plugins>
101+
</build>
102+
</project>

‎app-builder/plugins/aipp-file-extract-excel/src/main/java/modelengine/fit/jade/aipp/file/extract/ExcelFileExtractor.java‎

Lines changed: 189 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,189 @@
1+
/*---------------------------------------------------------------------------------------------
2+
* Copyright (c) 2025 Huawei Technologies Co., Ltd. All rights reserved.
3+
* This file is a part of the ModelEngine Project.
4+
* Licensed under the MIT License. See License.txt in the project root for license information.
5+
*--------------------------------------------------------------------------------------------*/
6+
7+
package modelengine.fit.jade.aipp.file.extract;
8+
9+
import cn.idev.excel.ExcelReader;
10+
import cn.idev.excel.FastExcel;
11+
import cn.idev.excel.context.AnalysisContext;
12+
import cn.idev.excel.converters.Converter;
13+
import cn.idev.excel.enums.CellDataTypeEnum;
14+
import cn.idev.excel.metadata.GlobalConfiguration;
15+
import cn.idev.excel.metadata.data.DataFormatData;
16+
import cn.idev.excel.metadata.data.ReadCellData;
17+
import cn.idev.excel.metadata.property.ExcelContentProperty;
18+
import cn.idev.excel.read.listener.ReadListener;
19+
import cn.idev.excel.read.metadata.ReadSheet;
20+
import cn.idev.excel.util.DateUtils;
21+
import cn.idev.excel.util.StringUtils;
22+
import lombok.NonNull;
23+
import modelengine.fit.jober.aipp.service.OperatorService;
24+
import modelengine.fitframework.annotation.Component;
25+
import modelengine.fitframework.annotation.Fitable;
26+
27+
import java.io.BufferedInputStream;
28+
import java.io.File;
29+
import java.io.IOException;
30+
import java.io.InputStream;
31+
import java.math.BigDecimal;
32+
import java.nio.file.Files;
33+
import java.nio.file.InvalidPathException;
34+
import java.nio.file.Path;
35+
import java.nio.file.Paths;
36+
import java.text.SimpleDateFormat;
37+
import java.util.Arrays;
38+
import java.util.Date;
39+
import java.util.List;
40+
import java.util.Map;
41+
import java.util.stream.Collectors;
42+
43+
/**
44+
* Excel文件的提取器。
45+
*
46+
* @author 黄政炫
47+
* @since 2025-09-06
48+
*/
49+
@Component
50+
public class ExcelFileExtractor implements FileExtractor {
51+
/**
52+
* 把单元格转换成格式化字符串。
53+
*
54+
* @param cell 表示单元格数据 {@link ReadCellData}。
55+
* @return 转换后的内容 {@link String}。
56+
*/
57+
private static String getCellValueAsString(@NonNull ReadCellData<?> cell) {
58+
switch (cell.getType()) {
59+
case STRING:
60+
return cell.getStringValue();
61+
case NUMBER:
62+
DataFormatData fmt = cell.getDataFormatData();
63+
if (DateUtils.isADateFormat(fmt.getIndex(), fmt.getFormat())) {
64+
double value = cell.getNumberValue().doubleValue();
65+
Date date = DateUtils.getJavaDate(value, true);
66+
return new SimpleDateFormat("yyyy-MM-dd").format(date);
67+
} else {
68+
BigDecimal num = cell.getNumberValue();
69+
return num.stripTrailingZeros().toPlainString();
70+
}
71+
case BOOLEAN:
72+
return Boolean.toString(cell.getBooleanValue());
73+
default:
74+
return "";
75+
}
76+
}
77+
78+
/**
79+
* 该文件提取器支持EXCEL和CSV类型。
80+
*
81+
* @return 支持的枚举常量类型列表 {@link List}{@code <}{@link String}{@code >}。
82+
*/
83+
@Override
84+
@Fitable(id = "get-fileType-excel")
85+
public List<String> supportedFileTypes() {
86+
return Arrays.asList(OperatorService.FileType.EXCEL.toString(), OperatorService.FileType.CSV.toString());
87+
}
88+
89+
/**
90+
* 判断文件路径是否有效
91+
*
92+
* @param fileUrl 表示文件路径 {@link String}。
93+
* @return 表示路径是否有效 {@code boolean}。
94+
*/
95+
private boolean isValidPath(String fileUrl) {
96+
try {
97+
Path path = Paths.get(fileUrl);
98+
return Files.exists(path) && Files.isRegularFile(path);
99+
} catch (InvalidPathException e) {
100+
return false;
101+
}
102+
}
103+
104+
/**
105+
* 从指定路径的 Excel 文件中提取内容，并返回为字符串形式。
106+
*
107+
* @param fileUrl 表示文件路径的 {@link String}。
108+
* @return 表示文件内容的 {@link String}。
109+
*/
110+
@Override
111+
@Fitable(id = "extract-file-excel")
112+
public String extractFile(String fileUrl) {
113+
if (!isValidPath(fileUrl)) {
114+
throw new IllegalArgumentException(String.format("Invalid FilePath. [fileUrl=%s]", fileUrl));
115+
}
116+
File file = Paths.get(fileUrl).toFile();
117+
StringBuilder excelContent = new StringBuilder();
118+
ExcelReadListener listener = new ExcelReadListener(excelContent);
119+
ExcelReader reader = null;
120+
try (InputStream is = new BufferedInputStream(Files.newInputStream(file.toPath()))) {
121+
reader = FastExcel.read(is, listener)
122+
.registerConverter(new CustomCellStringConverter())
123+
.headRowNumber(0)
124+
.build();
125+
126+
List<ReadSheet> sheets = reader.excelExecutor().sheetList();
127+
for (ReadSheet meta : sheets) {
128+
excelContent.append("Sheet ").append(meta.getSheetNo() + 1).append(':').append('\n');
129+
ReadSheet readSheet = FastExcel.readSheet(meta.getSheetNo()).headRowNumber(0).build();
130+
reader.read(readSheet);
131+
}
132+
excelContent.append('\n');
133+
} catch (IOException e) {
134+
throw new IllegalStateException(String.format("Fail to extract excel file. [exception=%s]", e.getMessage()),
135+
e);
136+
} finally {
137+
if (reader != null) {
138+
reader.finish(); // 关闭资源
139+
}
140+
}
141+
return excelContent.toString();
142+
}
143+
144+
/**
145+
* 读取监听器的内部类实现。
146+
*/
147+
private class ExcelReadListener implements ReadListener<Map<Integer, String>> {
148+
private final StringBuilder excelContent;
149+
150+
ExcelReadListener(StringBuilder excelContent) {
151+
this.excelContent = excelContent;
152+
}
153+
154+
@Override
155+
public void invoke(Map<Integer, String> data, AnalysisContext context) {
156+
String line = data.entrySet()
157+
.stream()
158+
.sorted(Map.Entry.comparingByKey())
159+
.map(e -> e.getValue() == null ? "" : e.getValue())
160+
.collect(Collectors.joining("\t"));
161+
this.excelContent.append(line).append('\n');
162+
}
163+
164+
@Override
165+
public void doAfterAllAnalysed(AnalysisContext context) {}
166+
}
167+
168+
/**
169+
* 自定义单元格数据转换器。
170+
* 该转换器实现了能够处理单元格数据并将其转换为字符串形式。
171+
*/
172+
public static class CustomCellStringConverter implements Converter<String> {
173+
@Override
174+
public Class<String> supportJavaTypeKey() {
175+
return String.class;
176+
}
177+
178+
@Override
179+
public CellDataTypeEnum supportExcelTypeKey() {
180+
return null;
181+
}
182+
183+
@Override
184+
public String convertToJavaData(ReadCellData<?> cellData, ExcelContentProperty contentProperty,
185+
GlobalConfiguration globalConfiguration) {
186+
return (cellData != null) ? getCellValueAsString(cellData) : StringUtils.EMPTY;
187+
}
188+
}
189+
}

‎app-builder/plugins/aipp-file-extract-excel/src/main/resources/application.yml‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
fit:
2+
beans:
3+
packages:
4+
- 'modelengine.fit.jade.aipp.file.extract'

‎app-builder/plugins/aipp-file-extract-excel/src/test/java/modelengine/fit/jade/aipp/file/extract/ExcelFileExtractorTest.java‎

Lines changed: 72 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,72 @@
1+
/*---------------------------------------------------------------------------------------------
2+
* Copyright (c) 2025 Huawei Technologies Co., Ltd. All rights reserved.
3+
* This file is a part of the ModelEngine Project.
4+
* Licensed under the MIT License. See License.txt in the project root for license information.
5+
*--------------------------------------------------------------------------------------------*/
6+
7+
package modelengine.fit.jade.aipp.file.extract;
8+
9+
import static org.assertj.core.api.Assertions.assertThat;
10+
import static org.junit.jupiter.api.Assertions.assertThrows;
11+
12+
import modelengine.fit.jober.aipp.service.OperatorService;
13+
import modelengine.fitframework.annotation.Fit;
14+
import modelengine.fitframework.test.annotation.FitTestWithJunit;
15+
16+
import org.junit.jupiter.api.DisplayName;
17+
import org.junit.jupiter.api.Test;
18+
19+
import java.io.File;
20+
import java.util.Arrays;
21+
import java.util.List;
22+
23+
/**
24+
* 表示{@link ExcelFileExtractor}的测试集。
25+
*
26+
* @author 黄政炫
27+
* @since 2025-09-06
28+
*/
29+
@FitTestWithJunit(includeClasses = ExcelFileExtractor.class)
30+
class ExcelFileExtractorTest {
31+
@Fit
32+
ExcelFileExtractor excelFileExtractor;
33+
34+
@Test
35+
@DisplayName("测试获取支持文件类型")
36+
void supportedFileType() {
37+
List<String> supportedTypes =
38+
Arrays.asList(OperatorService.FileType.EXCEL.toString(), OperatorService.FileType.CSV.toString());
39+
assertThat(this.excelFileExtractor.supportedFileTypes()).isEqualTo(supportedTypes);
40+
}
41+
42+
@Test
43+
@DisplayName("测试能否捕获错误路径")
44+
void validPath() {
45+
assertThrows(IllegalArgumentException.class, () -> {
46+
this.excelFileExtractor.extractFile("invalidPath.csv");
47+
});
48+
}
49+
50+
@Test
51+
@DisplayName("测试 excel 文件提取成功")
52+
void extractFile() {
53+
File file = new File(this.getClass().getClassLoader().getResource("file/content.csv").getFile());
54+
String expected = """
55+
Sheet 1:
56+
This is an excel test
57+
ID\tName\tAge\tJoinDate\tActive\tSalary\tDepartment\tNotes
58+
1\tJohn Doe\t25\t2023-01-15\tTRUE\t8000.50\tIT\tRegular employee
59+
2\tJane Smith\t30\t2022-05-20\tTRUE\t12000.00\tMarketing\tTeam leader
60+
3\tBob Johnson\t28\t2023-03-10\tFALSE\t7500.00\tSales\tLeft company
61+
4\tAlice Brown\t35\t2020-12-01\tTRUE\t15000.75\tIT\tSenior engineer
62+
5\tTom Wilson\t22\t2023-08-25\tTRUE\t6000.00\tHR\tIntern
63+
6\t\t40\t2019-06-15\tTRUE\t18000.00\tFinance\tDepartment manager
64+
7\tLucy Davis\t27\t2023-02-28\tFALSE\t7000.00\tOperations\tContract ended
65+
8\tMike Miller\t32\t2021-09-10\tTRUE\t13500.50\tIT\tProject lead
66+
9\tSarah Lee\t29\t2022-11-05\tTRUE\t9500.00\tMarketing\tMarketing specialist
67+
10\tDavid Zhang\t26\t2023-07-12\tTRUE\t8500.25\tSales\tSales representative
68+
69+
""";
70+
assertThat(this.excelFileExtractor.extractFile(file.getAbsolutePath())).isEqualTo(expected);
71+
}
72+
}

‎app-builder/plugins/aipp-file-extract-excel/src/test/resources/file/content.csv‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
This is an excel test
2+
ID,Name,Age,JoinDate,Active,Salary,Department,Notes
3+
1,John Doe,25,2023-01-15,TRUE,8000.50,IT,"Regular employee"
4+
2,Jane Smith,30,2022-05-20,TRUE,12000.00,Marketing,"Team leader"
5+
3,Bob Johnson,28,2023-03-10,FALSE,7500.00,Sales,"Left company"
6+
4,Alice Brown,35,2020-12-01,TRUE,15000.75,IT,"Senior engineer"
7+
5,Tom Wilson,22,2023-08-25,TRUE,6000.00,HR,"Intern"
8+
6,,40,2019-06-15,TRUE,18000.00,Finance,"Department manager"
9+
7,Lucy Davis,27,2023-02-28,FALSE,7000.00,Operations,"Contract ended"
10+
8,Mike Miller,32,2021-09-10,TRUE,13500.50,IT,"Project lead"
11+
9,Sarah Lee,29,2022-11-05,TRUE,9500.00,Marketing,"Marketing specialist"
12+
10,David Zhang,26,2023-07-12,TRUE,8500.25,Sales,"Sales representative"

0 commit comments

Comments
 (0)