部分重构

Author: jsbjfkbsjk

app-builder/plugins/aipp-plugin/src/main/java/modelengine/fit/jober/aipp/service/impl/OperatorServiceImpl.java

@@ -6,22 +6,11 @@
 
 package modelengine.fit.jober.aipp.service.impl;
 
-import cn.idev.excel.ExcelReader;
-import cn.idev.excel.FastExcel;
-import cn.idev.excel.context.AnalysisContext;
-import cn.idev.excel.converters.Converter;
-import cn.idev.excel.enums.CellDataTypeEnum;
-import cn.idev.excel.metadata.GlobalConfiguration;
-import cn.idev.excel.metadata.data.DataFormatData;
-import cn.idev.excel.metadata.data.ReadCellData;
-import cn.idev.excel.metadata.property.ExcelContentProperty;
-import cn.idev.excel.read.listener.ReadListener;
-import cn.idev.excel.read.metadata.ReadSheet;
-import cn.idev.excel.util.DateUtils;
 import modelengine.fit.jober.aipp.common.exception.AippErrCode;
 import modelengine.fit.jober.aipp.common.exception.AippException;
 import modelengine.fit.jober.aipp.service.LLMService;
 import modelengine.fit.jober.aipp.service.OperatorService;
+import modelengine.fit.jober.aipp.tool.FileExtractorContainer;
 import modelengine.fit.jober.aipp.util.AippFileUtils;
 import modelengine.fit.jober.aipp.util.AippStringUtils;
 import modelengine.fitframework.annotation.Component;
@@ -32,12 +21,7 @@
 import org.apache.pdfbox.pdmodel.PDDocument;
 import org.apache.pdfbox.text.PDFTextStripper;
 import org.apache.poi.poifs.filesystem.FileMagic;
-import org.apache.poi.ss.usermodel.Cell;
-import org.apache.poi.ss.usermodel.DateUtil;
-import org.apache.poi.ss.usermodel.Row;
-import org.apache.poi.ss.usermodel.Sheet;
-import org.apache.poi.ss.usermodel.Workbook;
-import org.apache.poi.xssf.usermodel.XSSFWorkbook;
+
 import org.apache.poi.xwpf.extractor.XWPFWordExtractor;
 import org.apache.poi.xwpf.usermodel.XWPFDocument;
 import org.apache.poi.xwpf.usermodel.XWPFParagraph;
@@ -46,13 +30,10 @@
 import org.openxmlformats.schemas.wordprocessingml.x2006.main.CTHdrFtr;
 
 import java.io.*;
-import java.math.BigDecimal;
 import java.nio.charset.StandardCharsets;
 import java.nio.file.Files;
 import java.nio.file.Paths;
-import java.text.SimpleDateFormat;
 import java.util.*;
-import java.util.concurrent.ConcurrentHashMap;
 import java.util.function.Function;
 import java.util.stream.Collectors;
 
@@ -99,7 +80,6 @@ public class OperatorServiceImpl implements OperatorService {
     private final LLMService llmService;
     private final BrokerClient client;
     private final Function<String, String> pdfExtractor = this::extractPdfFile;
-    private final Function<String, String> excelExtractor = this::extractExcelFile;
     private final Function<String, String> wordExtractor = this::extractWordFile;
     private final Function<String, String> textExtractor = this::extractTextFile;
     private final EnumMap<FileType, Function<File, String>> outlineOperatorMap =
@@ -109,45 +89,24 @@ public class OperatorServiceImpl implements OperatorService {
                 }
             };
 
-    private final EnumMap<FileType, Function<String, String>> fileOperatorMap
-            = new EnumMap<FileType, Function<String, String>>(FileType.class) {
-        {
-            put(FileType.PDF, pdfExtractor);
-            put(FileType.WORD, wordExtractor);
-            put(FileType.EXCEL, excelExtractor);
-            put(FileType.TXT, textExtractor);
-            put(FileType.HTML, textExtractor);
-            put(FileType.MARKDOWN, textExtractor);
-            put(FileType.CSV, textExtractor);
-        }
-    };
+    private final EnumMap<FileType, Function<String, String>> fileOperatorMap =
+            new EnumMap<FileType, Function<String, String>>(FileType.class) {
+                {
+                    put(FileType.PDF, pdfExtractor);
+                    put(FileType.WORD, wordExtractor);
+                    put(FileType.TXT, textExtractor);
+                    put(FileType.HTML, textExtractor);
+                    put(FileType.MARKDOWN, textExtractor);
+                    put(FileType.CSV, textExtractor);
+                }
+            };
+    private final FileExtractorContainer fileExtractorContainer;
 
-    public OperatorServiceImpl(LLMService llmService, BrokerClient client) {
+    public OperatorServiceImpl(LLMService llmService, BrokerClient client,
+            FileExtractorContainer fileExtractorContainer) {
         this.llmService = llmService;
         this.client = client;
-    }
-
-    private static String getCellValueAsString(ReadCellData<?> cell) {
-        switch (cell.getType()) {
-            case STRING:
-                return cell.getStringValue();
-            case NUMBER:
-                DataFormatData fmt = cell.getDataFormatData();
-                short formatIndex = fmt.getIndex();
-                String formatString = fmt.getFormat();
-                if (DateUtils.isADateFormat(formatIndex,formatString)) {
-                    double value = cell.getNumberValue().doubleValue();
-                    Date date = DateUtils.getJavaDate(value,true);
-                    return new SimpleDateFormat("yyyy-MM-dd").format(date);
-                } else {
-                    BigDecimal num = cell.getNumberValue();
-                    return num.stripTrailingZeros().toPlainString();
-                }
-            case BOOLEAN:
-                return Boolean.toString(cell.getBooleanValue());
-            default:
-                return "";
-        }
+        this.fileExtractorContainer = fileExtractorContainer;
     }
 
     private static String extractDocHandle(InputStream fis, String fileName) throws IOException {
@@ -250,61 +209,16 @@ public File createDoc(String instanceId, String fileName, String txt) throws IOE
      */
     public String fileExtractor(String fileUrl, Optional<FileType> optionalFileType) {
         if (optionalFileType.isPresent()) {
+            String res = fileExtractorContainer.extract(fileUrl, optionalFileType.get());
+            if (!res.isEmpty()) {
+                return res;
+            }
             Function<String, String> function = this.fileOperatorMap.get(optionalFileType.get());
             return Optional.ofNullable(function).map(f -> f.apply(fileUrl)).orElse(StringUtils.EMPTY);
         }
         return this.extractTextFile(fileUrl);
     }
 
-    /**
-     * 从指定路径的 Excel 文件中提取内容，并返回为字符串形式。
-     * 实现方式：
-     * 基于 fast-excel 包，使用流式读取（ReadListener）逐行解析，避免一次性加载整表造成的内存开销。
-     * 每行数据会被转换为以制表符（\t）分隔的文本，并在行末追加换行符。
-     * 支持多 sheet 解析，会依次读取工作簿中的每一个 sheet。
-     *
-     * @param fileUrl 表示文件路径的 {@link String}.
-     * @return 表示文件内容的 {@link String}。
-     * @throws RuntimeException 当文件读取或解析失败时抛出
-     */
-    private String extractExcelFile(String fileUrl) {
-        File file = Paths.get(fileUrl).toFile();
-        StringBuilder excelContent = new StringBuilder();
-        ReadListener<Map<Integer, String>> listener = new ReadListener<>() {
-            @Override
-            public void invoke(Map<Integer, String> data, AnalysisContext context) {
-                String line = data.entrySet().stream()
-                        .sorted(Map.Entry.comparingByKey())
-                        .map(e -> e.getValue() == null ? "" : e.getValue())
-                        .collect(Collectors.joining("\t"));
-                excelContent.append(line).append('\n');
-            }
-            @Override
-            public void doAfterAllAnalysed(AnalysisContext context) {
-            }
-        };
-        try (InputStream is = new BufferedInputStream(Files.newInputStream(file.toPath()))) {
-            ExcelReader reader = FastExcel.read(is, listener)
-                    .registerConverter(new CustomCellStringConverter())
-                    .headRowNumber(0)
-                    .build();
-
-            List<ReadSheet> sheets = reader.excelExecutor().sheetList();
-            for (ReadSheet meta : sheets) {
-                excelContent.append("Sheet ").append(meta.getSheetNo() + 1).append(':').append('\n');
-                ReadSheet readSheet = FastExcel.readSheet(meta.getSheetNo())
-                        .headRowNumber(0)
-                        .build();
-                reader.read(readSheet);
-            }
-            excelContent.append('\n');
-            reader.finish(); // 关闭资源
-        } catch (IOException e) {
-            throw new RuntimeException(e);
-        }
-        return excelContent.toString();
-    }
-
     private String iterPdf(PDDocument doc) throws IOException {
         int pages = doc.getNumberOfPages();
         StringBuilder sb = new StringBuilder();
@@ -359,25 +273,4 @@ private String extractTextFile(String fileUrl) {
         }
     }
 
-    /**
-     * 自定义单元格数据转换器。
-     * 将 Excel 单元格数据统一转换为字符串，避免数值/日期等类型在读取时格式不一致的问题。
-     * 缺点：由于采用fast excel包,没有 FORMULA类,会将公式单元格自动计算为值
-     *
-     */
-    public static class CustomCellStringConverter implements Converter<String> {
-        @Override
-        public Class<String> supportJavaTypeKey() {
-            return String.class;
-        }
-        @Override
-        public CellDataTypeEnum supportExcelTypeKey() {
-            return null;
-        }
-        @Override
-        public String convertToJavaData(ReadCellData<?> cellData, ExcelContentProperty contentProperty,
-                                        GlobalConfiguration globalConfiguration) {
-            return getCellValueAsString(cellData);
-        }
-    }
 }

app-builder/plugins/aipp-plugin/src/main/java/modelengine/fit/jober/aipp/tool/FileExtractor.java

@@ -6,16 +6,18 @@
 
 package modelengine.fit.jober.aipp.tool;
 
+import modelengine.fel.tool.annotation.Group;
+import modelengine.fel.tool.annotation.ToolMethod;
+import modelengine.fit.jober.aipp.service.OperatorService;
 import modelengine.fitframework.annotation.Genericable;
 
-import java.io.File;
-
 /**
  * 文件内容提取
  *
  * @author 孙怡菲
  * @since 2024-06-08
  */
+@Group(name = "defGroup-aipp-file-extract-tool")
 public interface FileExtractor {
     /**
      * 文件提取genericable接口gid
@@ -25,9 +27,12 @@ public interface FileExtractor {
     /**
      * 提取文件内容
      *
-     * @param file 待提取的文件
+     * @param fileUrl 待提取的文件地址
      * @return 文件内容。
      */
+    @ToolMethod(name = "file_extract", description = "提取文件信息")
     @Genericable(FILE_EXTRACTOR_GID)
-    String extractFile(File file);
+    String extractFile(String fileUrl);
+
+    OperatorService.FileType supportedType();
 }

app-builder/plugins/aipp-plugin/src/main/java/modelengine/fit/jober/aipp/tool/FileExtractorContainer.java

@@ -0,0 +1,36 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) 2025 Huawei Technologies Co., Ltd. All rights reserved.
+ *  This file is a part of the ModelEngine Project.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+package modelengine.fit.jober.aipp.tool;
+
+import modelengine.fit.jober.aipp.service.OperatorService;
+import modelengine.fitframework.annotation.Component;
+
+import java.util.EnumMap;
+import java.util.List;
+import java.util.Map;
+
+@Component
+public class FileExtractorContainer {
+    private Map<OperatorService.FileType, FileExtractor> map;
+
+    public FileExtractorContainer(List<FileExtractor> extractors) {
+        map = new EnumMap<>(OperatorService.FileType.class);
+        for (FileExtractor fileExtractor : extractors) {
+            map.put(fileExtractor.supportedType(), fileExtractor);
+        }
+    }
+
+    public String extract(String fileUrl, OperatorService.FileType fileType) {
+        if (map.containsKey(fileType)) {
+            return map.get(fileType).extractFile(fileUrl);
+        } else {
+            return "";//先返回空字符串
+        }
+    }
+
+}
+

app-builder/plugins/aipp-plugin/src/main/java/modelengine/fit/jober/aipp/tool/impl/AudioExtractor.java

@@ -8,6 +8,7 @@
 
 import static modelengine.fit.jober.aipp.constant.AippConstant.NAS_SHARE_DIR;
 
+import modelengine.fel.tool.annotation.Group;
 import modelengine.fit.jober.aipp.common.exception.AippErrCode;
 import modelengine.fit.jober.aipp.common.exception.AippException;
 import modelengine.fit.jober.aipp.dto.audio.AudioSplitInfo;
@@ -16,6 +17,7 @@
 import modelengine.fit.jober.aipp.entity.ffmpeg.FfmpegMeta;
 import modelengine.fit.jober.aipp.enums.LlmModelNameEnum;
 import modelengine.fit.jober.aipp.service.FfmpegService;
+import modelengine.fit.jober.aipp.service.OperatorService;
 import modelengine.fit.jober.aipp.tool.FileExtractor;
 import modelengine.fit.jober.aipp.util.AippFileUtils;
 import modelengine.fit.jober.aipp.util.JsonUtils;
@@ -55,7 +57,13 @@
  * @since 2024/1/8
  */
 @Component
+@Group(name = "defGroup-aipp-file-extract-tool-audio")
 public class AudioExtractor implements FileExtractor {
+    @Override
+    public OperatorService.FileType supportedType() {
+        return OperatorService.FileType.AUDIO;
+    }
+
     private static final Logger log = Logger.get(AudioExtractor.class);
 
     private static final String PROMPT = "\nPerform the following actions:\n"
@@ -77,9 +85,8 @@ public class AudioExtractor implements FileExtractor {
     private final String pathPrefix;
     private final FfmpegService ffmpegService;
 
-    public AudioExtractor(FfmpegService ffmpegService, @Fit ChatModel openAiClient,
-                          @Fit VoiceService voiceService, @Value("${app-engine.endpoint}") String endpoint,
-                          @Value("${app-engine.pathPrefix}") String pathPrefix) {
+    public AudioExtractor(FfmpegService ffmpegService, @Fit ChatModel openAiClient, @Fit VoiceService voiceService,
+            @Value("${app-engine.endpoint}") String endpoint, @Value("${app-engine.pathPrefix}") String pathPrefix) {
         this.ffmpegService = ffmpegService;
         this.openAiClient = openAiClient;
         this.voiceService = voiceService;
@@ -97,12 +104,14 @@ private SummaryDto batchSummary(List<File> audioList, int segmentSize) throws In
             SUMMARY_EXECUTOR.execute(() -> {
                 try {
                     File audio = audioList.get(id);
-                    String audioPath = AippFileUtils.getFileDownloadFilePath(
-                            endpoint, this.pathPrefix, audio.getPath());
+                    String audioPath =
+                            AippFileUtils.getFileDownloadFilePath(endpoint, this.pathPrefix, audio.getPath());
                     log.info("audio filePath: {}, audio fileName: {}", audioPath, audio.getName());
                     String text = voiceService.getText(audioPath + "&fileName=" + audio.getName());
-                    String summary = LLMUtils.askModelForSummary(openAiClient, String.format(PROMPT, text),
-                            LlmModelNameEnum.QWEN_72B, 16000);
+                    String summary = LLMUtils.askModelForSummary(openAiClient,
+                            String.format(PROMPT, text),
+                            LlmModelNameEnum.QWEN_72B,
+                            16000);
                     output.set(id, summary);
                 } catch (IOException e) {
                     output.set(id, "");
@@ -126,8 +135,10 @@ private SummaryDto generateSummary(List<String> output, int segmentSize) {
         StringBuilder sb = new StringBuilder();
         summaryDto.getSectionList().forEach(sec -> sb.append(sec.getText()));
         try {
-            String llmOutput = LLMUtils.askModelForSummary(openAiClient, String.format(PROMPT, sb),
-                    LlmModelNameEnum.QWEN_72B, 16000);
+            String llmOutput = LLMUtils.askModelForSummary(openAiClient,
+                    String.format(PROMPT, sb),
+                    LlmModelNameEnum.QWEN_72B,
+                    16000);
             SummarySection section =
                     JsonUtils.parseObject(LLMUtils.tryFixLlmJsonString(llmOutput), SummarySection.class);
             summaryDto.setSummary(section.getText());
@@ -163,8 +174,9 @@ private AudioSplitInfo covertAudioSimple(String dirName, File audio) throws IOEx
 
     @Fitable("llmAudio2Summary")
     @Override
-    public String extractFile(File file) {
+    public String extractFile(String fileUrl) {
         // file -> audioDir 切分为多个音频文件，存在临时目录下
+        File file = Paths.get(fileUrl).toFile();
         String tmpDir = TMP_DIR_PREFIX + UUIDUtil.uuid();
         AudioSplitInfo audioSplitInfo;
         try {