feature: refactor cleaning template service for simplified pagination handling and enhance operator management

hhhhsc701 · hhhhsc701 · commit bedf8679c21f · 2025-10-20T20:17:19.000+08:00
diff --git a/backend/services/data-cleaning-service/src/main/java/com/dataengine/cleaning/application/service/CleaningTemplateService.java b/backend/services/data-cleaning-service/src/main/java/com/dataengine/cleaning/application/service/CleaningTemplateService.java
@@ -12,7 +12,6 @@
 import com.dataengine.cleaning.interfaces.dto.UpdateCleaningTemplateRequest;
 import lombok.RequiredArgsConstructor;
 import org.apache.commons.lang3.StringUtils;
-import org.apache.ibatis.annotations.Param;
 import org.springframework.stereotype.Service;
 import org.springframework.transaction.annotation.Transactional;
 
@@ -30,16 +29,11 @@ public class CleaningTemplateService {
 
     private final OperatorInstanceMapper operatorInstanceMapper;
 
-    public List<CleaningTemplate> getTemplates(String keywords, Integer page, Integer size) {
+    public List<CleaningTemplate> getTemplates(String keywords) {
         List<OperatorResponse> allOperators = cleaningTemplateMapper.findAllOperators();
         Map<String, OperatorResponse> operatorsMap = allOperators.stream()
                 .collect(Collectors.toMap(OperatorResponse::getId, Function.identity()));
-
-        Integer offset = null;
-        if (page != null && size != null) {
-            offset = page * size;
-        }
-        List<TemplateWithInstance> allTemplates = cleaningTemplateMapper.findAllTemplates(keywords, size, offset);
+        List<TemplateWithInstance> allTemplates = cleaningTemplateMapper.findAllTemplates(keywords);
         Map<String, List<TemplateWithInstance>> templatesMap = allTemplates.stream()
                 .collect(Collectors.groupingBy(TemplateWithInstance::getId));
         return templatesMap.entrySet().stream().map(twi -> {
@@ -63,10 +57,6 @@ public List<CleaningTemplate> getTemplates(String keywords, Integer page, Intege
         }).toList();
     }
 
-    public int countTemplates(String keywords) {
-        return cleaningTemplateMapper.findAllTemplates(keywords, null, null).size();
-    }
-
     @Transactional
     public CleaningTemplate createTemplate(CreateCleaningTemplateRequest request) {
         CleaningTemplate template = new CleaningTemplate();
diff --git a/backend/services/data-cleaning-service/src/main/java/com/dataengine/cleaning/infrastructure/persistence/mapper/CleaningTemplateMapper.java b/backend/services/data-cleaning-service/src/main/java/com/dataengine/cleaning/infrastructure/persistence/mapper/CleaningTemplateMapper.java
@@ -11,8 +11,7 @@
 @Mapper
 public interface CleaningTemplateMapper {
 
-    List<TemplateWithInstance> findAllTemplates(@Param("keywords") String keywords,
-                                                @Param("size") Integer size, @Param("offset") Integer offset);
+    List<TemplateWithInstance> findAllTemplates(@Param("keywords") String keywords);
 
     List<OperatorResponse> findAllOperators();
 
diff --git a/backend/services/data-cleaning-service/src/main/java/com/dataengine/cleaning/interfaces/api/CleaningTemplateController.java b/backend/services/data-cleaning-service/src/main/java/com/dataengine/cleaning/interfaces/api/CleaningTemplateController.java
@@ -18,6 +18,7 @@
 import org.springframework.web.bind.annotation.RequestParam;
 import org.springframework.web.bind.annotation.RestController;
 
+import java.util.Comparator;
 import java.util.List;
 
 
@@ -32,14 +33,17 @@ public ResponseEntity<Response<PagedResponse<CleaningTemplate>>> cleaningTemplat
             @RequestParam(value = "page", required = false) Integer page,
             @RequestParam(value = "size", required = false) Integer size,
             @RequestParam(value = "keywords", required = false) String keyword) {
-        List<CleaningTemplate> templates = cleaningTemplateService.getTemplates(keyword, page, size);
-        if (page != null && size != null) {
-            int count  = cleaningTemplateService.countTemplates(keyword);
-            int totalPages = (count + size + 1) / size;
-            return ResponseEntity.ok(Response.ok(PagedResponse.of(templates, page, count, totalPages)));
-        } else {
+        List<CleaningTemplate> templates = cleaningTemplateService.getTemplates(keyword);
+        if (page == null || size == null) {
             return ResponseEntity.ok(Response.ok(PagedResponse.of(templates)));
         }
+        int count = templates.size();
+        int totalPages = (count + size + 1) / size;
+        List<CleaningTemplate> limitTemplates = templates.stream()
+                .sorted(Comparator.comparing(CleaningTemplate::getCreatedAt))
+                .skip((long) page * size)
+                .limit(size).toList();
+        return ResponseEntity.ok(Response.ok(PagedResponse.of(limitTemplates, page, count, totalPages)));
     }
 
     @PostMapping
diff --git a/backend/services/data-cleaning-service/src/main/resources/mappers/CleaningTemplateMapper.xml b/backend/services/data-cleaning-service/src/main/resources/mappers/CleaningTemplateMapper.xml
@@ -11,9 +11,6 @@
             </if>
         </where>
         ORDER BY created_at
-        <if test="size != null and offset != null">
-            LIMIT ${size} OFFSET ${offset}
-        </if>
     </select>
 
     <select id="findAllOperators" resultType="com.dataengine.cleaning.interfaces.dto.OperatorResponse">
diff --git a/backend/services/operator-market-service/src/main/java/com/dataengine/operator/application/CategoryService.java b/backend/services/operator-market-service/src/main/java/com/dataengine/operator/application/CategoryService.java
@@ -35,7 +35,7 @@ public List<CategoryTreeResponse> getAllCategories() {
     private List<CategoryTreeResponse> groupByParentIdSorted(List<RelationCategoryDTO> relations,
                                                              Map<Integer, Category> categories) {
         Map<Integer, List<RelationCategoryDTO>> groupedByParentId = relations.stream()
-            .filter(relation -> relation.getParentId() > 0)
+            .filter(relation -> StringUtils.isNotBlank(relation.getOperatorId()))
             .collect(Collectors.groupingBy(RelationCategoryDTO::getParentId));
 
         return groupedByParentId.entrySet().stream()
diff --git a/backend/services/operator-market-service/src/main/java/com/dataengine/operator/application/OperatorService.java b/backend/services/operator-market-service/src/main/java/com/dataengine/operator/application/OperatorService.java
@@ -1,6 +1,7 @@
 package com.dataengine.operator.application;
 
 import com.dataengine.operator.domain.converter.OperatorConverter;
+import com.dataengine.operator.infrastructure.persistence.mapper.CategoryRelationMapper;
 import com.dataengine.operator.infrastructure.persistence.mapper.OperatorMapper;
 import com.dataengine.operator.interfaces.dto.*;
 import com.dataengine.operator.domain.modal.Operator;
@@ -15,6 +16,8 @@
 public class OperatorService {
     private final OperatorMapper operatorMapper;
 
+    private final CategoryRelationMapper relationMapper;
+
     public List<OperatorResponse> getOperators(Integer page, Integer size, List<Integer> categories,
                                                String operatorName, Boolean isStar) {
         Integer offset = page * size;
@@ -28,26 +31,41 @@ public int getOperatorsCount(List<Integer> categories, String operatorName, Bool
         return operatorMapper.countOperatorsByCriteria(operatorName, categories, isStar);
     }
 
-    private OperatorResponse toDto(Operator entity) {
-        OperatorResponse dto = new OperatorResponse();
-        dto.setId(entity.getId());
-        dto.setName(entity.getName());
-        dto.setDescription(entity.getDescription());
-        dto.setVersion(entity.getVersion());
-        return dto;
-    }
     public OperatorResponse getOperatorById(String id) {
-        // TODO: 查询算子详情
-        return new OperatorResponse();
-    }
-    public OperatorResponse updateOperator(String id, UpdateOperatorRequest req) {
-        // TODO: 更新算子
-        return new OperatorResponse();
+        Operator operator = operatorMapper.findOperatorById(id);
+        return OperatorConverter.INSTANCE.operatorToResponse(operator);
     }
+
     public OperatorResponse createOperator(CreateOperatorRequest req) {
-        // TODO: 创建算子
-        return new OperatorResponse();
+        Operator operator = new Operator();
+        operator.setId(req.getId());
+        operator.setName(req.getName());
+        operator.setDescription(req.getDescription());
+        operator.setVersion(req.getVersion());
+        operator.setInputs(req.getInputs());
+        operator.setOutputs(req.getOutputs());
+        operator.setRuntime(req.getRuntime());
+        operator.setSettings(req.getSettings());
+        operatorMapper.insertOperator(operator);
+        relationMapper.batchInsert(req.getId(), req.getCategories());
+        return OperatorConverter.INSTANCE.operatorToResponse(operator);
+    }
+
+    public OperatorResponse updateOperator(String id, UpdateOperatorRequest req) {
+        Operator operator = new Operator();
+        operator.setId(id);
+        operator.setName(req.getName());
+        operator.setDescription(req.getDescription());
+        operator.setVersion(req.getVersion());
+        operator.setInputs(req.getInputs());
+        operator.setOutputs(req.getOutputs());
+        operator.setRuntime(req.getRuntime());
+        operator.setSettings(req.getSettings());
+        operatorMapper.updateOperator(operator);
+        relationMapper.batchInsert(id, req.getCategories());
+        return getOperatorById(id);
     }
+
     public OperatorResponse uploadOperator(MultipartFile file, String description) {
         // TODO: 文件上传与解析
         return new OperatorResponse();
diff --git a/backend/services/operator-market-service/src/main/java/com/dataengine/operator/infrastructure/persistence/mapper/CategoryRelationMapper.java b/backend/services/operator-market-service/src/main/java/com/dataengine/operator/infrastructure/persistence/mapper/CategoryRelationMapper.java
@@ -2,6 +2,7 @@
 
 import com.dataengine.operator.domain.modal.RelationCategoryDTO;
 import org.apache.ibatis.annotations.Mapper;
+import org.apache.ibatis.annotations.Param;
 
 import java.util.List;
 
@@ -11,4 +12,6 @@ public interface CategoryRelationMapper {
     List<RelationCategoryDTO> findAllRelationWithCategory();
 
     List<RelationCategoryDTO> findFullOuterJoinNative();
+
+    void batchInsert(@Param("operatorId") String operatorId, @Param("categories") List<Integer> categories);
 }
diff --git a/backend/services/operator-market-service/src/main/java/com/dataengine/operator/infrastructure/persistence/mapper/OperatorMapper.java b/backend/services/operator-market-service/src/main/java/com/dataengine/operator/infrastructure/persistence/mapper/OperatorMapper.java
@@ -18,4 +18,10 @@ List<Operator> findOperatorsByCriteria(@Param("size") Integer size, @Param("offs
     Integer countOperatorsByCriteria(@Param("operatorName") String operatorName,
                                      @Param("categories") List<Integer> categories,
                                      @Param("isStar") Boolean isStar);
+
+    Operator findOperatorById(@Param("id") String id);
+
+    void updateOperator(Operator operator);
+
+    void insertOperator(Operator operator);
 }
diff --git a/backend/services/operator-market-service/src/main/java/com/dataengine/operator/interfaces/dto/CreateOperatorRequest.java b/backend/services/operator-market-service/src/main/java/com/dataengine/operator/interfaces/dto/CreateOperatorRequest.java
@@ -4,22 +4,33 @@
 import lombok.Getter;
 import lombok.Setter;
 
+import java.util.List;
+
 /**
  * CreateOperatorRequest
  */
 
 @Getter
 @Setter
 public class CreateOperatorRequest {
+  private String id;
 
   private String name;
 
   private String description;
 
   private String version;
 
-  private String category;
+  private String inputs;
+
+  private String outputs;
+
+  private List<Integer> categories;
+
+  private String runtime;
+
+  private String settings;
 
-  private String documentation;
+  private String fileName;
 }
 
diff --git a/backend/services/operator-market-service/src/main/java/com/dataengine/operator/interfaces/dto/UpdateOperatorRequest.java b/backend/services/operator-market-service/src/main/java/com/dataengine/operator/interfaces/dto/UpdateOperatorRequest.java
@@ -4,6 +4,8 @@
 import lombok.Getter;
 import lombok.Setter;
 
+import java.util.List;
+
 /**
  * UpdateOperatorRequest
  */
@@ -17,8 +19,14 @@ public class UpdateOperatorRequest {
 
   private String version;
 
-  private String category;
+  private String inputs;
+
+  private String outputs;
+
+  private List<Integer> categories;
+
+  private String runtime;
 
-  private String documentation;
+  private String settings;
 }
 
diff --git a/backend/services/operator-market-service/src/main/resources/mappers/CategoryMapper.xml b/backend/services/operator-market-service/src/main/resources/mappers/CategoryMapper.xml
@@ -6,4 +6,3 @@
         SELECT * FROM t_operator_category
     </select>
 </mapper>
-
diff --git a/backend/services/operator-market-service/src/main/resources/mappers/CategoryRelationMapper.xml b/backend/services/operator-market-service/src/main/resources/mappers/CategoryRelationMapper.xml
@@ -17,4 +17,12 @@
         RIGHT JOIN t_operator_category toc ON tocr.category_id = toc.id
   </select>
 
+  <insert id="batchInsert" parameterType="java.util.List">
+        INSERT INTO t_operator_category_relation (operator_id, category_id)
+        VALUES
+        <foreach collection="categories" item="categoryId" separator=",">
+            (#{operatorId}, #{categoryId})
+        </foreach>
+    </insert>
+
 </mapper>
diff --git a/backend/services/operator-market-service/src/main/resources/mappers/OperatorMapper.xml b/backend/services/operator-market-service/src/main/resources/mappers/OperatorMapper.xml
@@ -49,4 +49,48 @@
             </if>
         </where>
     </select>
+
+    <select id="findOperatorById" parameterType="string" resultType="com.dataengine.operator.domain.modal.Operator">
+        SELECT operator_id as id, operator_name as name, description, version, inputs, outputs, runtime, settings,
+            is_star, created_at, updated_at, GROUP_CONCAT(category_id ORDER BY created_at DESC SEPARATOR ',') AS categories
+        FROM v_operator
+        WHERE operator_id = #{id}
+    </select>
+
+    <insert id="insertOperator" parameterType="com.dataengine.operator.domain.modal.Operator">
+        INSERT INTO t_operator (id, name, description, version, inputs, outputs, runtime, settings, is_star, created_at, updated_at)
+        VALUES (#{id}, #{name}, #{description}, #{version}, #{inputs}, #{outputs}, #{runtime}, #{settings}, #{isStar}, NOW(), NOW())
+    </insert>
+
+    <update id="updateOperator" parameterType="com.dataengine.operator.domain.modal.Operator">
+        UPDATE t_operator
+        <set>
+            <if test="name != null and name != ''">
+                name = #{name},
+            </if>
+            <if test="description != null and description != ''">
+                description = #{description},
+            </if>
+            <if test="version != null and version != ''">
+                version = #{version},
+            </if>
+            <if test="inputs != null and inputs != ''">
+                inputs = #{inputs},
+            </if>
+            <if test="outputs != null and outputs != ''">
+                outputs = #{outputs},
+            </if>
+            <if test="runtime != null and runtime != ''">
+                runtime = #{runtime},
+            </if>
+            <if test="settings != null and settings != ''">
+                settings = #{settings},
+            </if>
+            <if test="isStar != null">
+                is_star = #{isStar},
+            </if>
+            updated_at = NOW()
+        </set>
+        WHERE id = #{id}
+    </update>
 </mapper>
diff --git a/deployment/docker/data-platform/docker-compose.yml b/deployment/docker/data-platform/docker-compose.yml
@@ -40,6 +40,7 @@ services:
     volumes:
       - mysql_volume:/var/lib/mysql
       - ../../../scripts/db:/docker-entrypoint-initdb.d
+      - ./utf8.cnf:/etc/mysql/conf.d/utf8.cnf
       - log_volume:/var/log/data-platform
     networks: [ edatamate ]
 
diff --git a/deployment/docker/data-platform/utf8.cnf b/deployment/docker/data-platform/utf8.cnf
@@ -0,0 +1,15 @@
+[mysqld]
+# 设置服务器默认字符集为 utf8mb4 (推荐，支持完整的 UTF-8，包括 emoji)
+character-set-server = utf8mb4
+# 设置默认排序规则
+collation-server = utf8mb4_unicode_ci
+# 或者使用 utf8_general_ci (性能稍好，但排序规则稍宽松)
+default-time-zone = 'Asia/Shanghai'
+
+[client]
+# 设置客户端连接默认字符集
+default-character-set = utf8mb4
+
+[mysql]
+# 设置 mysql 命令行客户端默认字符集
+default-character-set = utf8mb4
diff --git a/deployment/kubernetes/mysql/configmap.yaml b/deployment/kubernetes/mysql/configmap.yaml
@@ -10,6 +10,7 @@ data:
     # 设置默认排序规则
     collation-server = utf8mb4_unicode_ci
     # 或者使用 utf8_general_ci (性能稍好，但排序规则稍宽松)
+    default-time-zone = 'Asia/Shanghai'
 
     [client]
     # 设置客户端连接默认字符集
diff --git a/scripts/db/data-engine-operator-init.sql b/scripts/db/data-engine-operator-init.sql
diff --git a/scripts/images/frontend/Dockerfile b/scripts/images/frontend/Dockerfile
diff --git a/scripts/images/runtime/Dockerfile b/scripts/images/runtime/Dockerfile
