feature: add data-evaluation

* feature: add evaluation task management function

* feature: add evaluation task detail page

* fix: delete duplicate definition for table t_model_config

* refactor: rename package synthesis to ratio

* refactor: add eval file table and  refactor related code

* fix: calling large models in parallel during evaluation

Author: hefanli

backend/services/data-management-service/src/main/java/com/datamate/datamanagement/application/DatasetApplicationService.java

@@ -21,6 +21,7 @@
 import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.collections4.CollectionUtils;
 import org.springframework.beans.factory.annotation.Value;
+import org.springframework.data.redis.core.StringRedisTemplate;
 import org.springframework.scheduling.annotation.Async;
 import org.springframework.stereotype.Service;
 import org.springframework.transaction.annotation.Transactional;
@@ -42,11 +43,13 @@
 @Transactional
 @RequiredArgsConstructor
 public class DatasetApplicationService {
+    private static final String DATASET_PVC_NAME = "sys.management.dataset.pvc.name";
     private final DatasetRepository datasetRepository;
     private final TagMapper tagMapper;
     private final DatasetFileRepository datasetFileRepository;
     private final CollectionTaskClient collectionTaskClient;
     private final DatasetFileApplicationService datasetFileApplicationService;
+    private final StringRedisTemplate redisTemplate;
 
     @Value("${datamate.data-management.base-path:/dataset}")
     private String datasetBasePath;
@@ -76,6 +79,10 @@ public Dataset createDataset(CreateDatasetRequest createDatasetRequest) {
         return dataset;
     }
 
+    public String getDatasetPvcName() {
+        return redisTemplate.opsForValue().get(DATASET_PVC_NAME);
+    }
+
     public Dataset updateDataset(String datasetId, UpdateDatasetRequest updateDatasetRequest) {
         Dataset dataset = datasetRepository.getById(datasetId);
         BusinessAssert.notNull(dataset, DataManagementErrorCode.DATASET_NOT_FOUND);
@@ -130,7 +137,10 @@ public Dataset getDataset(String datasetId) {
     public PagedResponse<DatasetResponse> getDatasets(DatasetPagingQuery query) {
         IPage<Dataset> page = new Page<>(query.getPage(), query.getSize());
         page = datasetRepository.findByCriteria(page, query);
-        return PagedResponse.of(DatasetConverter.INSTANCE.convertToResponse(page.getRecords()), page.getCurrent(), page.getTotal(), page.getPages());
+        String datasetPvcName = getDatasetPvcName();
+        List<DatasetResponse> datasetResponses = DatasetConverter.INSTANCE.convertToResponse(page.getRecords());
+        datasetResponses.forEach(dataset -> dataset.setPvcName(datasetPvcName));
+        return PagedResponse.of(datasetResponses, page.getCurrent(), page.getTotal(), page.getPages());
     }
 
     /**

backend/services/data-management-service/src/main/java/com/datamate/datamanagement/domain/model/dataset/FileTag.java

@@ -28,7 +28,7 @@ public class FileTag {
 
     private String fromName;
 
-    public List<String> getTags() {
+    public List<String> getTagValues() {
         List<String> tags = new ArrayList<>();
         Object tagValues = values.get(type);
         if (tagValues instanceof List) {
@@ -40,9 +40,6 @@ public List<String> getTags() {
         } else if (tagValues instanceof String) {
             tags.add((String) tagValues);
         }
-        if(StringUtils.isNotEmpty(fromName)) {
-            return tags.stream().map(tag -> fromName + " " + tag).toList();
-        }
         return tags;
     }
 }

backend/services/data-management-service/src/main/java/com/datamate/datamanagement/interfaces/converter/DatasetConverter.java

@@ -70,8 +70,8 @@ public interface DatasetConverter {
      * @return 标签分布
      */
     @Named("getDistribution")
-    default Map<String, Long> getDistribution(List<DatasetFile> datasetFiles) {
-        Map<String, Long> distribution = new HashMap<>();
+    default Map<String, Map<String, Long>> getDistribution(List<DatasetFile> datasetFiles) {
+        Map<String, Map<String, Long>> distribution = new HashMap<>();
         if (CollectionUtils.isEmpty(datasetFiles)) {
             return distribution;
         }
@@ -81,7 +81,9 @@ default Map<String, Long> getDistribution(List<DatasetFile> datasetFiles) {
                 return distribution;
             }
             for (FileTag tag : tags) {
-                tag.getTags().forEach(tagName -> distribution.put(tagName, distribution.getOrDefault(tagName, 0L) + 1));
+                Map<String, Long> tagValueMap = distribution.getOrDefault(tag.getFromName(), new HashMap<>());
+                tag.getTagValues().forEach(tagValue -> tagValueMap.put(tagValue, tagValueMap.getOrDefault(tagValue, 0L) + 1));
+                distribution.put(tag.getFromName(), tagValueMap);
             }
         }
         return distribution;

backend/services/data-management-service/src/main/java/com/datamate/datamanagement/interfaces/dto/DatasetResponse.java

@@ -46,5 +46,7 @@ public class DatasetResponse {
     /** 更新者 */
     private String updatedBy;
     /** 分布 */
-    private Map<String, Long> distribution ;
+    private Map<String, Map<String, Long>> distribution;
+    /** 数据集pvc名称 */
+    private String pvcName;
 }

backend/services/data-management-service/src/main/java/com/datamate/datamanagement/interfaces/rest/DatasetController.java

@@ -58,8 +58,9 @@ public DatasetResponse createDataset(@RequestBody @Valid CreateDatasetRequest cr
      */
     @GetMapping("/{datasetId}")
     public DatasetResponse getDatasetById(@PathVariable("datasetId") String datasetId) {
-        Dataset dataset = datasetApplicationService.getDataset(datasetId);
-        return DatasetConverter.INSTANCE.convertToResponse(dataset);
+        DatasetResponse dataset = DatasetConverter.INSTANCE.convertToResponse(datasetApplicationService.getDataset(datasetId));
+        dataset.setPvcName(datasetApplicationService.getDatasetPvcName());
+        return dataset;
     }
 
     /**

deployment/docker/datamate/backend.conf

@@ -14,6 +14,13 @@ server {
         proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
     }
 
+    location /api/evaluation/ {
+        proxy_pass http://datamate-backend-python:18000/api/evaluation/;
+        proxy_set_header Host $host;
+        proxy_set_header X-Real-IP $remote_addr;
+        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+    }
+
     location /api/annotation/ {
         proxy_pass http://datamate-backend-python:18000/api/annotation/;
         proxy_set_header Host $host;