Merge pull request #200 from shijinpjlab/dev_1009

feat: update ModelRes and add img rule

Author: e06084

dingo/exec/local.py

@@ -238,12 +238,30 @@ def evaluate_rule(self, group: List[BaseRule], d: Data) -> ResultInfo:
             # analyze result
             if tmp.error_status:
                 result_info.error_status = True
-                bad_type_list.append(tmp.type)
-                bad_name_list.append(tmp.type + "-" + tmp.name)
+                if isinstance(tmp.type, str) and isinstance(tmp.name, str):
+                    bad_type_list.append(tmp.type)
+                    bad_name_list.append(tmp.type + "-" + tmp.name)
+                elif isinstance(tmp.type, List) and isinstance(tmp.name, List):
+                    if len(tmp.type) != len(tmp.name):
+                        raise Exception(f'ModelRes.type is not the same length to ModelRes.name.\n type: {tmp.type} \n name: {tmp.name}')
+                    for i in range(len(tmp.type)):
+                        bad_type_list.append(tmp.type[i])
+                        bad_name_list.append(tmp.type[i] + "-" + tmp.name[i])
+                else:
+                    raise Exception('ModelRes.type and ModelRes.name are not str or List at the same time.')
                 bad_reason_list.extend(tmp.reason)
             else:
-                good_type_list.append(tmp.type)
-                good_name_list.append(tmp.type + "-" + tmp.name)
+                if isinstance(tmp.type, str) and isinstance(tmp.name, str):
+                    good_type_list.append(tmp.type)
+                    good_name_list.append(tmp.type + "-" + tmp.name)
+                elif isinstance(tmp.type, List) and isinstance(tmp.name, List):
+                    if len(tmp.type) != len(tmp.name):
+                        raise Exception(f'ModelRes.type is not the same length to ModelRes.name.\n type: {tmp.type} \n name: {tmp.name}')
+                    for i in range(len(tmp.type)):
+                        good_type_list.append(tmp.type[i])
+                        good_name_list.append(tmp.type[i] + "-" + tmp.name[i])
+                else:
+                    raise Exception('ModelRes.type and ModelRes.name are not str or List at the same time.')
                 good_reason_list.extend(tmp.reason)
         if result_info.error_status:
             result_info.type_list = list(set(bad_type_list))
@@ -271,12 +289,32 @@ def evaluate_prompt(self, group: List[BasePrompt], d: Data) -> ResultInfo:
             # analyze result
             if tmp.error_status:
                 result_info.error_status = True
-                bad_type_list.append(tmp.type)
-                bad_name_list.append(tmp.type + "-" + tmp.name)
+                if isinstance(tmp.type, str) and isinstance(tmp.name, str):
+                    bad_type_list.append(tmp.type)
+                    bad_name_list.append(tmp.type + "-" + tmp.name)
+                elif isinstance(tmp.type, List) and isinstance(tmp.name, List):
+                    if len(tmp.type) != len(tmp.name):
+                        raise Exception(
+                            f'ModelRes.type is not the same length to ModelRes.name.\n type: {tmp.type} \n name: {tmp.name}')
+                    for i in range(len(tmp.type)):
+                        bad_type_list.append(tmp.type[i])
+                        bad_name_list.append(tmp.type[i] + "-" + tmp.name[i])
+                else:
+                    raise Exception('ModelRes.type and ModelRes.name are not str or List at the same time.')
                 bad_reason_list.extend(tmp.reason)
             else:
-                good_type_list.append(tmp.type)
-                good_name_list.append(tmp.type + "-" + tmp.name)
+                if isinstance(tmp.type, str) and isinstance(tmp.name, str):
+                    good_type_list.append(tmp.type)
+                    good_name_list.append(tmp.type + "-" + tmp.name)
+                elif isinstance(tmp.type, List) and isinstance(tmp.name, List):
+                    if len(tmp.type) != len(tmp.name):
+                        raise Exception(
+                            f'ModelRes.type is not the same length to ModelRes.name.\n type: {tmp.type} \n name: {tmp.name}')
+                    for i in range(len(tmp.type)):
+                        good_type_list.append(tmp.type[i])
+                        good_name_list.append(tmp.type[i] + "-" + tmp.name[i])
+                else:
+                    raise Exception('ModelRes.type and ModelRes.name are not str or List at the same time.')
                 good_reason_list.extend(tmp.reason)
         if result_info.error_status:
             result_info.type_list = list(set(bad_type_list))

dingo/model/modelres.py

@@ -5,8 +5,8 @@
 
 class ModelRes(BaseModel):
     error_status: bool = False
-    type: str = "QUALITY_GOOD"
-    name: str = "Data"
+    type: str | List[str] = "QUALITY_GOOD"
+    name: str | List[str] = "Data"
     reason: List[str] = []
 
     # Optional fields for enhanced functionality (e.g., hallucination detection)

dingo/model/rule/rule_image.py

@@ -312,6 +312,46 @@ def eval(cls, input_data: Data) -> ModelRes:
             )
 
 
+@Model.rule_register("QUALITY_BAD_IMG_LABEL_OVERLAP", [])
+class RuleImageLabelOverlap(BaseRule):
+    _metric_info = {
+        "category": "Rule-Based IMG Quality Metrics",
+        "quality_dimension": "IMG_LABEL_OVERLAP",
+        "metric_name": "RuleImageLabelOverlap",
+        "description": "",  # TODO: lindong input
+        "paper_title": "",
+        "paper_url": "",
+        "paper_authors": "",
+        "evaluation_results": ""
+    }
+
+    dynamic_config = EvaluatorRuleArgs()
+
+    @classmethod
+    def eval(cls, input_data: Data) -> ModelRes:
+        pass
+
+
+@Model.rule_register("QUALITY_BAD_IMG_LABEL_VISUALIZATION", [])
+class RuleImageLabelVisualization(BaseRule):
+    _metric_info = {
+        "category": "Rule-Based IMG Quality Metrics",
+        "quality_dimension": "IMG_LABEL_VISUALIZATION",
+        "metric_name": "RuleImageLabelVisualization",
+        "description": "",  # TODO: lindong input
+        "paper_title": "",
+        "paper_url": "",
+        "paper_authors": "",
+        "evaluation_results": ""
+    }
+
+    dynamic_config = EvaluatorRuleArgs()
+
+    @classmethod
+    def eval(cls, input_data: Data) -> ModelRes:
+        pass
+
+
 if __name__ == "__main__":
     data = Data(
         data_id='1',

examples/image/sdk_image_label_overlap.py

@@ -0,0 +1,32 @@
+from dingo.config import InputArgs
+from dingo.exec import Executor
+
+
+def image_label_overlap():
+    input_data = {
+        "input_path": "../../test/data/img_label/test_img_label_overlap.jsonl",
+        "dataset": {
+            "source": "local",
+            "format": "image",
+            "field": {
+                "id": "id",
+                "content": "content",
+                "image": "img"
+            }
+        },
+        "executor": {
+            "rule_list": ["RuleImageLabelOverlap"],
+            "result_save": {
+                "bad": True,
+                "good": True
+            }
+        }
+    }
+    input_args = InputArgs(**input_data)
+    executor = Executor.exec_map["local"](input_args)
+    result = executor.execute()
+    print(result)
+
+
+if __name__ == '__main__':
+    image_label_overlap()

test/data/img_label/overlap_0.jpg

@@ -0,0 +1,47 @@
+import os
+import re
+from typing import List
+
+from dingo.config.input_args import EvaluatorRuleArgs
+from dingo.io import Data
+from dingo.model.model import Model
+from dingo.model.modelres import ModelRes
+from dingo.model.rule.base import BaseRule
+
+
+@Model.rule_register('QUALITY_BAD_RELEVANCE', ['test'])
+class RegisterRuleColon(BaseRule):
+    """let user input pattern to search"""
+    dynamic_config = EvaluatorRuleArgs(pattern = "blue")
+
+    @classmethod
+    def eval(cls, input_data: Data) -> ModelRes:
+        res = ModelRes()
+        content = input_data.content
+        if len(content) <= 0:
+            return res
+        if content[-1] == ":":
+            res.error_status = True
+            res.type = [cls.metric_type, 'TestType']
+            res.name = [cls.__name__, 'TestName']
+            res.reason = [content[-100:]]
+        return res
+
+
+class TestModelRes:
+    def test_type_name_list(self):
+
+        data = Data(
+            data_id='0',
+            prompt="",
+            content="Hello! The world is a vast and diverse place, full of wonders, cultures, and incredible natural beauty:"
+        )
+
+        res = RegisterRuleColon().eval(data)
+        # print(res)
+        assert isinstance(res.type, List)
+        assert isinstance(res.name, List)
+        assert len(res.type) == 2
+        assert len(res.name) == 2
+        assert 'TestType' in res.type
+        assert 'TestName' in res.name