Merge pull request #208 from e06084/dev

feat: add html_extract_compare_v2

Author: shijinpjlab

dingo/model/llm/llm_html_extract_compare_v2.py

@@ -0,0 +1,235 @@
+import re
+from typing import List
+
+import diff_match_patch as dmp_module
+
+from dingo.io import Data
+from dingo.model import Model
+from dingo.model.llm.base_openai import BaseOpenAI
+from dingo.model.modelres import ModelRes
+from dingo.model.prompt.prompt_html_extract_compare_v2 import PromptHtmlExtractCompareV2
+from dingo.model.response.response_class import ResponseNameReason
+from dingo.utils import log
+
+
+@Model.llm_register("LLMHtmlExtractCompareV2")
+class LLMHtmlExtractCompareV2(BaseOpenAI):
+    """
+    HTML提取工具对比评估 V2 版本
+
+    主要改进：
+    1. 使用 diff-match-patch 算法预先提取文本差异
+    2. 只向 LLM 提供独有内容和共同内容，大幅减少 token 消耗
+    3. 支持中英文双语提示词
+    4. 使用 A/B/C 判断格式，更清晰地表达哪个工具更好
+
+    输入数据要求：
+    - input_data.prompt: 工具A提取的文本
+    - input_data.content: 工具B提取的文本
+    - input_data.raw_data.get("language", "en"): 语言类型 ("zh" 或 "en")
+    """
+
+    prompt = PromptHtmlExtractCompareV2
+
+    @classmethod
+    def extract_text_diff(cls, text_a: str, text_b: str, max_diff_length: int = 10000) -> dict:
+        """
+        使用 diff-match-patch 算法提取两段文本的差异
+
+        Args:
+            text_a: 工具A提取的文本
+            text_b: 工具B提取的文本
+            max_diff_length: 差异文本的最大长度限制
+
+        Returns:
+            dict: 包含 unique_a, unique_b, common 三个字段
+        """
+        dmp = dmp_module.diff_match_patch()
+
+        # 计算差异
+        diff = dmp.diff_main(text_a, text_b)
+        dmp.diff_cleanupEfficiency(diff)
+
+        unique_a_list = []
+        unique_b_list = []
+        common_list = []
+
+        for single_diff in diff:
+            if single_diff[0] == -1:  # 仅在 text_a 中
+                unique_a_list.append(single_diff[1])
+            elif single_diff[0] == 1:  # 仅在 text_b 中
+                unique_b_list.append(single_diff[1])
+            elif single_diff[0] == 0:  # 共同内容
+                common_list.append(single_diff[1])
+
+        return {
+            "unique_a": "".join(unique_a_list)[:max_diff_length],
+            "unique_b": "".join(unique_b_list)[:max_diff_length],
+            "common": "".join(common_list)[:max_diff_length],
+        }
+
+    @classmethod
+    def build_messages(cls, input_data: Data) -> List:
+        """
+        构建 LLM 输入消息
+
+        主要流程：
+        1. 提取工具A和工具B的文本
+        2. 使用 diff-match-patch 计算差异
+        3. 根据语言选择合适的提示词
+        4. 填充差异内容到提示词中
+        """
+        # 获取输入文本
+        text_tool_a = input_data.prompt
+        text_tool_b = input_data.content
+
+        # 获取配置参数
+        language = input_data.raw_data.get("language", "en")
+
+        # 计算文本差异
+        diff_result = cls.extract_text_diff(text_tool_a, text_tool_b)
+
+        # 根据语言选择提示词
+        if language == "zh":
+            prompt_template = cls.prompt.content_cn
+        else:
+            prompt_template = cls.prompt.content_en
+
+        # 填充提示词
+        prompt_content = prompt_template.format(
+            text_unique_tool_a=diff_result["unique_a"],
+            text_unique_tool_b=diff_result["unique_b"],
+            text_common=diff_result["common"]
+        )
+
+        messages = [
+            {
+                "role": "user",
+                "content": prompt_content
+            }
+        ]
+
+        return messages
+
+    @classmethod
+    def _parse_response_to_structured(cls, response: str) -> ResponseNameReason:
+        """
+        将 LLM 原始响应解析为结构化的 ResponseNameReason 对象
+
+        解析格式：
+        1. 提取 <Judgement>A/B/C</Judgement> 标签中的判断结果
+        2. 其余内容作为推理过程
+
+        Args:
+            response: LLM 原始响应文本
+
+        Returns:
+            ResponseNameReason: 结构化响应对象，name 字段存储判断结果 (A/B/C)
+
+        Raises:
+            ValueError: 如果无法解析出有效的判断结果
+        """
+        log.info(response)
+
+        # 提取判断结果
+        judgement_match = re.search(r"<Judgement>([ABC])</Judgement>", response)
+
+        if not judgement_match:
+            # 如果没有找到标准格式，尝试其他可能的格式
+            judgement_match = re.search(r"判断[：:]\s*([ABC])", response)
+            if not judgement_match:
+                judgement_match = re.search(r"答案[：:]\s*([ABC])", response)
+
+        if not judgement_match:
+            raise ValueError(f"无法从响应中提取判断结果: {response}")
+
+        judgement = judgement_match.group(1)
+
+        # 提取推理过程（去除判断标签）
+        reason = re.sub(r"<Judgement>[ABC]</Judgement>", "", response).strip()
+
+        # 使用 Pydantic 模型进行验证，name 字段存储判断结果
+        return ResponseNameReason(
+            name=judgement,
+            reason=reason
+        )
+
+    @classmethod
+    def _convert_to_model_result(cls, structured_response: ResponseNameReason) -> ModelRes:
+        """
+        将结构化响应转换为 ModelRes 对象
+
+        映射规则：
+        - A -> TOOL_ONE_BETTER (工具A更好，error_status=False)
+        - B -> TOOL_EQUAL (两者相同，error_status=False)
+        - C -> TOOL_TWO_BETTER (工具B更好，error_status=True)
+
+        Args:
+            structured_response: 结构化响应对象，name 字段存储判断结果 (A/B/C)
+
+        Returns:
+            ModelRes: 评估结果对象
+        """
+        result = ModelRes()
+
+        # 从 name 字段获取判断结果
+        judgement = structured_response.name
+
+        # 映射判断结果到类型和状态
+        judgement_mapping = {
+            "A": {
+                "type": "TOOL_ONE_BETTER",
+                "error_status": False,  # 工具A更好，正常
+                "description": "工具A提取的信息更完整"
+            },
+            "B": {
+                "type": "TOOL_EQUAL",
+                "error_status": False,  # 两者相同，正常
+                "description": "两个工具提取的信息量相同"
+            },
+            "C": {
+                "type": "TOOL_TWO_BETTER",
+                "error_status": True,  # 工具B更好，标记为问题
+                "description": "工具B提取的信息更完整"
+            }
+        }
+
+        mapping = judgement_mapping.get(judgement)
+        if not mapping:
+            raise ValueError(f"无效的判断结果: {judgement}")
+
+        result.type = mapping["type"]
+        result.error_status = mapping["error_status"]
+        result.name = f"Judgement_{judgement}"
+        result.reason = [structured_response.reason]
+
+        return result
+
+    @classmethod
+    def process_response(cls, response: str) -> ModelRes:
+        """
+        处理 LLM 返回结果
+
+        数据流：
+        1. 原始响应 (str) -> 结构化响应 (ResponseNameReason)
+        2. 结构化响应 -> 评估结果 (ModelRes)
+
+        这种分层设计的好处：
+        - 更清晰的责任分离
+        - 利用 Pydantic 的验证功能
+        - 便于单元测试
+        - 便于扩展和维护
+
+        Args:
+            response: LLM 原始响应文本
+
+        Returns:
+            ModelRes: 评估结果对象
+        """
+        # 步骤1: 解析为结构化响应
+        structured_response = cls._parse_response_to_structured(response)
+
+        # 步骤2: 转换为模型结果
+        result = cls._convert_to_model_result(structured_response)
+
+        return result

dingo/model/prompt/prompt_html_extract_compare_v2.py

@@ -0,0 +1,106 @@
+from dingo.model.model import Model
+from dingo.model.prompt.base import BasePrompt
+
+
+@Model.prompt_register("Html_Extract_Compare_V2", ['html_extract_compare'], ['LLMHtmlExtractCompareV2'])
+class PromptHtmlExtractCompareV2(BasePrompt):
+    _metric_info = {
+        'category': 'SFT Data Assessment Metrics',
+        'metric_name': 'PromptHtmlExtractCompareV2',
+        'description': 'Compares HTML extraction results using diff-match-patch algorithm to identify unique and common content, then evaluates core informational content differences',
+        'paper_title': '',
+        'paper_url': '',
+        'paper_authors': '',
+        'evaluation_results': ''
+    }
+
+    content_en = r"""Please compare the following two texts, each extracted from the same webpage using different HTML parsing methods. Your task is to determine whether there is a difference in the core informational content between them.
+
+Guidelines:
+
+Core informational content refers to: main facts, key ideas, central explanations, important data, and the primary textual body of the page.
+
+DO NOT consider the following as core content:
+
+Related questions
+Related topics
+Recommended articles
+"You might also like" sections
+Titles or section headings
+Author names, credentials, affiliations, or bylines
+Reference lists, citations, or bibliographies (e.g., "[1] Smith, J. 2020…")
+Hyperlinks, URLs, or navigation elements (e.g., "Back to homepage", "Related articles", "Next/Previous")
+
+Other autogenerated content
+These elements are considered supplementary and should not influence your assessment of content differences.
+
+You should ignore differences in formatting, word order, or minor stylistic variations unless they affect the actual meaning or presence of important information.
+
+content 1:
+{text_unique_tool_a}
+
+content 2:
+{text_unique_tool_b}
+
+content 3:
+{text_common}
+
+Text A contains content 1 + content 3
+Text B contains content 2 + content 3
+You should focus on the intrinsic logic between the unique content (content 1, content 2) and the common content (content 3) as the crucial basis for judging whether there is significant informational content.
+Explain your reasoning briefly. Then judge the compare result as one of:
+A. Text A contains more core informational content than Text B
+B. Text A contains the same amount of core informational content as Text B
+C. Text A contains less core informational content than Text B
+
+Return the judgment using this format:
+<Judgement>A</Judgement> or <Judgement>B</Judgement> or <Judgement>C</Judgement>
+Please output your thought process first, and then provide your final judgement.
+"""
+
+    content_cn = r"""请比较以下两段文本，它们是使用不同的 HTML 解析方法从同一网页中提取的。你的任务是判断这两段文本在核心信息内容上是否存在差异。
+
+评判指南：
+
+"核心信息内容"是指：主要事实、关键信息、核心解释、重要数据以及网页的主要正文内容。
+
+请不要将以下内容视为核心信息：
+
+- 相关问题
+- 相关主题
+- 推荐文章
+- "你可能还喜欢" 类内容
+- 标题或章节标题
+- 作者姓名、资历、机构或署名
+- 参考文献、引用或文献列表
+- 超链接、网址或导航元素
+- 其他自动生成的内容
+- 主题总结
+
+这些元素被视为附加信息，不应影响你对信息差异的判断。
+
+除非会影响实际含义或重要信息的存在，否则请忽略格式、措辞顺序或轻微风格差异。
+
+content 1:
+{text_unique_tool_a}
+
+content 2:
+{text_unique_tool_b}
+
+content 3:
+{text_common}
+
+Text A 由 content 1 + content 3 构成
+Text B 由 content 2 + content 3 构成
+你应重点关注"独有内容（content 1、content 2）"与"共同内容（content 3）"之间的内在逻辑，作为判断是否存在重要信息差异的关键依据。
+
+请简要说明你的推理过程。然后给出如下三种判断之一：
+
+A. Text A 包含的核心信息内容多于 Text B
+B. Text A 与 Text B 包含相同量的核心信息内容
+C. Text A 包含的核心信息内容少于 Text B
+
+请按以下格式返回你的判断：
+<Judgement>A</Judgement> 或 <Judgement>B</Judgement> 或 <Judgement>C</Judgement>
+请首先输出思考过程，最后再输出你的答案。
+"""