change kbc op param schema (#104)

ZhaoyangHan04 · ZhaoyangHan04 · web-flow · commit c1159e70d5e6 · 2025-08-21T21:51:36.000+08:00
Co-authored-by: ZhaoyangHan04 &lt;319926404@qq.com&gt;
diff --git a/docs/en/notes/guide/domain_specific_operators/knowledgebase_QA_operators.md b/docs/en/notes/guide/domain_specific_operators/knowledgebase_QA_operators.md
@@ -76,7 +76,7 @@ For each operator, the following sections will detail its invocation methods and
 
 ## Detailed Operator Specifications
 
-### 1. knowledge_extractor
+### 1. FileOrURLToMarkdownConverter
 
 **Functional Description**:
 
@@ -87,10 +87,12 @@ The Knowledge Extractor operator is a versatile document processing tool that su
 - `__init__()`
   - `intermediate_dir`: Intermediate file output directory (default: "intermediate")
   - `lang`: Document language (default: "ch" for Chinese)
-- `run()`
-  - `storage`: Data flow storage interface object (required)
   - `raw_file`: Local file path (mutually exclusive with url)
   - `url`: Web URL address (mutually exclusive with raw_file)
+  
+- `run()`
+  - `storage`: Data flow storage interface object (required)
+
 
 **Key Features**:
 
@@ -116,14 +118,14 @@ The Knowledge Extractor operator is a versatile document processing tool that su
 **Usage Example**:
 
 ```python
-knowledge_extractor = KnowledgeExtractor(
+file_to_markdown_converter = FileOrURLToMarkdownConverter(
     intermediate_dir="../example_data/KBCleaningPipeline/raw/",
-    lang="en"
+    lang="en",
+    mineru_backend="vlm-sglang-engine",
+    raw_file = raw_file,
 )
-extracted=knowledge_extractor.run(
+extracted=file_to_markdown_converter.run(
     storage=self.storage,
-    raw_file=raw_file,
-    url=url,
 )
 ```
 
diff --git a/docs/en/notes/guide/pipelines/KnowledgeBaseCleaningPipeline.md b/docs/en/notes/guide/pipelines/KnowledgeBaseCleaningPipeline.md
@@ -24,7 +24,7 @@ The main workflow of the pipeline includes:
 
 ### 1. Information Extraction
 
-The first step of the pipeline is to extract textual knowledge from users' original documents or URLs using knowledge_extractor. This step is crucial as it converts various formats of raw documents into unified markdown text, facilitating subsequent cleaning processes.
+The first step of the pipeline is to extract textual knowledge from users' original documents or URLs using FileOrURLToMarkdownConverter. This step is crucial as it converts various formats of raw documents into unified markdown text, facilitating subsequent cleaning processes.
 
 > *Since `MinerU` is primarily deployed based on `SGLang`, the `open-dataflow[minerU]` environment mainly operates on `Dataflow[SGLang]`. Currently, there is no tutorial available for processing based on `Dataflow[vllm]`.*
 
@@ -138,16 +138,17 @@ PDF file extraction in this system is based on [MinerU](https://github.com/opend
 > 
 > #### 5. Tool Usage
 > 
-> The `KnowledgeExtractor` operator allows you to choose the desired backend engine of MinerU.
+> The `FileOrURLToMarkdownConverter` operator allows you to choose the desired backend engine of MinerU.
 > 
 > * If using `MinerU1`: set the `MinerU_Backend` parameter to `"pipeline"`, which uses the traditional pipeline approach.
 > * If using `MinerU2` **(recommended by default)**: set the `MinerU_Backend` parameter to `"vlm-sglang-engine"` to enable the vision-language model engine.
 > 
 > ```python
-> KnowledgeExtractor(
->     intermediate_dir="../example_data/KBCleaningPipeline/raw/",
->     lang="en",
->     MinerU_Backend="vlm-sglang-engine",
+> self.knowledge_cleaning_step1 = FileOrURLToMarkdownConverter(
+>    intermediate_dir="../example_data/KBCleaningPipeline/raw/",
+>    lang="en",
+>    mineru_backend="vlm-sglang-engine",
+>    raw_file = raw_file,
 > )
 > ```
 > 
@@ -160,15 +161,14 @@ PDF file extraction in this system is based on [MinerU](https://github.com/opend
 **Example**:
 
 ```python
-knowledge_extractor = KnowledgeExtractor(
+file_to_markdown_converter = FileOrURLToMarkdownConverter(
     intermediate_dir="../example_data/KBCleaningPipeline/raw/",
-    lang="en"
-    MinerU_Backend="vlm-sglang-engine",
+    lang="en",
+    mineru_backend="vlm-sglang-engine",
+    raw_file = raw_file,
 )
-extracted=knowledge_extractor.run(
+extracted=file_to_markdown_converter.run(
     storage=self.storage,
-    raw_file=raw_file,
-    url=url,
 )
 ```
 
@@ -283,22 +283,29 @@ from dataflow.operators.generate import (
     MultiHopQAGenerator,
 )
 from dataflow.utils.storage import FileStorage
-from dataflow.serving import LocalModelLLMServing_vllm
+from dataflow.serving import APILLMServing_request
 
-class KBCleaningPipeline():
-    def __init__(self):
+class KBCleaningPDF_APIPipeline():
+    def __init__(self, url:str=None, raw_file:str=None):
 
         self.storage = FileStorage(
             first_entry_file_name="../example_data/KBCleaningPipeline/kbc_placeholder.json",
-            cache_path="./.cache/gpu",
+            cache_path="./.cache/api",
             file_name_prefix="pdf_cleaning_step",
             cache_type="json",
         )
 
+        self.llm_serving = APILLMServing_request(
+                api_url="https://api.openai.com/v1/chat/completions",
+                model_name="gpt-4o",
+                max_workers=100
+        )
+
         self.knowledge_cleaning_step1 = FileOrURLToMarkdownConverter(
             intermediate_dir="../example_data/KBCleaningPipeline/raw/",
             lang="en",
             mineru_backend="vlm-sglang-engine",
+            raw_file = raw_file,
         )
 
         self.knowledge_cleaning_step2 = CorpusTextSplitter(
@@ -307,37 +314,27 @@ class KBCleaningPipeline():
             tokenizer_name="Qwen/Qwen2.5-7B-Instruct",
         )
 
-    def forward(self, url:str=None, raw_file:str=None):
+        self.knowledge_cleaning_step3 = KnowledgeCleaner(
+            llm_serving=self.llm_serving,
+            lang="en"
+        )
+
+        self.knowledge_cleaning_step4 = MultiHopQAGenerator(
+            llm_serving=self.llm_serving,
+            lang="en"
+        )
+
+    def forward(self):
         extracted=self.knowledge_cleaning_step1.run(
             storage=self.storage,
-            raw_file=raw_file,
-            url=url,
         )
-  
+        
         self.knowledge_cleaning_step2.run(
             storage=self.storage.step(),
             input_file=extracted,
             output_key="raw_content",
         )
 
-        local_llm_serving = LocalModelLLMServing_vllm(
-            hf_model_name_or_path="Qwen/Qwen2.5-7B-Instruct",
-            vllm_max_tokens=2048,
-            vllm_tensor_parallel_size=4,
-            vllm_gpu_memory_utilization=0.6,
-            vllm_repetition_penalty=1.2
-        )
-
-        self.knowledge_cleaning_step3 = KnowledgeCleaner(
-            llm_serving=local_llm_serving,
-            lang="en"
-        )
-
-        self.knowledge_cleaning_step4 = MultiHopQAGenerator(
-            llm_serving=local_llm_serving,
-            lang="en"
-        )
-
         self.knowledge_cleaning_step3.run(
             storage=self.storage.step(),
             input_key= "raw_content",
@@ -348,9 +345,9 @@ class KBCleaningPipeline():
             input_key="cleaned",
             output_key="MultiHop_QA"
         )
-  
+        
 if __name__ == "__main__":
-    model = KBCleaningPipeline()
-    model.forward(raw_file="../example_data/KBCleaningPipeline/test.pdf")
+    model = KBCleaningPDF_APIPipeline(raw_file="../example_data/KBCleaningPipeline/test.pdf")
+    model.forward()
 ```
 
diff --git a/docs/zh/notes/guide/domain_specific_operators/knowledgebase_QA_operators.md b/docs/zh/notes/guide/domain_specific_operators/knowledgebase_QA_operators.md
@@ -8,7 +8,7 @@ permalink: /zh/guide/Knowledgebase_QA_operators/
 
 ## 概述
 
-知识库清洗算子适用于面向RAG，RARE，RAFT等下游任务的知识库提取，整理，精调，主要包括：**知识提取算子(KnowledgeExtractor**)，**语料分块算子(CorpusTextSpliiter)**和**知识清洗算子(KnowledgeCleaner)**, **Multi-Hop QA Generation Operator**。这些算子能够用于多种原始格式的文件整理，以及爬取特定URL对应的网页内容，并将这些文本知识整理成可读、易用、安全的RAG知识库。
+知识库清洗算子适用于面向RAG，RARE，RAFT等下游任务的知识库提取，整理，精调，主要包括：**知识提取算子(FileOrURLToMarkdownConverter**)，**语料分块算子(CorpusTextSpliiter)**和**知识清洗算子(KnowledgeCleaner)**, **Multi-Hop QA Generation Operator**。这些算子能够用于多种原始格式的文件整理，以及爬取特定URL对应的网页内容，并将这些文本知识整理成可读、易用、安全的RAG知识库。
 
 本文中算子标记继承自[强推理算子](https://opendcai.github.io/DataFlow-Doc/zh/guide/Reasoning_operators/)
 
@@ -72,7 +72,7 @@ self.storage = FileStorage(
 
 ## 详细算子说明
 
-### 1. KnowledgeExtractor
+### 1. FileOrURLToMarkdownConverter
 
 **功能描述**：
 
@@ -83,11 +83,11 @@ self.storage = FileStorage(
    - `__init__()`
      - `intermediate_dir`：中间文件输出目录（默认："intermediate"）
      - `lang`：文档语言（默认："ch"中文）
+     - `raw_file`：本地文件路径（与url二选一）
+     - `url`：网页URL地址（与raw_file二选一）
 
    - `run()`
      - `storage`：数据流存储接口对象（必须）
-     - `raw_file`：本地文件路径（与url二选一）
-     - `url`：网页URL地址（与raw_file二选一）
 
    **主要特性**：
 
@@ -116,14 +116,14 @@ self.storage = FileStorage(
 **使用示例：**
 
 ```python
-knowledge_extractor = KnowledgeExtractor(
+file_to_markdown_converter = FileOrURLToMarkdownConverter(
     intermediate_dir="../example_data/KBCleaningPipeline/raw/",
-    lang="en"
+    lang="en",
+    mineru_backend="vlm-sglang-engine",
+    raw_file = raw_file,
 )
-extracted=knowledge_extractor.run(
+extracted=file_to_markdown_converter.run(
     storage=self.storage,
-    raw_file=raw_file,
-    url=url,
 )
 ```
 
diff --git a/docs/zh/notes/guide/pipelines/KnowledgeBaseCleaningPipeline.md b/docs/zh/notes/guide/pipelines/KnowledgeBaseCleaningPipeline.md
@@ -134,16 +134,17 @@ pip install -e .[mineru]
 >
 > #### 5. 工具使用
 >
-> `KnowledgeExtractor` 算子提供了 MinerU 版本的选择接口，允许用户根据需求选择合适的后端引擎。
+> `FileOrURLToMarkdownConverter` 算子提供了 MinerU 版本的选择接口，允许用户根据需求选择合适的后端引擎。
 >
 > * 如果用户使用 `MinerU1`：请将 `MinerU_Backend` 参数设置为 `"pipeline"`。这将启用传统的流水线处理方式。
 > * 如果用户使用 `MinerU2` **(默认推荐)**：请将 `MinerU_Backend` 参数设置为 `"vlm-sglang-engine"`。这将启用基于多模态语言模型的新引擎。
 >
 > ```bash
-> KnowledgeExtractor(
->     intermediate_dir="../example_data/KBCleaningPipeline/raw/",
->     lang="en",
->     MinerU_Backend="vlm-sglang-engine",
+> self.knowledge_cleaning_step1 = FileOrURLToMarkdownConverter(
+>    intermediate_dir="../example_data/KBCleaningPipeline/raw/",
+>    lang="en",
+>    mineru_backend="vlm-sglang-engine",
+>    raw_file = raw_file,
 > )
 > ```
 >
@@ -154,15 +155,14 @@ pip install -e .[mineru]
 **示例**：
 
 ```python
-knowledge_extractor = KnowledgeExtractor(
+file_to_markdown_converter = FileOrURLToMarkdownConverter(
     intermediate_dir="../example_data/KBCleaningPipeline/raw/",
-    lang="en"
-    MinerU_Backend="vlm-sglang-engine",
+    lang="en",
+    mineru_backend="vlm-sglang-engine",
+    raw_file = raw_file,
 )
-extracted=knowledge_extractor.run(
+extracted=file_to_markdown_converter.run(
     storage=self.storage,
-    raw_file=raw_file,
-    url=url,
 )
 ```
 
@@ -273,22 +273,29 @@ from dataflow.operators.generate import (
     MultiHopQAGenerator,
 )
 from dataflow.utils.storage import FileStorage
-from dataflow.serving import LocalModelLLMServing_vllm
+from dataflow.serving import APILLMServing_request
 
-class KBCleaningPipeline():
-    def __init__(self):
+class KBCleaningPDF_APIPipeline():
+    def __init__(self, url:str=None, raw_file:str=None):
 
         self.storage = FileStorage(
             first_entry_file_name="../example_data/KBCleaningPipeline/kbc_placeholder.json",
-            cache_path="./.cache/gpu",
+            cache_path="./.cache/api",
             file_name_prefix="pdf_cleaning_step",
             cache_type="json",
         )
 
+        self.llm_serving = APILLMServing_request(
+                api_url="https://api.openai.com/v1/chat/completions",
+                model_name="gpt-4o",
+                max_workers=100
+        )
+
         self.knowledge_cleaning_step1 = FileOrURLToMarkdownConverter(
             intermediate_dir="../example_data/KBCleaningPipeline/raw/",
             lang="en",
             mineru_backend="vlm-sglang-engine",
+            raw_file = raw_file,
         )
 
         self.knowledge_cleaning_step2 = CorpusTextSplitter(
@@ -297,37 +304,27 @@ class KBCleaningPipeline():
             tokenizer_name="Qwen/Qwen2.5-7B-Instruct",
         )
 
-    def forward(self, url:str=None, raw_file:str=None):
+        self.knowledge_cleaning_step3 = KnowledgeCleaner(
+            llm_serving=self.llm_serving,
+            lang="en"
+        )
+
+        self.knowledge_cleaning_step4 = MultiHopQAGenerator(
+            llm_serving=self.llm_serving,
+            lang="en"
+        )
+
+    def forward(self):
         extracted=self.knowledge_cleaning_step1.run(
             storage=self.storage,
-            raw_file=raw_file,
-            url=url,
         )
-  
+        
         self.knowledge_cleaning_step2.run(
             storage=self.storage.step(),
             input_file=extracted,
             output_key="raw_content",
         )
 
-        local_llm_serving = LocalModelLLMServing_vllm(
-            hf_model_name_or_path="Qwen/Qwen2.5-7B-Instruct",
-            vllm_max_tokens=2048,
-            vllm_tensor_parallel_size=4,
-            vllm_gpu_memory_utilization=0.6,
-            vllm_repetition_penalty=1.2
-        )
-
-        self.knowledge_cleaning_step3 = KnowledgeCleaner(
-            llm_serving=local_llm_serving,
-            lang="en"
-        )
-
-        self.knowledge_cleaning_step4 = MultiHopQAGenerator(
-            llm_serving=local_llm_serving,
-            lang="en"
-        )
-
         self.knowledge_cleaning_step3.run(
             storage=self.storage.step(),
             input_key= "raw_content",
@@ -338,8 +335,8 @@ class KBCleaningPipeline():
             input_key="cleaned",
             output_key="MultiHop_QA"
         )
-  
+        
 if __name__ == "__main__":
-    model = KBCleaningPipeline()
-    model.forward(raw_file="../example_data/KBCleaningPipeline/test.pdf")
+    model = KBCleaningPDF_APIPipeline(raw_file="../example_data/KBCleaningPipeline/test.pdf")
+    model.forward()
 ```
