0.0.3 debug run passed

Author: davidkhala

.env

@@ -1,4 +1,4 @@
 INSTALL_METHOD=remote
 REMOTE_INSTALL_HOST=debug.dify.ai
 REMOTE_INSTALL_PORT=5003
-REMOTE_INSTALL_KEY=89807907-f186-4f5a-a1d3-299bb28c2226
+REMOTE_INSTALL_KEY=b0895dc3-efe2-45d1-a8a7-39124d14ee46

manifest.yaml

@@ -1,4 +1,4 @@
-version: 0.0.2
+version: 0.0.3
 type: plugin
 author: davidkhala
 name: html_extractor

tests/html_test.py

@@ -20,8 +20,8 @@ def test_html_extractor_parses_file(self):
         result = extractor.extract()
 
         self.assertIsInstance(result, ExtractorResult)
-        self.assertIn("Hello World", result.md_content)
-        self.assertIn("test@example.com", result.md_content)
+        self.assertIn("Hello World", result.text)
+        self.assertIn("test@example.com", result.text)
 
     def test_thei_page(self):
         html_content = """
@@ -79,16 +79,16 @@ def test_thei_page(self):
 </html>
         """
 
-        no_footer = HtmlExtractor(html_content, "", "-").extract().md_content
+        no_footer = HtmlExtractor(html_content, "", "-").extract().text
         self.assertNotIn("Copyright", no_footer)
-        no_header = HtmlExtractor(html_content, "-", "").extract().md_content
+        no_header = HtmlExtractor(html_content, "-", "").extract().text
         self.assertNotIn("THEi 设计与建筑系", no_header)
     def test_real_page(self):
         with open("tests/fixtures/Bachelor of Engineering (Honours) in Building Services Engineering - Technological and Higher Education Institute of Hong Kong.html", 'r', encoding='utf-8') as f:
             html_content = f.read()
         header_classes ="mobile-menu-wrapper, breadcrumbs-wrapper, popup-login-wrapper, toolbar"
         footer_classes = 'footer,header-info-swapper,thim-widget-button,elementor-widget-social-icons'
-        clean = HtmlExtractor(html_content, header_classes, footer_classes).extract().md_content
+        clean = HtmlExtractor(html_content, header_classes, footer_classes).extract().text
         self.assertNotIn("About", clean)
         self.assertNotIn("Copyright", clean)
         self.assertNotIn("Apply now", clean)
@@ -97,8 +97,6 @@ def test_real_page(self):
         self.assertIn("Fluid Mechanics", clean)
         self.assertIn("4,230", clean)
         print(clean)
-        with open("output.txt", "w", encoding="utf-8") as f:
-            f.write(clean)
 
 
 if __name__ == '__main__':

tools/__init__.py

@@ -16,6 +16,5 @@ def _invoke(self, tool_parameters: dict[str, Any]) -> Generator[ToolInvokeMessag
         remove_footer = tool_parameters.get("footer_class")
 
         extractor = HtmlExtractor(html_content, remove_header, remove_footer)
-
-        extractor_result = extractor.extract()
-        yield self.create_text_message(extractor_result.md_content)
+        t = extractor.extract().text
+        yield self.create_text_message(t)

tools/document.py

@@ -1,39 +1,11 @@
 from typing import Optional
 
-from dify_plugin.invocations.file import UploadFileResponse
-from pydantic import BaseModel, Field
-
-
-class ChildDocument(BaseModel):
-    """Class for storing a piece of text and associated metadata."""
-    page_content: str
-    vector: Optional[list[float]] = None
-    """Arbitrary metadata about the page content (e.g., source, relationships to other
-        documents, etc.).
-    """
-    metadata: dict = Field(default_factory=dict)
-
-
-class Document(ChildDocument):
-    provider:Optional[str] = "davidkhala"
-    children: Optional[list[ChildDocument]] = None
-
-    def to_dict(self) -> dict:
-        return {
-            "page_content": self.page_content,
-            "vector": self.vector if self.vector is not None else None,
-            "metadata": self.metadata,
-            "provider": self.provider,
-            "children": [child.model_dump() for child in self.children]
-            if self.children
-            else None,
-        }
+from pydantic import BaseModel
 
 
 class ExtractorResult(BaseModel):
     """Class for storing the result of an extractor."""
 
-    md_content: str
-    documents: Optional[list[Document]] = None
-    img_list: Optional[list[UploadFileResponse]] = None
-    origin_result: Optional[dict] = None
+    text: str
+    files: Optional[list] = None
+    json: Optional[list[dict]] = None

tools/extractor_base.py

@@ -3,6 +3,5 @@
 
 class BaseExtractor(ABC):
 
-    @abstractmethod
     def extract(self):
-        raise NotImplementedError
+        ...

tools/html_extractor.py

@@ -19,7 +19,7 @@ def __init__(self, html_content: str, remove_header: str, remove_footer: str):
 
     def extract(self) -> ExtractorResult:
         return ExtractorResult(
-            md_content=self._load_as_text(),
+            text=self._load_as_text(),
         )
 
     @staticmethod
@@ -45,7 +45,6 @@ def _load_as_text(self) -> str:
                 tag.decompose()
             HtmlExtractor.class_remover(soup, self.remove_footer)
 
-        text = soup.get_text()
+        text = soup.get_text(separator='\n')
         text = re.sub(r'\n{3,}', '\n\n', text)
-        text = re.sub(r'(?<!\n)\n(?!\n)', '\n\n', text)
         return text.strip() if text else ""