rfctr(chunk): prep for adding TableSplitter (#3510)

**Summary**
Mechanical refactoring in preparation for adding (pre-chunk)
`TableSplitter` in a PR stacked on this one.

Author: scanny

CHANGELOG.md

@@ -1,4 +1,4 @@
-## 0.15.2-dev6
+## 0.15.2-dev7
 
 ### Enhancements
 

test_unstructured/chunking/test_base.py

@@ -0,0 +1,33 @@
+"""Unit-test suite for the `unstructured.common.html_table` module."""
+
+from __future__ import annotations
+
+from unstructured.common.html_table import htmlify_matrix_of_cell_texts
+
+
+class Describe_htmlify_matrix_of_cell_texts:
+    """Unit-test suite for `unstructured.common.html_table.htmlify_matrix_of_cell_texts()`."""
+
+    def test_htmlify_matrix_handles_empty_cells(self):
+        assert htmlify_matrix_of_cell_texts([["cell1", "", "cell3"], ["", "cell5", ""]]) == (
+            "<table>"
+            "<tr><td>cell1</td><td></td><td>cell3</td></tr>"
+            "<tr><td></td><td>cell5</td><td></td></tr>"
+            "</table>"
+        )
+
+    def test_htmlify_matrix_handles_special_characters(self):
+        assert htmlify_matrix_of_cell_texts([['<>&"', "newline\n"]]) == (
+            "<table><tr><td>&lt;&gt;&amp;&quot;</td><td>newline<br/></td></tr></table>"
+        )
+
+    def test_htmlify_matrix_handles_multiple_rows_and_cells(self):
+        assert htmlify_matrix_of_cell_texts([["cell1", "cell2"], ["cell3", "cell4"]]) == (
+            "<table>"
+            "<tr><td>cell1</td><td>cell2</td></tr>"
+            "<tr><td>cell3</td><td>cell4</td></tr>"
+            "</table>"
+        )
+
+    def test_htmlify_matrix_handles_empty_matrix(self):
+        assert htmlify_matrix_of_cell_texts([]) == ""

test_unstructured/common/__init__.py

@@ -339,30 +339,6 @@ def test_validate_date_args_raises_for_invalid_formats(date):
         assert utils.validate_date_args(date)
 
 
-def test_htmlify_matrix_handles_empty_cells():
-    assert utils.htmlify_matrix_of_cell_texts([["cell1", "", "cell3"], ["", "cell5", ""]]) == (
-        "<table><tr><td>cell1</td><td></td><td>cell3</td></tr>"
-        "<tr><td></td><td>cell5</td><td></td></tr></table>"
-    )
-
-
-def test_htmlify_matrix_handles_special_characters():
-    assert utils.htmlify_matrix_of_cell_texts([['<>&"', "newline\n"]]) == (
-        "<table><tr><td>&lt;&gt;&amp;&quot;</td><td>newline<br/></td></tr></table>"
-    )
-
-
-def test_htmlify_matrix_handles_multiple_rows_and_cells():
-    assert utils.htmlify_matrix_of_cell_texts([["cell1", "cell2"], ["cell3", "cell4"]]) == (
-        "<table><tr><td>cell1</td><td>cell2</td></tr>"
-        "<tr><td>cell3</td><td>cell4</td></tr></table>"
-    )
-
-
-def test_htmlify_matrix_handles_empty_matrix():
-    assert utils.htmlify_matrix_of_cell_texts([]) == ""
-
-
 def test_only_returns_singleton_iterable():
     singleton_iterable = [42]
     result = utils.only(singleton_iterable)

test_unstructured/common/test_html_table.py

@@ -6,7 +6,7 @@ from typing import Any, Callable, Collection, Protocol, TypeVar
 
 from typing_extensions import TypeAlias
 
-from .etree import QName, _Element, _ElementTree
+from .etree import HTMLParser, QName, XMLParser, _Element, _ElementTree
 
 _ET = TypeVar("_ET", bound=_Element, default=_Element)
 _ET_co = TypeVar("_ET_co", bound=_Element, default=_Element, covariant=True)
@@ -30,5 +30,8 @@ _TextArg: TypeAlias = str | bytes | QName
 
 _XPathObject = Any
 
+# The basic parsers bundled in lxml.etree
+_DefEtreeParsers = XMLParser[_ET_co] | HTMLParser[_ET_co]
+
 class SupportsLaxedItems(Protocol[_KT_co, _VT_co]):
     def items(self) -> Collection[tuple[_KT_co, _VT_co]]: ...

test_unstructured/test_utils.py

@@ -2,11 +2,12 @@
 
 from __future__ import annotations
 
-from typing import Collection, Generic, Iterator, TypeVar, overload
+from typing import Collection, Generic, Iterable, Iterator, TypeVar, overload
 
 from typing_extensions import Self
 
 from .. import _types as _t
+from ._module_misc import CDATA, QName
 
 _T = TypeVar("_T")
 
@@ -23,6 +24,12 @@ class _Element:
     def get(self, key: _t._AttrName) -> str | None: ...
     @overload
     def get(self, key: _t._AttrName, default: _T) -> str | _T: ...
+    @overload
+    def iter(self, *tags: _t._TagSelector) -> Iterator[Self]: ...
+    @overload
+    def iter(
+        self, *, tag: _t._TagSelector | Iterable[_t._TagSelector] | None = None
+    ) -> Iterator[Self]: ...
     def iterancestors(
         self, *, tag: _t._TagSelector | Collection[_t._TagSelector] | None = None
     ) -> Iterator[Self]: ...
@@ -39,8 +46,12 @@ class _Element:
     def tag(self) -> str: ...
     @property
     def tail(self) -> str | None: ...
+    @tail.setter
+    def tail(self, value: str | CDATA | None) -> None: ...
     @property
     def text(self) -> str | None: ...
+    @text.setter
+    def text(self, value: str | QName | CDATA | None) -> None: ...
     def xpath(
         self,
         _path: str,

typings/lxml/_types.pyi

@@ -2,4 +2,7 @@
 
 from __future__ import annotations
 
+class CDATA:
+    def __init__(self, data: str) -> None: ...
+
 class QName: ...

typings/lxml/etree/_element.pyi

@@ -1,8 +1,16 @@
+# pyright: reportPrivateUsage=false
+
 from __future__ import annotations
 
+from typing import Generic
+
+from .._types import _ET_co
 from ._classlookup import ElementClassLookup
 
-class HTMLParser:
+# Includes most stuff in _BaseParser
+class _FeedParser(Generic[_ET_co]): ...
+
+class HTMLParser(_FeedParser[_ET_co]):
     def __init__(
         self,
         *,
@@ -20,7 +28,7 @@ class HTMLParser:
     ) -> None: ...
     def set_element_class_lookup(self, lookup: ElementClassLookup | None = None) -> None: ...
 
-class XMLParser:
+class XMLParser(_FeedParser[_ET_co]):
     def __init__(
         self,
         *,

typings/lxml/etree/_module_misc.pyi

@@ -0,0 +1,8 @@
+from __future__ import annotations
+
+from ._element import (
+    HtmlElement as HtmlElement,
+)
+from ._parse import (
+    fragment_fromstring as fragment_fromstring,
+)