feat: remodel lists, add MD & HTML ser. params, enable unset marker (#339)

* feat: remodel lists

Signed-off-by: Panos Vagenas <[email protected]>

* prepare test document separate markers

Signed-off-by: Panos Vagenas <[email protected]>

* add MD/HTML serializer options, expand test data

Signed-off-by: Panos Vagenas <[email protected]>

* create list groups where not in place

Signed-off-by: Panos Vagenas <[email protected]>

* add auto-increment logic

Signed-off-by: Panos Vagenas <[email protected]>

* restore UnorderedList as deprecated alias (for backwards compatibility)

Signed-off-by: Panos Vagenas <[email protected]>

* enable unset marker case

Signed-off-by: Panos Vagenas <[email protected]>

* make ordered markers able to be "unset" (empty) too

Signed-off-by: Panos Vagenas <[email protected]>

* rename default marker mode to auto

Signed-off-by: Panos Vagenas <[email protected]>

---------

Signed-off-by: Panos Vagenas <[email protected]>

Author: vagenas

docling_core/transforms/chunker/hierarchical_chunker.py

@@ -35,11 +35,10 @@
     DocumentOrigin,
     InlineGroup,
     LevelNumber,
-    OrderedList,
+    ListGroup,
     SectionHeaderItem,
     TableItem,
     TitleItem,
-    UnorderedList,
 )
 
 _VERSION: Final = "1.0.0"
@@ -240,7 +239,7 @@ def chunk(
                     heading_by_level.pop(k, None)
                 continue
             elif (
-                isinstance(item, (OrderedList, UnorderedList, InlineGroup, DocItem))
+                isinstance(item, (ListGroup, InlineGroup, DocItem))
                 and item.self_ref not in visited
             ):
                 ser_res = my_doc_ser.serialize(item=item, visited=visited)

docling_core/transforms/serializer/base.py

@@ -17,12 +17,11 @@
     FormItem,
     InlineGroup,
     KeyValueItem,
+    ListGroup,
     NodeItem,
-    OrderedList,
     PictureItem,
     TableItem,
     TextItem,
-    UnorderedList,
 )
 
 
@@ -128,7 +127,7 @@ class BaseListSerializer(ABC):
     def serialize(
         self,
         *,
-        item: Union[UnorderedList, OrderedList],
+        item: ListGroup,
         doc_serializer: "BaseDocSerializer",
         doc: DoclingDocument,
         **kwargs: Any,

docling_core/transforms/serializer/common.py

@@ -39,8 +39,8 @@
     FormItem,
     InlineGroup,
     KeyValueItem,
+    ListGroup,
     NodeItem,
-    OrderedList,
     PictureClassificationData,
     PictureDataType,
     PictureItem,
@@ -49,7 +49,6 @@
     TableAnnotationType,
     TableItem,
     TextItem,
-    UnorderedList,
 )
 from docling_core.types.doc.labels import DocItemLabel
 
@@ -89,7 +88,7 @@ def _iterate_items(
     ):
         if add_page_breaks:
             if (
-                isinstance(item, (UnorderedList, OrderedList, InlineGroup))
+                isinstance(item, (ListGroup, InlineGroup))
                 and item.self_ref not in my_visited
             ):
                 # if group starts with new page, yield page break before group node
@@ -316,7 +315,7 @@ def serialize(
         ########
         # groups
         ########
-        if isinstance(item, (UnorderedList, OrderedList)):
+        if isinstance(item, ListGroup):
             part = self.list_serializer.serialize(
                 item=item,
                 doc_serializer=self,

docling_core/transforms/serializer/doctags.py

@@ -1,7 +1,7 @@
 """Define classes for Doctags serialization."""
 
 from enum import Enum
-from typing import Any, Dict, List, Optional, Union
+from typing import Any, Dict, List, Optional
 
 from pydantic import BaseModel
 from typing_extensions import override
@@ -34,17 +34,16 @@
     FormItem,
     InlineGroup,
     KeyValueItem,
+    ListGroup,
     ListItem,
     NodeItem,
-    OrderedList,
     PictureClassificationData,
     PictureItem,
     PictureMoleculeData,
     PictureTabularChartData,
     ProvenanceItem,
     TableItem,
     TextItem,
-    UnorderedList,
 )
 from docling_core.types.doc.labels import DocItemLabel, PictureClassificationLabel
 from docling_core.types.doc.tokens import DocumentToken
@@ -376,7 +375,7 @@ class DocTagsListSerializer(BaseModel, BaseListSerializer):
     def serialize(
         self,
         *,
-        item: Union[UnorderedList, OrderedList],
+        item: ListGroup,
         doc_serializer: "BaseDocSerializer",
         doc: DoclingDocument,
         list_level: int = 0,
@@ -406,7 +405,7 @@ def serialize(
             text_res = f"{text_res}{delim}"
             wrap_tag = (
                 DocumentToken.ORDERED_LIST.value
-                if isinstance(item, OrderedList)
+                if item.first_item_is_enumerated(doc)
                 else DocumentToken.UNORDERED_LIST.value
             )
             text_res = _wrap(text=text_res, wrap_tag=wrap_tag)

docling_core/transforms/serializer/html.py

@@ -58,9 +58,9 @@
     ImageRef,
     InlineGroup,
     KeyValueItem,
+    ListGroup,
     ListItem,
     NodeItem,
-    OrderedList,
     PictureClassificationData,
     PictureItem,
     PictureMoleculeData,
@@ -70,7 +70,6 @@
     TableItem,
     TextItem,
     TitleItem,
-    UnorderedList,
 )
 from docling_core.types.doc.labels import DocItemLabel
 from docling_core.types.doc.utils import (
@@ -117,6 +116,8 @@ class HTMLParams(CommonParams):
 
     include_annotations: bool = True
 
+    show_original_list_item_marker: bool = True
+
 
 class HTMLTextSerializer(BaseModel, BaseTextSerializer):
     """HTML-specific text item serializer."""
@@ -162,7 +163,19 @@ def serialize(
         elif isinstance(item, ListItem):
             # List items are handled by list serializer
             text_inner = self._prepare_content(item.text)
-            text = get_html_tag_with_text_direction(html_tag="li", text=text_inner)
+            text = (
+                get_html_tag_with_text_direction(
+                    html_tag="li",
+                    text=text_inner,
+                    attrs=(
+                        {"style": f"list-style-type: '{item.marker} ';"}
+                        if params.show_original_list_item_marker and item.marker
+                        else {}
+                    ),
+                )
+                if text_inner
+                else ""
+            )
 
         elif is_inline_scope:
             text = self._prepare_content(item.text)
@@ -680,7 +693,7 @@ class HTMLListSerializer(BaseModel, BaseListSerializer):
     def serialize(
         self,
         *,
-        item: Union[UnorderedList, OrderedList],
+        item: ListGroup,
         doc_serializer: "BaseDocSerializer",
         doc: DoclingDocument,
         list_level: int = 0,
@@ -690,7 +703,7 @@ def serialize(
     ) -> SerializationResult:
         """Serializes a list to HTML."""
         my_visited: set[str] = visited if visited is not None else set()
-
+        params = HTMLParams(**kwargs)
         # Get all child parts
         parts = doc_serializer.get_parts(
             item=item,
@@ -706,17 +719,51 @@ def serialize(
                 (
                     p.text
                     if (
-                        (p.text.startswith("<li>") and p.text.endswith("</li>"))
-                        or (p.text.startswith("<ol>") and p.text.endswith("</ol>"))
-                        or (p.text.startswith("<ul>") and p.text.endswith("</ul>"))
+                        (
+                            p.text.startswith(("<li>", "<li "))
+                            and p.text.endswith("</li>")
+                        )
+                        or (
+                            p.text.startswith(("<ol>", "<ol "))
+                            and p.text.endswith("</ol>")
+                        )
+                        or (
+                            p.text.startswith(("<ul>", "<ul "))
+                            and p.text.endswith("</ul>")
+                        )
+                    )
+                    else (
+                        get_html_tag_with_text_direction(
+                            html_tag="li",
+                            text=p.text,
+                            attrs=(
+                                {
+                                    "style": f"list-style-type: '{grandparent_item.marker} ';"
+                                }
+                                if params.show_original_list_item_marker
+                                and grandparent_item.marker
+                                else {}
+                            ),
+                        )
+                        if p.spans
+                        and p.spans[0].item.parent
+                        and isinstance(
+                            (parent_item := p.spans[0].item.parent.resolve(doc)),
+                            InlineGroup,
+                        )
+                        and parent_item.parent
+                        and isinstance(
+                            (grandparent_item := parent_item.parent.resolve(doc)),
+                            ListItem,
+                        )
+                        else f"<li>{p.text}</li>"
                     )
-                    else f"<li>{p.text}</li>"
                 )
                 for p in parts
             ]
         )
         if text_res:
-            tag = "ol" if isinstance(item, OrderedList) else "ul"
+            tag = "ol" if item.first_item_is_enumerated(doc) else "ul"
             text_res = f"<{tag}>\n{text_res}\n</{tag}>"
 
         return create_ser_result(text=text_res, span_source=parts)