fix(markdown): fix escaping in case of nesting (#180)

Signed-off-by: Panos Vagenas <[email protected]>

Author: vagenas

docling_core/types/doc/document.py

@@ -2487,7 +2487,6 @@ def _ingest_text(text: str, do_escape_html=True, do_escape_underscores=True):
                         is_inline_scope=is_inline_scope,
                         visited=visited,
                     )
-                    # NOTE: assumes unordered (flag & marker currently in ListItem)
                     indent_str = list_level * indent * " "
                     is_ol = item.label == GroupLabel.ORDERED_LIST
                     text = "\n".join(
@@ -2501,7 +2500,12 @@ def _ingest_text(text: str, do_escape_html=True, do_escape_underscores=True):
                             for i, c in enumerate(comps)
                         ]
                     )
-                    _ingest_text(text=text)
+                    _ingest_text(
+                        text=text,
+                        # special chars have already been escaped as needed
+                        do_escape_html=False,
+                        do_escape_underscores=False,
+                    )
                 elif item.label == GroupLabel.INLINE:
                     comps = self._get_markdown_components(
                         node=item,
@@ -2520,7 +2524,13 @@ def _ingest_text(text: str, do_escape_html=True, do_escape_underscores=True):
                         is_inline_scope=True,
                         visited=visited,
                     )
-                    _ingest_text(" ".join(comps))
+                    text = " ".join(comps)
+                    _ingest_text(
+                        text=text,
+                        # special chars have already been escaped as needed
+                        do_escape_html=False,
+                        do_escape_underscores=False,
+                    )
                 else:
                     continue
 

test/data/doc/2206.01062.yaml.md

@@ -55,7 +55,7 @@ In this paper, we present the DocLayNet dataset. It provides pageby-page layout
 
 This enables experimentation with annotation uncertainty and quality control analysis.
 
-- (5) Pre-defined Train-, Test- & Validation-set : Like DocBank, we provide fixed train-, test- & validation-sets to ensure proportional representation of the class-labels. Further, we prevent leakage of unique layouts across sets, which has a large effect on model accuracy scores.
+- (5) Pre-defined Train-, Test- & Validation-set : Like DocBank, we provide fixed train-, test- & validation-sets to ensure proportional representation of the class-labels. Further, we prevent leakage of unique layouts across sets, which has a large effect on model accuracy scores.
 
 All aspects outlined above are detailed in Section 3. In Section 4, we will elaborate on how we designed and executed this large-scale human annotation campaign. We will also share key insights and lessons learned that might prove helpful for other parties planning to set up annotation campaigns.
 

test/data/doc/constructed_doc.dt

@@ -30,7 +30,7 @@ Affiliation 2</text>
 <list_item>item 2 of neighboring list</list_item>
 <unordered_list><list_item>item 1 of sub list</list_item>
 <paragraph>Here a code snippet:</paragraph>
-<code<_unknown_>print("Hello world")</code
+<code<_unknown_><p>Hello world</p></code
 <paragraph>(to be displayed inline)</paragraph>
 </unordered_list>
 <paragraph>Here a formula:</paragraph>

test/data/doc/constructed_doc.dt.gt

@@ -30,7 +30,7 @@ Affiliation 2</text>
 <list_item>item 2 of neighboring list</list_item>
 <unordered_list><list_item>item 1 of sub list</list_item>
 <paragraph>Here a code snippet:</paragraph>
-<code<_unknown_>print("Hello world")</code
+<code<_unknown_><p>Hello world</p></code
 <paragraph>(to be displayed inline)</paragraph>
 </unordered_list>
 <paragraph>Here a formula:</paragraph>

test/data/doc/constructed_doc.embedded.html.gt

@@ -108,7 +108,7 @@
 <ul>
 <li>item 1 of sub list</li>
 <p>Here a code snippet:</p>
-<pre><code>print("Hello world")</code></pre>
+<pre><code><p>Hello world</p></code></pre>
 <p>(to be displayed inline)</p>
 </ul>
 <p>Here a formula:</p>

test/data/doc/constructed_doc.embedded.json.gt

@@ -636,8 +636,8 @@
       "content_layer": "body",
       "label": "code",
       "prov": [],
-      "orig": "print(\"Hello world\")",
-      "text": "print(\"Hello world\")",
+      "orig": "<p>Hello world</p>",
+      "text": "<p>Hello world</p>",
       "captions": [],
       "references": [],
       "footnotes": [],

test/data/doc/constructed_doc.embedded.md.gt

@@ -44,7 +44,7 @@ This is the caption of figure 2.
 - item 1 of neighboring list
 - item 2 of neighboring list
     - item 1 of sub list
-    - Here a code snippet: `print("Hello world")` (to be displayed inline)
+    - Here a code snippet: `<p>Hello world</p>` (to be displayed inline)
     - Here a formula: $E=mc^2$ (to be displayed inline)
 
 Here a code block:

test/data/doc/constructed_doc.embedded.yaml.gt

@@ -676,13 +676,13 @@ texts:
   content_layer: body
   footnotes: []
   label: code
-  orig: print("Hello world")
+  orig: <p>Hello world</p>
   parent:
     $ref: '#/groups/10'
   prov: []
   references: []
   self_ref: '#/texts/24'
-  text: print("Hello world")
+  text: <p>Hello world</p>
 - children: []
   content_layer: body
   label: paragraph

test/data/doc/constructed_doc.placeholder.html.gt

@@ -108,7 +108,7 @@
 <ul>
 <li>item 1 of sub list</li>
 <p>Here a code snippet:</p>
-<pre><code>print("Hello world")</code></pre>
+<pre><code><p>Hello world</p></code></pre>
 <p>(to be displayed inline)</p>
 </ul>
 <p>Here a formula:</p>

test/data/doc/constructed_doc.placeholder.md.gt

@@ -44,7 +44,7 @@ This is the caption of figure 2.
 - item 1 of neighboring list
 - item 2 of neighboring list
     - item 1 of sub list
-    - Here a code snippet: `print("Hello world")` (to be displayed inline)
+    - Here a code snippet: `<p>Hello world</p>` (to be displayed inline)
     - Here a formula: $E=mc^2$ (to be displayed inline)
 
 Here a code block: