add fenced-code extension to the md parser (#4044)

jiajun-unstructured · qued · web-flow · commit 92965fb28670 · 2025-07-07T21:05:54.000Z
#3578 --------- Co-authored-by: qued <64741807+qued@users.noreply.github.com> Co-authored-by: Alan Bertl <alan@unstructured.io>
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,3 +1,12 @@
+## 0.18.5-dev0
+
+### Enhancements
+
+### Features
+
+### Fixes
+- **Improve markdown code block handling** Code blocks in markdown were previously being processed as embedded code instead of plain text.
+
 ## 0.18.4
 
 ### Enhancements
diff --git a/example-docs/codeblock.md b/example-docs/codeblock.md
@@ -0,0 +1,33 @@
+## HTML Example
+
+```html
+<!DOCTYPE html>
+<html lang="en">
+<head>
+    <meta charset="UTF-8">
+    <meta name="viewport" content="width=device-width, initial-scale=1.0">
+    <title>Sample HTML</title>
+</head>
+<body>
+    <h1>Hello, World!</h1>
+    <p>This is a simple HTML example.</p>
+</body>
+</html>
+```
+## XML Example
+```xml
+<note>
+    <to>Tove</to>
+    <from>Jani</from>
+    <heading>Reminder</heading>
+    <body>Don't forget me this weekend!</body>
+</note>
+```
+```xml
+<note>
+    <to>Tove</to>
+    <from>Jani</from>
+    <heading>Reminder</heading>
+    <body>Don't forget me this weekend!</body>
+</note>
+```
diff --git a/test_unstructured/partition/test_md.py b/test_unstructured/partition/test_md.py
@@ -296,3 +296,39 @@ def test_partition_md_non_xml_processing_instruction():
 
     elements = partition_md(text=php_content)
     assert len(elements) == 1
+
+
+def test_partition_fenced_code():
+    filename = example_doc_path("codeblock.md")
+    elements = partition_md(filename=filename)
+
+    # Should have 5 elements: 2 titles and 3 code blocks
+    assert len(elements) == 5
+
+    assert elements[0].text == "HTML Example"
+
+    expected_html = """<!DOCTYPE html>
+<html lang="en">
+<head>
+    <meta charset="UTF-8">
+    <meta name="viewport" content="width=device-width, initial-scale=1.0">
+    <title>Sample HTML</title>
+</head>
+<body>
+    <h1>Hello, World!</h1>
+    <p>This is a simple HTML example.</p>
+</body>
+</html>"""
+    assert elements[1].text == expected_html
+
+    assert elements[2].text == "XML Example"
+
+    expected_xml = """<note>
+    <to>Tove</to>
+    <from>Jani</from>
+    <heading>Reminder</heading>
+    <body>Don't forget me this weekend!</body>
+</note>"""
+    assert elements[3].text == expected_xml
+
+    assert elements[4].text == expected_xml
diff --git a/unstructured/__version__.py b/unstructured/__version__.py
@@ -1 +1 @@
-__version__ = "0.18.4"  # pragma: no cover
+__version__ = "0.18.5-dev0"  # pragma: no cover
diff --git a/unstructured/partition/md.py b/unstructured/partition/md.py
@@ -1,7 +1,6 @@
 from __future__ import annotations
 
-import re
-from typing import IO, Any, Match
+from typing import IO, Any
 
 import markdown
 import requests
@@ -23,30 +22,6 @@ def optional_decode(contents: str | bytes) -> str:
 DETECTION_ORIGIN: str = "md"
 
 
-def _preprocess_markdown_code_blocks(text: str) -> str:
-    """Pre-process code blocks so that processing instructions can be properly escaped.
-
-    The markdown library can fail to properly escape processing instructions like <?xml>, <?php>,
-    etc. in code blocks. This function adds minimal indentation to the processing instruction line
-    to force markdown to treat it as text content rather than XML.
-    """
-    # Breakdown of the regex:
-    # ```\s*\n           - Opening triple backticks + optional whitespace + newline
-    # ([ \t]{0,3})?      - Capture group 1: optional 0-3 spaces/tabs (existing indentation)
-    # (<\?[a-zA-Z][^>]*\?>.*?) - Capture group 2: processing instruction + any following content
-    # \n?```             - Optional newline + closing triple backticks
-    code_block_pattern = r"```\s*\n([ \t]{0,3})?(<\?[a-zA-Z][^>]*\?>.*?)\n?```"
-
-    def indent_processing_instruction(match: Match[str]) -> str:
-        content = match.group(2)
-        # Ensure processing instruction has at least 4-space indentation
-        if content.lstrip().startswith("<?"):
-            content = "    " + content.lstrip()
-        return f"```\n{content}\n```"
-
-    return re.sub(code_block_pattern, indent_processing_instruction, text, flags=re.DOTALL)
-
-
 def partition_md(
     filename: str | None = None,
     file: IO[bytes] | None = None,
@@ -98,8 +73,7 @@ def partition_md(
 
         text = response.text
 
-    processed_text = _preprocess_markdown_code_blocks(text)
-    html = markdown.markdown(processed_text, extensions=["tables"])
+    html = markdown.markdown(text, extensions=["tables", "fenced_code"])
 
     return partition_html(
         text=html,

Original file line number	Diff line number	Diff line change
`@@ -1 +1 @@`
`1`		`-__version__ = "0.18.4" # pragma: no cover`
	`1`	`+__version__ = "0.18.5-dev0" # pragma: no cover`