feat: add code chunking functionality

docling_core/transforms/chunker/hierarchical_chunker.py

@@ -9,17 +9,9 @@
 
 import logging
 import re
+from abc import ABC, abstractmethod
 from enum import Enum
-from typing import (
-    TYPE_CHECKING,
-    Any,
-    ClassVar,
-    Final,
-    Iterator,
-    Literal,
-    Optional,
-    Protocol,
-)
+from typing import TYPE_CHECKING, Any, ClassVar, Final, Iterator, Literal, Optional
 
 from pydantic import ConfigDict, Field, StringConstraints, field_validator
 from typing_extensions import Annotated, override
@@ -161,9 +153,10 @@ class CodeChunkType(str, Enum):
     CODE_BLOCK = "code_block"
 
 
-class CodeChunkingStrategy(Protocol):
+class CodeChunkingStrategy(ABC):
     """Protocol for code chunking strategies that can be plugged into HierarchicalChunker."""
 
+    @abstractmethod
     def chunk_code_item(
         self, code_text: str, language: Language, **kwargs: Any
     ) -> Iterator[CodeChunk]:
@@ -317,11 +310,13 @@ def chunk(
                     )
                     is not None
                 ):
-                    ser_res = my_doc_ser.serialize(item=item, visited=visited)
+                    # Serialize without markdown formatting for code items that will be parsed by tree-sitter
+                    ser_res = my_doc_ser.serialize(
+                        item=item, visited=visited, format_code_blocks=False, **kwargs
+                    )
                     if ser_res.text:
-                        code_text = self._strip_markdown_code_formatting(ser_res.text)
                         for code_chunk in self.code_chunking_strategy.chunk_code_item(
-                            code_text=code_text,
+                            code_text=ser_res.text,
                             language=language,
                             original_doc=dl_doc,
                             original_item=item,
@@ -348,14 +343,3 @@ def chunk(
                     ),
                 )
                 yield c
-
-    def _strip_markdown_code_formatting(self, text: str) -> str:
-        """Strip markdown code block formatting from text."""
-        if not text.startswith("```") or not text.endswith("```"):
-            return text
-
-        lines = text.split("\n")
-        if len(lines) >= 3 and lines[0].startswith("```") and lines[-1] == "```":
-            return "\n".join(lines[1:-1])
-
-        return text

docling_core/transforms/serializer/markdown.py

@@ -122,6 +122,10 @@ class MarkdownParams(CommonParams):
     )
     orig_list_item_marker_mode: OrigListItemMarkerMode = OrigListItemMarkerMode.AUTO
     ensure_valid_list_item_marker: bool = True
+    format_code_blocks: bool = Field(
+        default=True,
+        description="Whether to wrap code items in markdown code block formatting (```). ",
+    )
 
 
 class MarkdownTextSerializer(BaseModel, BaseTextSerializer):
@@ -222,7 +226,10 @@ def serialize(
                 num_hashes = 1 if isinstance(item, TitleItem) else item.level + 1
                 text_part = f"{num_hashes * '#'} {text}"
         elif isinstance(item, CodeItem):
-            text_part = f"`{text}`" if is_inline_scope else f"```\n{text}\n```"
+            if params.format_code_blocks:
+                text_part = f"`{text}`" if is_inline_scope else f"```\n{text}\n```"
+            else:
+                text_part = text
             escape_html = False
             escape_underscores = False
         elif isinstance(item, FormulaItem):

docling_core/types/doc/labels.py

@@ -266,14 +266,22 @@ def __str__(self):
         return str(self.value)
 
     def to_language(self):
-        """Convert CodeLanguageLabel to CodeLanguage enum."""
+        """Convert CodeLanguageLabel to Language enum for code chunking."""
         from docling_core.transforms.chunker.code_chunk_utils.utils import Language
 
-        mapping = {
-            CodeLanguageLabel.PYTHON: Language.PYTHON,
-            CodeLanguageLabel.JAVA: Language.JAVA,
-            CodeLanguageLabel.C: Language.C,
-            CodeLanguageLabel.TYPESCRIPT: Language.TYPESCRIPT,
-            CodeLanguageLabel.JAVASCRIPT: Language.JAVASCRIPT,
+        return Language.from_code_language_label(self)
+
+    @classmethod
+    def get_supported_languages(cls):
+        """Get the set of CodeLanguageLabels that are supported for code chunking."""
+        return {
+            cls.PYTHON,
+            cls.JAVA,
+            cls.C,
+            cls.TYPESCRIPT,
+            cls.JAVASCRIPT,
         }
-        return mapping.get(self, None)
+
+    def is_supported_for_chunking(self) -> bool:
+        """Check if this language is supported for code chunking."""
+        return self in self.get_supported_languages()