feat(chunking): allow customizing separators for the splitter (#584)

Author: badmonster0

python/cocoindex/functions.py

@@ -1,6 +1,7 @@
 """All builtin functions."""
 
 from typing import Annotated, Any, TYPE_CHECKING
+import dataclasses
 
 from .typing import Float32, Vector, TypeAttr
 from . import op, llm
@@ -14,9 +15,20 @@ class ParseJson(op.FunctionSpec):
     """Parse a text into a JSON object."""
 
 
+@dataclasses.dataclass
+class CustomLanguageSpec:
+    """Custom language specification."""
+
+    language_name: str
+    separators_regex: list[str]
+    aliases: list[str] = dataclasses.field(default_factory=list)
+
+
 class SplitRecursively(op.FunctionSpec):
     """Split a document (in string) recursively."""
 
+    custom_languages: list[CustomLanguageSpec] = dataclasses.field(default_factory=list)
+
 
 class ExtractByLlm(op.FunctionSpec):
     """Extract information from a text using a LLM."""