fix(HybridChunker): remove max_length from tokenization (#178)

Signed-off-by: Panos Vagenas <[email protected]>

Author: vagenas

docling_core/transforms/chunker/hybrid_chunker.py

@@ -73,7 +73,7 @@ def _count_text_tokens(self, text: Optional[Union[str, list[str]]]):
             for t in text:
                 total += self._count_text_tokens(t)
             return total
-        return len(self._tokenizer.tokenize(text, max_length=None))
+        return len(self._tokenizer.tokenize(text))
 
     class _ChunkLengthInfo(BaseModel):
         total_len: int
@@ -82,7 +82,7 @@ class _ChunkLengthInfo(BaseModel):
 
     def _count_chunk_tokens(self, doc_chunk: DocChunk):
         ser_txt = self.serialize(chunk=doc_chunk)
-        return len(self._tokenizer.tokenize(text=ser_txt, max_length=None))
+        return len(self._tokenizer.tokenize(text=ser_txt))
 
     def _doc_chunk_length(self, doc_chunk: DocChunk):
         text_length = self._count_text_tokens(doc_chunk.text)

test/test_hybrid_chunker.py

@@ -98,7 +98,7 @@ def test_serialize():
             dict(
                 text=chunk.text,
                 ser_text=(ser_text := chunker.serialize(chunk)),
-                num_tokens=len(TOKENIZER.tokenize(ser_text, max_length=None)),
+                num_tokens=len(TOKENIZER.tokenize(ser_text)),
             )
             for chunk in chunks
         ]
@@ -171,7 +171,7 @@ def test_serialize_altered_delim():
             dict(
                 text=chunk.text,
                 ser_text=(ser_text := chunker.serialize(chunk)),
-                num_tokens=len(TOKENIZER.tokenize(ser_text, max_length=None)),
+                num_tokens=len(TOKENIZER.tokenize(ser_text)),
             )
             for chunk in chunks
         ]