Fix tokenizer removal from chunker

Author: natoverse

packages/graphrag/graphrag/chunking/chunker_factory.py

@@ -38,8 +38,8 @@ def register_chunker(
 
 def create_chunker(
     config: ChunkingConfig,
-    encode: Callable[[str], list[int]] | None,
-    decode: Callable[[list[int]], str] | None,
+    encode: Callable[[str], list[int]] | None = None,
+    decode: Callable[[list[int]], str] | None = None,
 ) -> Chunker:
     """Create a chunker implementation based on the given configuration.
 

packages/graphrag/graphrag/prompt_tune/loader/input.py

@@ -62,7 +62,7 @@ async def load_docs_in_chunks(
         cache=NoopCache(),
     )
     tokenizer = get_tokenizer(embeddings_llm_settings)
-    chunker = create_chunker(config.chunks, tokenizer)
+    chunker = create_chunker(config.chunks, tokenizer.encode, tokenizer.decode)
     input_storage = create_storage(config.input.storage)
     input_reader = InputReaderFactory().create(
         config.input.file_type,

tests/unit/chunking/test_chunker.py

@@ -66,7 +66,7 @@ def test_basic_functionality(self, mock_get_encoding):
             strategy=ChunkStrategyType.Tokens,
         )
 
-        chunker = create_chunker(config, tokenizer=tokenizer)
+        chunker = create_chunker(config, mock_encoder.encode, mock_encoder.decode)
         chunks = chunker.chunk(input)
 
         assert len(chunks) > 0