We read every piece of feedback, and take your input very seriously.
To see all available qualifiers, see our documentation.
There was an error while loading. Please reload this page.
1 parent b5ed725 commit b3ae288Copy full SHA for b3ae288
graphgen/models/tokenizer/__init__.py
@@ -3,7 +3,6 @@
3
4
from graphgen.bases import BaseTokenizer
5
6
-from .hf_tokenizer import HFTokenizer
7
from .tiktoken_tokenizer import TiktokenTokenizer
8
9
try:
@@ -22,6 +21,8 @@ def get_tokenizer_impl(tokenizer_name: str = "cl100k_base") -> BaseTokenizer:
22
21
23
# 2. HuggingFace
24
if _HF_AVAILABLE:
+ from .hf_tokenizer import HFTokenizer
25
+
26
return HFTokenizer(model_name=tokenizer_name)
27
28
raise ValueError(
0 commit comments