Skip to content

Commit 0c76a10

Browse files
committed
fix count tokens
1 parent f041e64 commit 0c76a10

File tree

1 file changed

+2
-2
lines changed

1 file changed

+2
-2
lines changed

src/agentlab/llm/llm_utils.py

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -188,8 +188,8 @@ def get_tokenizer(model_name="gpt-4"):
188188
logging.info(f"Could not find a tokenizer for model {model_name}. Trying HuggingFace.")
189189
try:
190190
return AutoTokenizer.from_pretrained(model_name)
191-
except OSError:
192-
logging.info(f"Could not find a tokenizer for model {model_name}. Defaulting to gpt-4.")
191+
except Exception as e:
192+
logging.info(f"Could not find a tokenizer for model {model_name}: {e} Defaulting to gpt-4.")
193193
return tiktoken.encoding_for_model("gpt-4")
194194

195195

0 commit comments

Comments
 (0)