Merge pull request #254 from ServiceNow/fix_count_tokens

Fix count tokens

Author: amanjaiswal73892

src/agentlab/llm/llm_utils.py

@@ -516,8 +516,8 @@ def get_tokenizer(model_name="gpt-4"):
         logging.info(f"Could not find a tokenizer for model {model_name}. Trying HuggingFace.")
     try:
         return AutoTokenizer.from_pretrained(model_name)
-    except OSError:
-        logging.info(f"Could not find a tokenizer for model {model_name}. Defaulting to gpt-4.")
+    except Exception as e:
+        logging.info(f"Could not find a tokenizer for model {model_name}: {e} Defaulting to gpt-4.")
     return tiktoken.encoding_for_model("gpt-4")