Merge pull request #741 from parea-ai/fix-get-tokens-no-model

fix: get tokens if no model is provided

Author: joschkabraun

parea/evals/utils.py

@@ -169,10 +169,11 @@ def run_evals_synchronous(trace_id: str, log: Log, eval_funcs: List[EvalFuncTupl
 def get_tokens(model: str, text: str) -> List[int]:
     if not text:
         return []
+    fallback_model = "cl100k_base"
     try:
-        encoding = tiktoken.encoding_for_model(model)
+        encoding = tiktoken.encoding_for_model(model or fallback_model)
     except KeyError:
-        encoding = tiktoken.get_encoding("cl100k_base")
+        encoding = tiktoken.get_encoding(fallback_model)
     try:
         return encoding.encode(text)
     except Exception as e:

pyproject.toml

@@ -6,7 +6,7 @@ build-backend = "poetry.core.masonry.api"
 [tool.poetry]
 name = "parea-ai"
 packages = [{ include = "parea" }]
-version = "0.2.129"
+version = "0.2.130"
 description = "Parea python sdk"
 readme = "README.md"
 authors = ["joel-parea-ai <[email protected]>"]