Better error reporting in get_default_tokenizer()

Author: cpsievert

shiny/ui/_chat.py

@@ -914,15 +914,6 @@ def _get_token_count(
         if self._tokenizer is None:
             self._tokenizer = get_default_tokenizer()
 
-        if self._tokenizer is None:
-            raise ValueError(
-                "A tokenizer is required to impose `token_limits` on messages. "
-                "To get a generic default tokenizer, install the `tokenizers` "
-                "package (`pip install tokenizers`). "
-                "To get a more precise token count, provide a specific tokenizer "
-                "to the `Chat` constructor."
-            )
-
         encoded = self._tokenizer.encode(content)
         if isinstance(encoded, TokenizersEncoding):
             return len(encoded.ids)

shiny/ui/_chat_tokenizer.py

@@ -45,12 +45,23 @@ def encode(
 TokenEncoding = Union[TiktokenEncoding, TokenizersTokenizer]
 
 
-def get_default_tokenizer() -> TokenizersTokenizer | None:
+def get_default_tokenizer() -> TokenizersTokenizer:
     try:
         from tokenizers import Tokenizer
 
         return Tokenizer.from_pretrained("bert-base-cased")  # type: ignore
-    except Exception:
-        pass
-
-    return None
+    except ImportError:
+        raise ValueError(
+            "A tokenizer is required to impose `token_limits` on messages. "
+            "To get a generic default tokenizer, install the `tokenizers` "
+            "package (`pip install tokenizers`). "
+            "To get a more precise token count, provide a specific tokenizer "
+            "to the `Chat` constructor."
+        )
+    except Exception as e:
+        raise ValueError(
+            "Failed to load the default tokenizer. "
+            "Ensure that the `tokenizers` package is installed and "
+            "that the `bert-base-cased` model is available. "
+            f"Error: {e}"
+        ) from e