Changed test model from isaacus/emubert to isaacus/kanon-tokenizer.

Author: umarbutler

CHANGELOG.md

@@ -1,6 +1,10 @@
 ## Changelog 🔄
 All notable changes to `semchunk` will be documented here. This project adheres to [Keep a Changelog](https://keepachangelog.com/en/1.1.0/) and [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [3.1.2] - 2025-03-06
+### Changed
+- Changed test model from `isaacus/emubert` to `isaacus/kanon-tokenizer`.
+
 ## [3.1.1] - 2025-02-18
 ### Added
 - Added a note to the quickstart section of the README advising users to deduct the number of special tokens automatically added by their tokenizer from their chunk size. This note had been removed in version 3.0.0 but has been readded as it is unlikely to be obvious to users.

README.md

@@ -43,10 +43,10 @@ text = 'The quick brown fox jumps over the lazy dog.'
 # OpenAI `tiktoken` encoding or Hugging Face model, or a custom tokenizer that has an `encode()`
 # method (like a `tiktoken`, `transformers` or `tokenizers` tokenizer) or a custom token counting
 # function that takes a text and returns the number of tokens in it.
-chunker = semchunk.chunkerify('isaacus/emubert', chunk_size) or \
+chunker = semchunk.chunkerify('isaacus/kanon-tokenizer', chunk_size) or \
           semchunk.chunkerify('gpt-4', chunk_size) or \
           semchunk.chunkerify('cl100k_base', chunk_size) or \
-          semchunk.chunkerify(AutoTokenizer.from_pretrained('isaacus/emubert'), chunk_size) or \
+          semchunk.chunkerify(AutoTokenizer.from_pretrained('isaacus/kanon-tokenizer'), chunk_size) or \
           semchunk.chunkerify(tiktoken.encoding_for_model('gpt-4'), chunk_size) or \
           semchunk.chunkerify(lambda text: len(text.split()), chunk_size)
 

pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "semchunk"
-version = "3.1.1"
+version = "3.1.2"
 authors = [
     {name="Isaacus", email="[email protected]"},
     {name="Umar Butler", email="[email protected]"},

tests/helpers.py

@@ -37,7 +37,7 @@ def initialize_test_token_counters() -> dict[str, Callable[[str], int]]:
     """Initialize `tiktoken`, `transformers`, character and word token counters for testing purposes."""
 
     gpt4_tiktoken_tokenizer = tiktoken.encoding_for_model('gpt-4').encode
-    emubert_transformers_tokenizer = make_transformers_tokenizer(transformers.AutoTokenizer.from_pretrained('isaacus/emubert'))
+    emubert_transformers_tokenizer = make_transformers_tokenizer(transformers.AutoTokenizer.from_pretrained('isaacus/kanon-tokenizer'))
 
     def word_tokenizer(text: str) -> list[str]:
         """Tokenize a text into words."""

tests/test_semchunk.py

@@ -159,7 +159,7 @@ def test_semchunk() -> None:
     assert error_raised
 
     # Test using `tiktoken` tokenizers, encodings and a `transformers` tokenizer by name with `chunkerify()`.
-    for name in ['cl100k_base', 'gpt-4', 'isaacus/emubert']:
+    for name in ['cl100k_base', 'gpt-4', 'isaacus/kanon-tokenizer']:
         chunker = semchunk.chunkerify(name, 1)
         chunker(DETERMINISTIC_TEST_INPUT)
         if TEST_OFFSETS: chunker(DETERMINISTIC_TEST_INPUT, offsets = True)
@@ -175,7 +175,7 @@ def test_semchunk() -> None:
     assert error_raised
 
     # Test using a `transformers` tokenizer directly.
-    tokenizer = AutoTokenizer.from_pretrained('isaacus/emubert')
+    tokenizer = AutoTokenizer.from_pretrained('isaacus/kanon-tokenizer')
     chunker = semchunk.chunkerify(tokenizer, 1)
 
     # Test using a `tiktoken` tokenizer directly.