We read every piece of feedback, and take your input very seriously.
To see all available qualifiers, see our documentation.
There was an error while loading. Please reload this page.
1 parent 90bf1cc commit a044ff0Copy full SHA for a044ff0
tests/test_semchunk.py
@@ -23,7 +23,7 @@ def tiktoken_token_counter(text: str) -> int:
23
return len(tiktoken_tokenizer.encode(text))
24
25
# Test chunking with a variety of chunk sizes.
26
- for chunk_size in {1, 2, 512}:
+ for chunk_size in {1, 512, 1024}:
27
# Test chunking with a variety of texts.
28
for fileid in {'austen-emma.txt', 'carroll-alice.txt', 'shakespeare-macbeth.txt'}:
29
sample = gutenberg.raw(fileid)
0 commit comments