Add test for encoding really big byte sequences

Author: Lőrinc

tests/test_encoding.py

@@ -7,7 +7,6 @@
 import pytest
 
 import tiktoken
-
 from .test_helpers import ENCODING_FACTORIES, MAX_EXAMPLES
 
 
@@ -61,13 +60,16 @@ def test_simple_regex():
 def test_basic_encode():
     enc = tiktoken.get_encoding("r50k_base")
     assert enc.encode("hello world") == [31373, 995]
+    assert enc.encode("a" * 1000) == [24794] * 250
 
     enc = tiktoken.get_encoding("p50k_base")
     assert enc.encode("hello world") == [31373, 995]
+    assert enc.encode("a" * 1000) == [24794] * 250
 
     enc = tiktoken.get_encoding("cl100k_base")
     assert enc.encode("hello world") == [15339, 1917]
     assert enc.encode(" \x850") == [220, 126, 227, 15]
+    assert enc.encode("a" * 1000) == [70540] * 125
 
 
 def test_encode_empty():
@@ -100,14 +102,14 @@ def test_encode_surrogate_pairs():
 def test_basic_roundtrip(make_enc):
     enc = make_enc()
     for value in (
-        "hello",
-        "hello ",
-        "hello  ",
-        " hello",
-        " hello ",
-        " hello  ",
-        "hello world",
-        "请考试我的软件！12345",
+            "hello",
+            "hello ",
+            "hello  ",
+            " hello",
+            " hello ",
+            " hello  ",
+            "hello world",
+            "请考试我的软件！12345",
     ):
         assert value == enc.decode(enc.encode(value))
         assert value == enc.decode(enc.encode_ordinary(value))