Add tests for humongous encodings

Author: Lőrinc

tests/test_encoding.py

@@ -11,6 +11,12 @@
 from .test_helpers import ENCODING_FACTORIES, MAX_EXAMPLES
 
 
+def test_extremely_big_encoding():
+    big_value = "^" * 1000000
+    enc = tiktoken.get_encoding("r50k_base")
+    assert big_value == enc.decode(enc.encode(big_value))
+
+
 def test_simple():
     enc = tiktoken.get_encoding("gpt2")
     assert enc.encode("hello world") == [31373, 995]