update

DrownFish19 · DrownFish19 · commit 21a46dcad80d · 2025-01-08T07:49:36.000Z
diff --git a/requirements-dev.txt b/requirements-dev.txt
@@ -29,4 +29,4 @@ data
 wget
 huggingface_hub>=0.19.2
 tiktoken
-tokenizers<0.21,<0.22
+tokenizers<0.21
diff --git a/requirements.txt b/requirements.txt
@@ -26,5 +26,5 @@ jinja2
 regex
 numpy<=1.26.4
 tiktoken
-tokenizers<0.21,<0.22
+tokenizers<0.21
 omegaconf
