fix requirments.txt for py38

Author: DrownFish19

requirements-dev.txt

@@ -29,4 +29,4 @@ data
 wget
 huggingface_hub>=0.19.2
 tiktoken
-tokenizers>=0.21,<0.22
+tokenizers<0.21,<0.22

requirements.txt

@@ -26,4 +26,4 @@ jinja2
 regex
 numpy<=1.26.4
 tiktoken
-tokenizers>=0.21,<0.22
+tokenizers<0.21

scripts/unit_test/ci_unit.sh

@@ -28,6 +28,7 @@ install_requirements() {
     python -m pip config --user set global.trusted-host pip.baidu-int.com
     python -m pip install -r requirements.txt
     python -m pip install -r requirements-dev.txt
+    python -m pip install tokenizers>=0.21,<0.22 # update tokenizers for transformers
     python -m pip install -r tests/requirements.txt
     python -m pip install -r paddlenlp/experimental/autonlp/requirements.txt 
     python -m pip uninstall paddlepaddle paddlepaddle_gpu -y