feat: Specify the directory for calculating tokens models

Author: shaohuzhang1

apps/common/config/tokenizer_manage_config.py

@@ -6,7 +6,6 @@
     @date：2024/4/28 10:17
     @desc:
 """
-import os
 from pathlib import Path
 
 BASE_DIR = Path(__file__).resolve().parent.parent.parent
@@ -27,6 +26,6 @@ class TokenizerManage:
     def get_tokenizer():
         from tokenizers import Tokenizer
         # 创建Tokenizer
-        s = os.path.join(BASE_DIR.parent, 'tokenizer', 'bert-base-cased', 'tokenizer.json')
-        TokenizerManage.tokenizer = Tokenizer.from_file(s)
+        TokenizerManage.tokenizer = Tokenizer.from_file(
+            '/opt/maxkb-app/model/tokenizer/models--bert-base-cased/snapshots/cd5ef92a9fb2f889e972770a36d4ed042daf221e')
         return MKTokenizer(TokenizerManage.tokenizer)

apps/maxkb/wsgi/web.py

@@ -32,7 +32,7 @@ def __call__(self, name, *args, **kwargs):
 builtins.__import__ = TorchBlocker()
 
 os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'maxkb.settings')
-
+os.environ['TIKTOKEN_CACHE_DIR'] = '/opt/maxkb-app/model/tokenizer/openai-tiktoken-cl100k-base'
 application = get_wsgi_application()