refactor: add cl100k_base.tiktoken tokenizer.

Author: shaohuzhang1

apps/maxkb/wsgi/web.py

@@ -32,7 +32,7 @@ def __call__(self, name, *args, **kwargs):
 builtins.__import__ = TorchBlocker()
 
 os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'maxkb.settings')
-
+os.environ['TIKTOKEN_CACHE_DIR'] = '/opt/maxkb-app/model/tokenizer/openai-tiktoken-cl100k-base'
 application = get_wsgi_application()