perf: Add custom token_ids method (#2264)

Author: shaohuzhang1

apps/setting/models_provider/impl/base_chat_open_ai.py

@@ -15,8 +15,14 @@
 from common.config.tokenizer_manage_config import TokenizerManage
 
 
+def custom_get_token_ids(text: str):
+    tokenizer = TokenizerManage.get_tokenizer()
+    return tokenizer.encode(text)
+
+
 class BaseChatOpenAI(ChatOpenAI):
     usage_metadata: dict = {}
+    custom_get_token_ids = custom_get_token_ids
 
     def get_last_generation_info(self) -> Optional[Dict[str, Any]]:
         return self.usage_metadata