add tiktoken package for qwen infer (#4990)(#4991)

ZhaoqiongZ · web-flow · commit 6ff711cf2513 · 2024-11-06T16:21:47.000+08:00
diff --git a/examples/gpu/llm/inference/requirements.txt b/examples/gpu/llm/inference/requirements.txt
@@ -1,2 +1,3 @@
 numpy
 deepspeed==0.14.4
+tiktoken==0.8.0
