fix(openai): switched to pure js tiktoken (#248)

Author: nirga

package-lock.json

@@ -40,7 +40,7 @@
     "@opentelemetry/instrumentation": "^0.49.0",
     "@opentelemetry/semantic-conventions": "^1.22.0",
     "@traceloop/ai-semantic-conventions": "^0.8.0",
-    "tiktoken": "^1.0.13"
+    "js-tiktoken": "^1.0.11"
   },
   "devDependencies": {
     "@pollyjs/adapter-node-http": "^6.0.6",

packages/instrumentation-openai/package.json

@@ -45,7 +45,7 @@ import type {
 } from "openai/resources";
 import type { Stream } from "openai/streaming";
 import { version } from "../package.json";
-import { encoding_for_model, TiktokenModel, Tiktoken } from "tiktoken";
+import { encodingForModel, TiktokenModel, Tiktoken } from "js-tiktoken";
 
 export class OpenAIInstrumentation extends InstrumentationBase<any> {
   protected declare _config: OpenAIInstrumentationConfig;
@@ -726,7 +726,7 @@ export class OpenAIInstrumentation extends InstrumentationBase<any> {
 
     if (!encoding) {
       try {
-        encoding = encoding_for_model(model as TiktokenModel);
+        encoding = encodingForModel(model as TiktokenModel);
         this._encodingCache.set(model, encoding);
       } catch (e) {
         this._diag.warn(e);