Update prompt_token_optimizer.html

TobiasMue91 · web-flow · commit a5d11bcab36e · 2025-01-23T11:22:31.000+01:00
diff --git a/tools/prompt_token_optimizer.html b/tools/prompt_token_optimizer.html
@@ -6,7 +6,7 @@
         <title>Prompt Token Optimizer</title>
         <script src="https://cdn.tailwindcss.com"></script>
         <script src="https://cdn.jsdelivr.net/npm/clipboard@2.0.8/dist/clipboard.min.js"></script>
-        <script src="https://cdn.jsdelivr.net/npm/gpt-tokenizer@2.1.1/dist/gpt-tokenizer.min.js"></script>
+        <script src="https://cdn.jsdelivr.net/npm/@dqbd/tiktoken@1.0.7/dist/index.global.js"></script>
     </head>
     <body class="bg-gray-100 dark:bg-gray-900 transition-colors duration-200">
         <div class="container mx-auto px-4 py-8 max-w-4xl">
@@ -108,9 +108,9 @@ <h1 class="text-3xl font-bold text-gray-800 dark:text-white mb-6">Prompt Token O
                 therefore: "thus",
                 versus: "vs",
             };
-            const tokenizer = new GPTTokenizer({ type: "gpt3" });
+            const encoder = new tiktoken.Tokenizer(tiktoken.load());
             function countTokens(text) {
-                return tokenizer.encode(text).length;
+                return encoder.encode(text).length;
             }
             function optimizePrompt(text) {
                 let optimized = text.replace(/\s+/g, " ").trim();
