We read every piece of feedback, and take your input very seriously.
To see all available qualifiers, see our documentation.
There was an error while loading. Please reload this page.
1 parent 0e6fcb9 commit 8a722a2Copy full SHA for 8a722a2
mtokenizer/index.html
@@ -212,9 +212,6 @@ <h1 style="color: orange">Tokenize!</h1>
212
}
213
214
input.addEventListener('input', async () => {
215
- errorMessage.textContent = 'Tokenizing...';
216
- errorMessage.style.color = 'yellow';
217
-
218
if (input.textContent === ""){
219
tokensDisplay.innerHTML = "";
220
return;
@@ -224,6 +221,9 @@ <h1 style="color: orange">Tokenize!</h1>
224
221
225
222
if (!tokenizer) return;
226
223
+ errorMessage.textContent = 'Tokenizing...';
+ errorMessage.style.color = 'yellow';
+
227
console.log(text)
228
const tokens = tokenizer.tokenize("<s>"+text).slice(1);
229
console.log(tokens)
0 commit comments