Add special tokens in text-generation pipeline if tokenizer requires

Author: xenova

src/pipelines.js

@@ -996,6 +996,11 @@ export class TextGenerationPipeline extends (/** @type {new (options: TextPipeli
         let isBatched = false;
         let isChatInput = false;
 
+        // By default, do not add special tokens, unless the tokenizer specifies otherwise
+        let add_special_tokens = generate_kwargs.add_special_tokens
+            ?? (this.tokenizer.add_bos_token || this.tokenizer.add_eos_token)
+            ?? false;
+
         // Normalize inputs
         /** @type {string[]} */
         let inputs;
@@ -1021,11 +1026,9 @@ export class TextGenerationPipeline extends (/** @type {new (options: TextPipeli
                     add_generation_prompt: true,
                 })
             ));
+            add_special_tokens = false; // Chat template handles this already
         }
 
-        // By default, do not add special tokens
-        const add_special_tokens = generate_kwargs.add_special_tokens ?? false;
-
         // By default, return full text
         const return_full_text = isChatInput
             ? false

src/tokenizers.js

@@ -2659,6 +2659,9 @@ export class PreTrainedTokenizer extends Callable {
             this.padding_side = tokenizerConfig.padding_side;
         }
 
+        this.add_bos_token = tokenizerConfig.add_bos_token;
+        this.add_eos_token = tokenizerConfig.add_eos_token;
+
         this.legacy = false;
 
         this.chat_template = tokenizerConfig.chat_template ?? null;

tests/pipelines/test_pipelines_text_generation.js

@@ -20,7 +20,7 @@ export default () => {
 
     describe("batch_size=1", () => {
       const text_input = "hello";
-      const generated_text_target = "erdingsAndroid Load";
+      const generated_text_target = "erdingsdelete mely";
       const text_target = [{ generated_text: text_input + generated_text_target }];
       const new_text_target = [{ generated_text: generated_text_target }];