feat: Make tokenizer add_special_tokens option configurable

In particular so that it can be disabled for chat/instruct models where an explicit template is used that already includes these tokens.

(for example the leading <s> token added by llama and mixtral tokenizers)

Signed-off-by: Nick Hill <[email protected]>

Author: njhill

integration_tests/test_cases_tinyllama.yaml

@@ -28,14 +28,66 @@
       method: GREEDY
       stopping: {"maxNewTokens": 17}
     requests:
-      - {"text":  "Once upon a time,"}
+      - {"text": "Once upon a time,"}
   response:
     responses:
     - generatedTokenCount: 17
       inputTokenCount: 6
       stopReason: MAX_TOKENS
       text: ' there was a little girl named Lily. She loved to play with her toy car.'
 
+
+# Input tokens
+- name: Return input tokens
+  request:
+    params:
+      method: GREEDY
+      stopping: {"maxNewTokens": 8}
+      response:
+        inputTokens: true
+    requests:
+      - {"text": "Once upon a time,"}
+  response:
+    responses:
+      - generatedTokenCount: 8
+        inputTokenCount: 6
+        inputTokens:
+          - text: <s>
+            #TODO this is wrong token: https://github.com/huggingface/transformers/issues/28622
+          - text: "\u2581Once"
+          - text: "\u2581upon"
+          - text: "\u2581a"
+          - text: "\u2581time"
+          - text: ','
+        stopReason: MAX_TOKENS
+        text: ' there was a little girl named Lily.'
+
+
+# Tokenize with tokens
+- name: Tokenize with tokens
+  request_type: tokenize
+  request:
+    return_tokens: true
+    requests:
+      - {"text": "The very long story is written by a very long story"}
+  response:
+    responses:
+      - tokenCount: 12
+        tokens:
+          - <s>
+          - "\u2581The"
+          - "\u2581very"
+          - "\u2581long"
+          - "\u2581story"
+          - "\u2581is"
+          - "\u2581written"
+          - "\u2581by"
+          - "\u2581a"
+          - "\u2581very"
+          - "\u2581long"
+          - "\u2581story"
+
+
 - name: Long input with tokens truncated
   request:
     params: