✨ Add CLI to convert tokenizer

This PR adds a new convert-to-fast-tokenizer command to the cli to add tokenizer.json files to models.
This is also invoked on download-weights if the auto_convert flag is true.

Signed-off-by: Joe Runde <[email protected]>

Author: joerunde

server/text_generation_server/cli.py

@@ -93,7 +93,8 @@ def download_weights(
             convert_to_safetensors(model_name, revision)
         elif not any(f.endswith(".safetensors") for f in files):
             print(".safetensors weights not found on hub, but were found locally. Remove them first to re-convert")
-
+    if auto_convert:
+        convert_to_fast_tokenizer(model_name, revision)
 
 @app.command()
 def convert_to_onnx(
@@ -193,5 +194,34 @@ def quantize(
     )
 
 
+@app.command()
+def convert_to_fast_tokenizer(
+    model_name: str,
+    revision: Optional[str] = None,
+    output_path: Optional[str] = None,
+):
+    from text_generation_server import utils
+
+    # Check for existing "tokenizer.json"
+    model_path = utils.get_model_path(model_name, revision)
+
+    if os.path.exists(os.path.join(model_path, "tokenizer.json")):
+        print(f"Model {model_name} already has a fast tokenizer")
+        return
+
+    if output_path is not None:
+        if not os.path.isdir(output_path):
+            print(f"Output path {output_path} must exist and be a directory")
+            return
+    else:
+        output_path = model_path
+
+    import transformers
+    tokenizer = transformers.AutoTokenizer.from_pretrained(model_name, revision=revision)
+    tokenizer.save_pretrained(output_path)
+
+    print(f"Saved tokenizer to {output_path}")
+
+
 if __name__ == "__main__":
     app()