|
| 1 | +""" |
| 2 | +Test that the AutoGuess feature picks the correct model for every template. |
| 3 | +Also checks that every template is being tested so that when new AutoGuess additions are made, this test fails unless an accompanying test is included. |
| 4 | +""" |
| 5 | +import os |
| 6 | +import sys |
| 7 | +import requests |
| 8 | +import json |
| 9 | + |
| 10 | + |
| 11 | +# Map an AutoGuess name to a HuggingFace model ID |
| 12 | +# THIS LIST MUST BE UPDATED WHEN A NEW MODEL IS ADDED |
| 13 | +AUTOGUESS_MAPPING = { |
| 14 | + "ChatML (Phi 4)": "microsoft/phi-4", |
| 15 | + "ChatML (Qwen 2.5 based)": "Qwen/Qwen2.5-0.5B-Instruct", |
| 16 | + "ChatML (Kimi)": "moonshotai/Kimi-K2-Instruct", |
| 17 | + "Google Gemma 2": "Efficient-Large-Model/gemma-2-2b-it", |
| 18 | + "Google Gemma 3": "scb10x/typhoon2.1-gemma3-12b", |
| 19 | + "Google Gemma 3n": "lmstudio-community/gemma-3n-E4B-it-MLX-bf16", |
| 20 | + "Llama 3.x": "Steelskull/L3.3-Shakudo-70b", |
| 21 | + "Llama 4": "meta-llama/Llama-4-Scout-17B-16E-Instruct", |
| 22 | + "Mistral V7 (with system prompt)": "Doctor-Shotgun/MS3.2-24B-Magnum-Diamond", |
| 23 | + "Mistral V3": "mistralai/Mistral-7B-Instruct-v0.3", |
| 24 | + "GLM-4": "THUDM/glm-4-9b-chat-hf", |
| 25 | + "Phi 3.5": "microsoft/Phi-3.5-mini-instruct", |
| 26 | + "Phi 4 (mini)": "microsoft/Phi-4-mini-instruct", |
| 27 | + "Cohere (Aya Expanse 32B based)": "CohereLabs/aya-expanse-32b", |
| 28 | + "DeepSeek V2.5": "deepseek-ai/DeepSeek-V2.5", |
| 29 | + "Jamba": "ai21labs/Jamba-tiny-dev", |
| 30 | + "Dots": "rednote-hilab/dots.llm1.inst", |
| 31 | + "RWKV World": "fla-hub/rwkv7-1.5B-world", |
| 32 | + "Mistral (Generic)": "mistralai/Mistral-Nemo-Instruct-2407", |
| 33 | + "ChatML (Generic)": "NewEden/Gemma-27B-chatml", |
| 34 | +} |
| 35 | + |
| 36 | +# User may be running this test from ./ or from ../ -- we want to be in ./ (i.e. tests) |
| 37 | +if os.path.exists("tests"): |
| 38 | + os.chdir("tests") |
| 39 | + |
| 40 | +with open("../kcpp_adapters/AutoGuess.json") as f: |
| 41 | + autoguess = json.load(f) |
| 42 | + |
| 43 | +def get_tokenizer_config_for_huggingface_model_id(huggingface_model_id: str): |
| 44 | + fname = f"gated-tokenizers/tokenizer_configs/{huggingface_model_id.replace('/','_')}.json" |
| 45 | + if os.path.exists(fname): |
| 46 | + with open(fname) as f: |
| 47 | + return json.load(f) |
| 48 | + |
| 49 | + for filename in ["tokenizer_config.json", "chat_template.json"]: |
| 50 | + url = f"https://huggingface.co/{huggingface_model_id}/resolve/main/{filename}" |
| 51 | + response = requests.get(url) |
| 52 | + if response.status_code == 200: |
| 53 | + v = json.loads(response.text) |
| 54 | + if 'chat_template' in v: |
| 55 | + return v |
| 56 | + raise ValueError(f"Failed to fetch tokenizer config for {huggingface_model_id}.") |
| 57 | + |
| 58 | +def match_chat_template_to_adapter(chat_template: str|list) -> tuple[str, str|None]|None: |
| 59 | + # Additional code in tester not present in application: support for multiple chat templates, and use default if present |
| 60 | + sub_template: str|None = None |
| 61 | + if isinstance(chat_template, list): |
| 62 | + found = False |
| 63 | + for template in chat_template: |
| 64 | + # {"name": .., "template": ...} |
| 65 | + if template['name'] == "default": |
| 66 | + sub_template = "default" |
| 67 | + chat_template = template['template'] |
| 68 | + found = True |
| 69 | + break |
| 70 | + if not found: |
| 71 | + # We pick the first template if no default is present |
| 72 | + sub_template = chat_template[0]['name'] |
| 73 | + chat_template = chat_template[0]['template'] |
| 74 | + if chat_template != "": |
| 75 | + for entry in autoguess: |
| 76 | + if all(s in chat_template for s in entry['search']): |
| 77 | + return entry['name'], sub_template |
| 78 | + |
| 79 | +failures = 0 |
| 80 | +seen = set() |
| 81 | +namefmt = "{name:<" + str(max(len(name) for name in AUTOGUESS_MAPPING.keys())) + "}" |
| 82 | +hmifmt = "{huggingface_model_id:<" + str(max(len(huggingface_model_id) for huggingface_model_id in AUTOGUESS_MAPPING.values())) + "}" |
| 83 | +for name, huggingface_model_id in AUTOGUESS_MAPPING.items(): |
| 84 | + seen.add(name) |
| 85 | + if huggingface_model_id == "***UNKNOWN***": |
| 86 | + print(namefmt.format(name=name) + " = " + namefmt.format(name="***UNKNOWN***") + " : PENDING") |
| 87 | + continue |
| 88 | + tokenizer_config = get_tokenizer_config_for_huggingface_model_id(huggingface_model_id) |
| 89 | + assert 'chat_template' in tokenizer_config |
| 90 | + matched = match_chat_template_to_adapter(tokenizer_config['chat_template']) |
| 91 | + if matched is None: |
| 92 | + matched, sub_template = "MISSING MAPPING", None |
| 93 | + else: |
| 94 | + matched, sub_template = matched |
| 95 | + sub_template = f"[{sub_template}]" if sub_template else "" |
| 96 | + print(namefmt.format(name=name) + " = " + namefmt.format(name=matched) + " : " + ("OK " if name == matched else "FAILURE") + " " + hmifmt.format(huggingface_model_id=huggingface_model_id) + " " + sub_template) |
| 97 | + failures += name != matched |
| 98 | + |
| 99 | +for entry in autoguess: |
| 100 | + if entry['name'] not in seen: |
| 101 | + print(namefmt.format(name=entry['name']) + " MISSING MAPPING") |
| 102 | + failures += 1 |
| 103 | + |
| 104 | +if failures > 0: |
| 105 | + print(f"There were {failures} failure(s)!") |
| 106 | + sys.exit(1) |
0 commit comments