Rename tokenizer -> tokenizer_config in tasks and widgets (#494)

cc @coyotte508 @Wauplin

Author: xenova

packages/tasks/src/model-data.ts

@@ -54,7 +54,7 @@ export interface ModelData {
 			base_model_name?: string;
 			task_type?: string;
 		};
-		tokenizer?: TokenizerConfig;
+		tokenizer_config?: TokenizerConfig;
 	};
 	/**
 	 * all the model tags

packages/widgets/src/lib/components/InferenceWidget/widgets/ConversationalWidget/ConversationalWidget.svelte

@@ -49,11 +49,11 @@
 			return;
 		}
 
-		if (config.tokenizer === undefined) {
+		if (config.tokenizer_config === undefined) {
 			error = "Tokenizer config not found";
 			return;
 		}
-		tokenizerConfig = config.tokenizer;
+		tokenizerConfig = config.tokenizer_config;
 
 		const chatTemplate = tokenizerConfig.chat_template;
 		if (chatTemplate === undefined) {

packages/widgets/src/routes/+page.svelte

@@ -36,7 +36,7 @@
 			tags: ["conversational"],
 			inference: InferenceDisplayability.Yes,
 			config: {
-				tokenizer: {
+				tokenizer_config: {
 					bos_token: "<s>",
 					chat_template:
 						"{% for message in messages %}\n{% if message['role'] == 'user' %}\n{{ '<|user|>\n' + message['content'] + eos_token }}\n{% elif message['role'] == 'system' %}\n{{ '<|system|>\n' + message['content'] + eos_token }}\n{% elif message['role'] == 'assistant' %}\n{{ '<|assistant|>\n'  + message['content'] + eos_token }}\n{% endif %}\n{% if loop.last and add_generation_prompt %}\n{{ '<|assistant|>' }}\n{% endif %}\n{% endfor %}",