Merge pull request #1 from neuralmagic/begin

Create repo and add initial files

Author: dhuangnm

README.md

@@ -1 +1,10 @@
-# model-validation-configs
+# model-validation-configs
+
+This repository contains configurations for model validation.
+
+The `accuracy` folder contains YAML files for each model that configures information needed for the model to be validated through the [llm-eval-test](https://github.com/openshift-psap/llm-eval-test). There are 4 config files for each model:
+
+* server.yml: contains settings to start a vllm server with the model
+* client.yml: contains settings for the llm-eval-test harness for the model
+* accuracy.yml: contains evaluation tasks and accuracy expectations for the model
+* storage.yml: specifies where mode and dataset is located

accuracy/Meta-Llama-3.1-8B-Instruct/accuracy.yml

@@ -0,0 +1,28 @@
+# accuracy configs for https://huggingface.co/meta-llama/Meta-Llama-3.1-8B-Instruct
+tasks:
+- name: "leaderboard_bbh"
+  metrics:
+  - name: "acc_norm,none"
+    value: 0.50946
+- name: "leaderboard_gpqa"
+  metrics:
+  - name: "acc_norm,none"
+    value: 0.29698
+- name: "leaderboard_ifeval"
+  metrics:
+  - name: "inst_level_loose_acc,none"
+    value: 0.85851
+  - name: "inst_level_strict_acc,none"
+    value: 0.82374
+  - name: "prompt_level_loose_acc,none"
+    value: 0.79667
+  - name: "prompt_level_strict_acc,none"
+    value: 0.74861
+- name: "leaderboard_math_hard"
+  metrics:
+  - name: "exact_match,none"
+    value: 0.19864
+- name: "leaderboard_musr"
+  metrics:
+  - name: "acc_norm,none"
+    value: 0.38359

accuracy/Meta-Llama-3.1-8B-Instruct/client.yml

@@ -0,0 +1,8 @@
+# llm-eval-test configs for https://huggingface.co/meta-llama/Meta-Llama-3.1-8B-Instruct
+model: "vllm"
+model_args:
+  pretrained: "meta-llama/Meta-Llama-3.1-8B-Instruct"
+num_fewshot:
+apply_chat_template: true
+fewshot_as_multiturn: true
+add_bos_token: false

accuracy/Meta-Llama-3.1-8B-Instruct/server.yml

@@ -0,0 +1,6 @@
+# server configs for https://huggingface.co/meta-llama/Meta-Llama-3.1-8B-Instruct
+model: "meta-llama/Meta-Llama-3.1-8B-Instruct"
+trust_remote_code: true
+enable_chunked_prefill: true
+tensor_parallel_size: 
+max_model_len: 4096

accuracy/Meta-Llama-3.1-8B-Instruct/storage.yml

@@ -0,0 +1,3 @@
+# storage configs for https://huggingface.co/meta-llama/Meta-Llama-3.1-8B-Instruct
+model: hf
+data: hf