update

Author: dhuangnm

README.md

@@ -2,4 +2,9 @@
 
 This repository contains configurations for model validation.
 
-The `accuracy` folder contains YAML files for each model that configures information needed for the model to be validated through the [llm-eval-test](https://github.com/openshift-psap/llm-eval-test)
+The `accuracy` folder contains YAML files for each model that configures information needed for the model to be validated through the [llm-eval-test](https://github.com/openshift-psap/llm-eval-test). There are 4 config files for each model:
+
+* server.yml: contains settings to start a vllm server with the model
+* client.yml: contains settings for the llm-eval-test harness for the model
+* accuracy.yml: contains evaluation tasks and accuracy expectations for the model
+* storage.yml: specifies where mode and dataset is located

accuracy/Meta-Llama-3.1-8B-Instruct/accuracy.yml

@@ -1,11 +1,4 @@
 # accuracy configs for https://huggingface.co/meta-llama/Meta-Llama-3.1-8B-Instruct
-model: "vllm"
-model_args:
-  pretrained: "meta-llama/Meta-Llama-3.1-8B-Instruct"
-num_fewshot:
-apply_chat_template: true
-fewshot_as_multiturn: true
-add_bos_token: false
 tasks:
 - name: "leaderboard_bbh"
   metrics:

accuracy/Meta-Llama-3.1-8B-Instruct/client.yml

@@ -1,3 +1,8 @@
-# storage configs for https://huggingface.co/meta-llama/Meta-Llama-3.1-8B-Instruct
-model: hf
-data: hf
+# llm-eval-test configs for https://huggingface.co/meta-llama/Meta-Llama-3.1-8B-Instruct
+model: "vllm"
+model_args:
+  pretrained: "meta-llama/Meta-Llama-3.1-8B-Instruct"
+num_fewshot:
+apply_chat_template: true
+fewshot_as_multiturn: true
+add_bos_token: false

accuracy/Meta-Llama-3.1-8B-Instruct/storage.yml

@@ -0,0 +1,3 @@
+# storage configs for https://huggingface.co/meta-llama/Meta-Llama-3.1-8B-Instruct
+model: hf
+data: hf