add mistralai/Mistral-Small-24B-Instruct-2501 model accuracy (#9)

Author: derekk-nm

mistralai/Mistral-Small-24B-Instruct-2501/accuracy/client.yml

@@ -0,0 +1,8 @@
+# llm-eval-test configs for # storage configs for https://huggingface.co/mistralai/Mistral-Small-24B-Instruct-2501
+model: "vllm"
+model_args:
+  pretrained: "mistralai/Mistral-Small-24B-Instruct-2501"
+num_fewshot:
+apply_chat_template: true
+fewshot_as_multiturn: true
+add_bos_token: false

mistralai/Mistral-Small-24B-Instruct-2501/accuracy/server.yml

@@ -0,0 +1,5 @@
+# server configs for https://huggingface.co/mistralai/Mistral-Small-24B-Instruct-2501
+model: "mistralai/Mistral-Small-24B-Instruct-2501"
+trust-remote-code: true
+enable-chunked-prefill: true
+max-model-len: 4096

mistralai/Mistral-Small-24B-Instruct-2501/accuracy/tasks.yml

@@ -0,0 +1,197 @@
+# collected vllm v0.8.3.post1 on k8s-a100-duo
+tasks:
+  - name: leaderboard_math_algebra_hard
+    metrics:
+      - name: exact_match,none
+        value: 0.703
+
+  - name: leaderboard_math_counting_and_prob_hard
+    metrics:
+      - name: exact_match,none
+        value: 0.489
+
+  - name: leaderboard_math_geometry_hard
+    metrics:
+      - name: exact_match,none
+        value: 0.366
+
+  - name: leaderboard_math_intermediate_algebra_hard
+    metrics:
+      - name: exact_match,none
+        value: 0.283
+
+  - name: leaderboard_math_num_theory_hard
+    metrics:
+      - name: exact_match,none
+        value: 0.476
+
+  - name: leaderboard_math_prealgebra_hard
+    metrics:
+      - name: exact_match,none
+        value: 0.695
+
+  - name: leaderboard_math_precalculus_hard
+    metrics:
+      - name: exact_match,none
+        value: 0.355
+
+  - name: leaderboard_bbh_boolean_expressions
+    metrics:
+      - name: acc_norm,none
+        value: 0.876
+
+  - name: leaderboard_bbh_causal_judgement
+    metrics:
+      - name: acc_norm,none
+        value: 0.652
+
+  - name: leaderboard_bbh_date_understanding
+    metrics:
+      - name: acc_norm,none
+        value: 0.796
+
+  - name: leaderboard_bbh_disambiguation_qa
+    metrics:
+      - name: acc_norm,none
+        value: 0.696
+
+  - name: leaderboard_bbh_formal_fallacies
+    metrics:
+      - name: acc_norm,none
+        value: 0.684
+
+  - name: leaderboard_bbh_geometric_shapes
+    metrics:
+      - name: acc_norm,none
+        value: 0.508
+
+  - name: leaderboard_bbh_hyperbaton
+    metrics:
+      - name: acc_norm,none
+        value: 0.78
+
+  - name: leaderboard_bbh_logical_deduction_five_objects
+    metrics:
+      - name: acc_norm,none
+        value: 0.632
+
+  - name: leaderboard_bbh_logical_deduction_seven_objects
+    metrics:
+      - name: acc_norm,none
+        value: 0.636
+
+  - name: leaderboard_bbh_logical_deduction_three_objects
+    metrics:
+      - name: acc_norm,none
+        value: 0.876
+
+  - name: leaderboard_bbh_movie_recommendation
+    metrics:
+      - name: acc_norm,none
+        value: 0.848
+
+  - name: leaderboard_bbh_navigate
+    metrics:
+      - name: acc_norm,none
+        value: 0.688
+
+  - name: leaderboard_bbh_object_counting
+    metrics:
+      - name: acc_norm,none
+        value: 0.42
+
+  - name: leaderboard_bbh_penguins_in_a_table
+    metrics:
+      - name: acc_norm,none
+        value: 0.767
+
+  - name: leaderboard_bbh_reasoning_about_colored_objects
+    metrics:
+      - name: acc_norm,none
+        value: 0.764
+
+  - name: leaderboard_bbh_ruin_names
+    metrics:
+      - name: acc_norm,none
+        value: 0.868
+
+  - name: leaderboard_bbh_salient_translation_error_detection
+    metrics:
+      - name: acc_norm,none
+        value: 0.684
+
+  - name: leaderboard_bbh_snarks
+    metrics:
+      - name: acc_norm,none
+        value: 0.725
+
+  - name: leaderboard_bbh_sports_understanding
+    metrics:
+      - name: acc_norm,none
+        value: 0.836
+
+  - name: leaderboard_bbh_temporal_sequences
+    metrics:
+      - name: acc_norm,none
+        value: 0.984
+
+  - name: leaderboard_bbh_tracking_shuffled_objects_five_objects
+    metrics:
+      - name: acc_norm,none
+        value: 0.288
+
+  - name: leaderboard_bbh_tracking_shuffled_objects_seven_objects
+    metrics:
+      - name: acc_norm,none
+        value: 0.224
+
+  - name: leaderboard_bbh_tracking_shuffled_objects_three_objects
+    metrics:
+      - name: acc_norm,none
+        value: 0.348
+
+  - name: leaderboard_bbh_web_of_lies
+    metrics:
+      - name: acc_norm,none
+        value: 0.52
+
+  - name: leaderboard_gpqa_diamond
+    metrics:
+      - name: acc_norm,none
+        value: 0.399
+
+  - name: leaderboard_gpqa_extended
+    metrics:
+      - name: acc_norm,none
+        value: 0.405
+
+  - name: leaderboard_gpqa_main
+    metrics:
+      - name: acc_norm,none
+        value: 0.393
+
+  - name: leaderboard_musr_murder_mysteries
+    metrics:
+      - name: acc_norm,none
+        value: 0.556
+
+  - name: leaderboard_musr_object_placements
+    metrics:
+      - name: acc_norm,none
+        value: 0.437
+
+  - name: leaderboard_musr_team_allocation
+    metrics:
+      - name: acc_norm,none
+        value: 0.404
+
+  - name: leaderboard_ifeval
+    metrics:
+      - name: prompt_level_strict_acc,none
+        value: 0.582
+      - name: prompt_level_loose_acc,none
+        value: 0.647
+      - name: inst_level_loose_acc,none
+        value: 0.748
+      - name: inst_level_strict_acc,none
+        value: 0.693

mistralai/Mistral-Small-24B-Instruct-2501/storage.yml

@@ -0,0 +1,3 @@
+# storage configs for https://huggingface.co/mistralai/Mistral-Small-24B-Instruct-2501
+model: hf
+data: hf