Sample to sample compare for integration tests (#977)

NathanHB · web-flow · commit 35ae2d158f72 · 2025-09-22T10:49:51.000+02:00
* Track parquet files with Git LFS

* init

* remove logging in slow tests and log nvidia smi
diff --git a/.gitattributes b/.gitattributes
@@ -1,2 +1,3 @@
 *.json filter=lfs diff=lfs merge=lfs -text
 tests/unit/metrics/test_cases/*.json -filter -diff -merge text
+*.parquet filter=lfs diff=lfs merge=lfs -text
diff --git a/.github/workflows/slow_tests.yaml b/.github/workflows/slow_tests.yaml
@@ -38,5 +38,8 @@ jobs:
         run: uv sync --extra dev
 
 
+      - name: run nvidia-smi
+        run: nvidia-smi
+
       - name: Run tests
-        run: uv run pytest --disable-pytest-warnings -o log_cli=true -o log_cli_level=INFO --runslow tests/slow_tests/
+        run: uv run pytest --disable-pytest-warnings --runslow tests/slow_tests/
diff --git a/examples/model_configs/vllm_model_config.yaml b/examples/model_configs/vllm_model_config.yaml
@@ -21,13 +21,11 @@ model_parameters:
     presence_penalty: 0.0
     repetition_penalty: 1.0
     frequency_penalty: 0.0
-    temperature: 0.1
+    temperature: 0.0
     top_k: null
     min_p: 0.0
     top_p: 0.9
     seed: 42
     stop_tokens: null
     max_new_tokens: 2048
     min_new_tokens: 0
-metrics_options:
-  yo: null
diff --git a/src/lighteval/main_accelerate.py b/src/lighteval/main_accelerate.py
@@ -146,7 +146,8 @@ def accelerate(  # noqa C901
     pipeline.show_results()
 
     results = pipeline.get_results()
+    details = pipeline.get_details()
 
     pipeline.save_and_push_results()
 
-    return results
+    return results, details
diff --git a/src/lighteval/main_vllm.py b/src/lighteval/main_vllm.py
@@ -131,7 +131,8 @@ def vllm(
     pipeline.show_results()
 
     results = pipeline.get_results()
+    details = pipeline.get_details()
 
     pipeline.save_and_push_results()
 
-    return results
+    return results, details
diff --git a/src/lighteval/pipeline.py b/src/lighteval/pipeline.py
@@ -437,3 +437,6 @@ def show_results(self):
     def get_results(self):
         self._init_final_dict()
         return self.final_dict
+
+    def get_details(self):
+        return self.evaluation_tracker.details_logger.details
diff --git a/tests/reference_details/Qwen2.5-VL-3B-Instruct-vlm/details_lighteval|mmmu_pro:standard-10|0_2025-09-19T14-25-10.670316.parquet b/tests/reference_details/Qwen2.5-VL-3B-Instruct-vlm/details_lighteval|mmmu_pro:standard-10|0_2025-09-19T14-25-10.670316.parquet
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:228e58884c153beb351f47a3d0c5704d9e764844aa6990375855ebdddfe234cb
+size 11827875
diff --git a/tests/reference_details/Qwen2.5-VL-3B-Instruct-vlm/details_lighteval|mmmu_pro:standard-4|0_2025-09-19T14-25-10.670316.parquet b/tests/reference_details/Qwen2.5-VL-3B-Instruct-vlm/details_lighteval|mmmu_pro:standard-4|0_2025-09-19T14-25-10.670316.parquet
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1f470cfb11d5901a644ccf28b267829448c511929034ee520a4f870253e3be99
+size 11540572
diff --git a/tests/reference_details/Qwen2.5-VL-3B-Instruct-vlm/details_lighteval|mmmu_pro:vision|0_2025-09-19T14-25-10.670316.parquet b/tests/reference_details/Qwen2.5-VL-3B-Instruct-vlm/details_lighteval|mmmu_pro:vision|0_2025-09-19T14-25-10.670316.parquet
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c2168453c9482af24bcb501f7b1b5f4f83f261e416c40db8e23ec123a13ef27f
+size 27746331
diff --git a/tests/reference_details/SmolLM2-1.7B-Instruct-transformers/details_leaderboard|arc:challenge|25_2025-09-19T14-21-59.670987.parquet b/tests/reference_details/SmolLM2-1.7B-Instruct-transformers/details_leaderboard|arc:challenge|25_2025-09-19T14-21-59.670987.parquet
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bf8c0061d55f76265ec3d88762e4d806ce2d932909384cd03637413fd5cb89be
+size 88248
diff --git a/tests/reference_details/SmolLM2-1.7B-Instruct-transformers/details_leaderboard|hellaswag|10_2025-09-19T14-21-59.670987.parquet b/tests/reference_details/SmolLM2-1.7B-Instruct-transformers/details_leaderboard|hellaswag|10_2025-09-19T14-21-59.670987.parquet
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f95752c0d186667610852bcaf34cdaa0aa0e1114bff50dc8a10244c31610eb3d
+size 107042
diff --git a/tests/reference_details/SmolLM2-1.7B-Instruct-transformers/details_leaderboard|mmlu:college_chemistry|5_2025-09-19T14-21-59.670987.parquet b/tests/reference_details/SmolLM2-1.7B-Instruct-transformers/details_leaderboard|mmlu:college_chemistry|5_2025-09-19T14-21-59.670987.parquet
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b53028865bed1ef182d8f797f0bf2ad189c512e2bb3042a2469ec202068ae22b
+size 37410
diff --git a/tests/reference_details/SmolLM2-1.7B-Instruct-transformers/details_leaderboard|mmlu:us_foreign_policy|5_2025-09-19T14-21-59.670987.parquet b/tests/reference_details/SmolLM2-1.7B-Instruct-transformers/details_leaderboard|mmlu:us_foreign_policy|5_2025-09-19T14-21-59.670987.parquet
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6776849a962942fe0484967b11a0dc229bd89a132c4dfdda93923ff53bebaff5
+size 37924
diff --git a/tests/reference_details/SmolLM2-1.7B-Instruct-transformers/details_leaderboard|truthfulqa:mc|0_2025-09-19T14-21-59.670987.parquet b/tests/reference_details/SmolLM2-1.7B-Instruct-transformers/details_leaderboard|truthfulqa:mc|0_2025-09-19T14-21-59.670987.parquet
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bacac56e0ed5c6c65642e1c610cbcdf4ea20d8f41160942198f25ab6bf04c99a
+size 26097
diff --git a/tests/reference_details/SmolLM2-1.7B-Instruct-transformers/details_lighteval|agieval:aqua-rat|0_2025-09-19T14-21-59.670987.parquet b/tests/reference_details/SmolLM2-1.7B-Instruct-transformers/details_lighteval|agieval:aqua-rat|0_2025-09-19T14-21-59.670987.parquet
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:eec7ced557be46cd1f3a54f8f51109512af7c0912b735d46adc63e1bdf0db21f
+size 22385
diff --git a/tests/reference_details/SmolLM2-1.7B-Instruct-transformers/details_lighteval|agieval:logiqa-en|0_2025-09-19T14-21-59.670987.parquet b/tests/reference_details/SmolLM2-1.7B-Instruct-transformers/details_lighteval|agieval:logiqa-en|0_2025-09-19T14-21-59.670987.parquet
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e7ffedbe7a6ae8de9c9a58e35e7510a547eec23aac3dbd7d478ad78f7d21d280
+size 34634
diff --git a/tests/reference_details/SmolLM2-1.7B-Instruct-transformers/details_lighteval|agieval:lsat-ar|0_2025-09-19T14-21-59.670987.parquet b/tests/reference_details/SmolLM2-1.7B-Instruct-transformers/details_lighteval|agieval:lsat-ar|0_2025-09-19T14-21-59.670987.parquet
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f337a4e40a4bd6c4f2473f7bc70c1c604c068f582bebc1d6625c725cef74f2a3
+size 31273
diff --git a/tests/reference_details/SmolLM2-1.7B-Instruct-transformers/details_lighteval|agieval:lsat-lr|0_2025-09-19T14-21-59.670987.parquet b/tests/reference_details/SmolLM2-1.7B-Instruct-transformers/details_lighteval|agieval:lsat-lr|0_2025-09-19T14-21-59.670987.parquet
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7fe3574ba3da54027a4a76049a166c5791820d3cbaef079d58f63ec6be61ead4
+size 39946
diff --git a/tests/reference_details/SmolLM2-1.7B-Instruct-transformers/details_lighteval|agieval:lsat-rc|0_2025-09-19T14-21-59.670987.parquet b/tests/reference_details/SmolLM2-1.7B-Instruct-transformers/details_lighteval|agieval:lsat-rc|0_2025-09-19T14-21-59.670987.parquet
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:75ffe74a6739cc2b6972421475a334ffba1230edb4cce73fce3eb11a29972768
+size 74738
diff --git a/tests/reference_details/SmolLM2-1.7B-Instruct-transformers/details_lighteval|agieval:sat-en-without-passage|0_2025-09-19T14-21-59.670987.parquet b/tests/reference_details/SmolLM2-1.7B-Instruct-transformers/details_lighteval|agieval:sat-en-without-passage|0_2025-09-19T14-21-59.670987.parquet
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4e1810d70099d811628030d2573ea8f2c5df9483b633a30520eaaef1f081a5b8
+size 26472
diff --git a/tests/reference_details/SmolLM2-1.7B-Instruct-transformers/details_lighteval|agieval:sat-en|0_2025-09-19T14-21-59.670987.parquet b/tests/reference_details/SmolLM2-1.7B-Instruct-transformers/details_lighteval|agieval:sat-en|0_2025-09-19T14-21-59.670987.parquet
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6f66340d43bb29578493951d3bf01925b22331b1860c263d63d1b5933bf590c5
+size 73055
diff --git a/tests/reference_details/SmolLM2-1.7B-Instruct-transformers/details_lighteval|bigbench:causal_judgment|3_2025-09-19T14-21-59.670987.parquet b/tests/reference_details/SmolLM2-1.7B-Instruct-transformers/details_lighteval|bigbench:causal_judgment|3_2025-09-19T14-21-59.670987.parquet
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c05a40009c7ae83f39a5264a766b6a03e9766e86b9a812a7159ff19c0a64cd1b
+size 47558
diff --git a/tests/reference_details/SmolLM2-1.7B-Instruct-transformers/details_lighteval|bigbench:date_understanding|3_2025-09-19T14-21-59.670987.parquet b/tests/reference_details/SmolLM2-1.7B-Instruct-transformers/details_lighteval|bigbench:date_understanding|3_2025-09-19T14-21-59.670987.parquet
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:829ab8ae538885f1cf0a192cd8f366ee5e720cb7262b6a6d73189bff3aba4570
+size 30006
diff --git a/tests/reference_details/SmolLM2-1.7B-Instruct-transformers/details_lighteval|bigbench:disambiguation_qa|3_2025-09-19T14-21-59.670987.parquet b/tests/reference_details/SmolLM2-1.7B-Instruct-transformers/details_lighteval|bigbench:disambiguation_qa|3_2025-09-19T14-21-59.670987.parquet
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a415b704660f5280a8bac16e2fa294696f5f2a2258fda5513af4494e172c5b78
+size 30932
diff --git a/tests/reference_details/SmolLM2-1.7B-Instruct-transformers/details_lighteval|bigbench:geometric_shapes|3_2025-09-19T14-21-59.670987.parquet b/tests/reference_details/SmolLM2-1.7B-Instruct-transformers/details_lighteval|bigbench:geometric_shapes|3_2025-09-19T14-21-59.670987.parquet
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:363e6b765a94a330b0641f5423d850ed9e807f8e2d3c3ae48bdfe6e5796a4338
+size 32464
diff --git a/tests/reference_details/SmolLM2-1.7B-Instruct-transformers/details_lighteval|bigbench:logical_deduction_five_objects|3_2025-09-19T14-21-59.670987.parquet b/tests/reference_details/SmolLM2-1.7B-Instruct-transformers/details_lighteval|bigbench:logical_deduction_five_objects|3_2025-09-19T14-21-59.670987.parquet
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8a396dc44ab69bb5bcc617b3239f866f32adfd049da369dde0fbe013684dd2c2
+size 34653
diff --git a/tests/reference_details/SmolLM2-1.7B-Instruct-transformers/details_lighteval|bigbench:logical_deduction_seven_objects|3_2025-09-19T14-21-59.670987.parquet b/tests/reference_details/SmolLM2-1.7B-Instruct-transformers/details_lighteval|bigbench:logical_deduction_seven_objects|3_2025-09-19T14-21-59.670987.parquet
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ef4a546703a76733fac1be5c8c15f4fc700ceeaa510831185bc35af645167018
+size 38176
diff --git a/tests/reference_details/SmolLM2-1.7B-Instruct-transformers/details_lighteval|bigbench:movie_recommendation|3_2025-09-19T14-21-59.670987.parquet b/tests/reference_details/SmolLM2-1.7B-Instruct-transformers/details_lighteval|bigbench:movie_recommendation|3_2025-09-19T14-21-59.670987.parquet
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6a3b5caa35dc5efc889303e65dea70e8b3d88dd1de80eaf319191d8ef28c8a7b
+size 29221
diff --git a/tests/reference_details/SmolLM2-1.7B-Instruct-transformers/details_lighteval|bigbench:navigate|3_2025-09-19T14-21-59.670987.parquet b/tests/reference_details/SmolLM2-1.7B-Instruct-transformers/details_lighteval|bigbench:navigate|3_2025-09-19T14-21-59.670987.parquet
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1ba4c719cb007c60aff2dd472feb73450b81ce0bdb11ffb20cc8c35e829cc703
+size 28833
diff --git a/tests/reference_details/SmolLM2-1.7B-Instruct-transformers/details_lighteval|bigbench:ruin_names|3_2025-09-19T14-21-59.670987.parquet b/tests/reference_details/SmolLM2-1.7B-Instruct-transformers/details_lighteval|bigbench:ruin_names|3_2025-09-19T14-21-59.670987.parquet
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:aa81111096b52397c3209b64bbdddd8ee2d159ea85c0a3ffc1434aa143c4e3e5
+size 27971
diff --git a/tests/reference_details/SmolLM2-1.7B-Instruct-transformers/details_lighteval|bigbench:salient_translation_error_detection|3_2025-09-19T14-21-59.670987.parquet b/tests/reference_details/SmolLM2-1.7B-Instruct-transformers/details_lighteval|bigbench:salient_translation_error_detection|3_2025-09-19T14-21-59.670987.parquet
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:856678e2f0f10c44da6534ea7a782816ad362c97f4c1a7f9b6d4f856b662f176
+size 49337
diff --git a/tests/reference_details/SmolLM2-1.7B-Instruct-transformers/details_lighteval|bigbench:snarks|3_2025-09-19T14-21-59.670987.parquet b/tests/reference_details/SmolLM2-1.7B-Instruct-transformers/details_lighteval|bigbench:snarks|3_2025-09-19T14-21-59.670987.parquet
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6254940d104b3a34770484ba11a72716b592db4d3afe0d3ab531ad9a88ed1ac7
+size 29148
diff --git a/tests/reference_details/SmolLM2-1.7B-Instruct-transformers/details_lighteval|bigbench:temporal_sequences|3_2025-09-19T14-21-59.670987.parquet b/tests/reference_details/SmolLM2-1.7B-Instruct-transformers/details_lighteval|bigbench:temporal_sequences|3_2025-09-19T14-21-59.670987.parquet
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:82ab2fb145f0ce5ae0d6924052fbce3ce18d20b63f3c1c137722f78388ea75df
+size 34393
diff --git a/tests/reference_details/SmolLM2-1.7B-Instruct-transformers/details_lighteval|bigbench:tracking_shuffled_objects_five_objects|3_2025-09-19T14-21-59.670987.parquet b/tests/reference_details/SmolLM2-1.7B-Instruct-transformers/details_lighteval|bigbench:tracking_shuffled_objects_five_objects|3_2025-09-19T14-21-59.670987.parquet
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dc6b79929d5fef85a7d1b4981c1d730b411345a7b9efa8f4b742c7513e16e13f
+size 34844
diff --git a/tests/reference_details/SmolLM2-1.7B-Instruct-transformers/details_lighteval|bigbench:tracking_shuffled_objects_seven_objects|3_2025-09-19T14-21-59.670987.parquet b/tests/reference_details/SmolLM2-1.7B-Instruct-transformers/details_lighteval|bigbench:tracking_shuffled_objects_seven_objects|3_2025-09-19T14-21-59.670987.parquet
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fe94944f1a6118ded8a22fb47d98f52de893814fe95458223a98bcbd2ae03784
+size 37390
diff --git a/tests/reference_details/SmolLM2-1.7B-Instruct-transformers/details_test|gsm8k|0_2025-09-19T14-21-59.670987.parquet b/tests/reference_details/SmolLM2-1.7B-Instruct-transformers/details_test|gsm8k|0_2025-09-19T14-21-59.670987.parquet
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:61496881db6249102b168966fbe5093b3c9cf67a4045109201ef819772329b5f
+size 35694
diff --git a/tests/reference_details/SmolLM2-1.7B-Instruct-vllm/details_leaderboard|arc:challenge|25_2025-09-19T14-18-26.717757.parquet b/tests/reference_details/SmolLM2-1.7B-Instruct-vllm/details_leaderboard|arc:challenge|25_2025-09-19T14-18-26.717757.parquet
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0b7f37981377cb066cc2641ef6f781ec933f06ce3c1c1906003b088d05909ac1
+size 145429
diff --git a/tests/reference_details/SmolLM2-1.7B-Instruct-vllm/details_leaderboard|hellaswag|10_2025-09-19T14-18-26.717757.parquet b/tests/reference_details/SmolLM2-1.7B-Instruct-vllm/details_leaderboard|hellaswag|10_2025-09-19T14-18-26.717757.parquet
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d9dc068024b011a8364fe359f38b1ca0db8f90d477594985585d3de20914b9e3
+size 138083
diff --git a/tests/reference_details/SmolLM2-1.7B-Instruct-vllm/details_leaderboard|mmlu:college_chemistry|5_2025-09-19T14-18-26.717757.parquet b/tests/reference_details/SmolLM2-1.7B-Instruct-vllm/details_leaderboard|mmlu:college_chemistry|5_2025-09-19T14-18-26.717757.parquet
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5bfd101b63593347b62390965402416dd6d28fe28dfe7ab74437809c3c89a1cd
+size 54470
diff --git a/tests/reference_details/SmolLM2-1.7B-Instruct-vllm/details_leaderboard|mmlu:us_foreign_policy|5_2025-09-19T14-18-26.717757.parquet b/tests/reference_details/SmolLM2-1.7B-Instruct-vllm/details_leaderboard|mmlu:us_foreign_policy|5_2025-09-19T14-18-26.717757.parquet
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d4fa4a5b8ac112bcb5dfb77220b8c6aa7cb85c2800f5f87ee9007a1a20c2b316
+size 55827
diff --git a/tests/reference_details/SmolLM2-1.7B-Instruct-vllm/details_leaderboard|truthfulqa:mc|0_2025-09-19T14-18-26.717757.parquet b/tests/reference_details/SmolLM2-1.7B-Instruct-vllm/details_leaderboard|truthfulqa:mc|0_2025-09-19T14-18-26.717757.parquet
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:452fa36aba5f0a2c0e8db48ec668282d57deea2ea0fb6a6c53f96e9f601a8e4b
+size 32430
diff --git a/tests/reference_details/SmolLM2-1.7B-Instruct-vllm/details_lighteval|agieval:aqua-rat|0_2025-09-19T14-18-26.717757.parquet b/tests/reference_details/SmolLM2-1.7B-Instruct-vllm/details_lighteval|agieval:aqua-rat|0_2025-09-19T14-18-26.717757.parquet
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1edb7b5e552b7306bfc30a4684b2281a9900ce066937d2eff5629cbd954f247a
+size 27096
diff --git a/tests/reference_details/SmolLM2-1.7B-Instruct-vllm/details_lighteval|agieval:logiqa-en|0_2025-09-19T14-18-26.717757.parquet b/tests/reference_details/SmolLM2-1.7B-Instruct-vllm/details_lighteval|agieval:logiqa-en|0_2025-09-19T14-18-26.717757.parquet
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3fb3ccafc05dca361dd081aaab0e52eb91b1b21e4e2d5353898132c5d16bb323
+size 46313
diff --git a/tests/reference_details/SmolLM2-1.7B-Instruct-vllm/details_lighteval|agieval:lsat-ar|0_2025-09-19T14-18-26.717757.parquet b/tests/reference_details/SmolLM2-1.7B-Instruct-vllm/details_lighteval|agieval:lsat-ar|0_2025-09-19T14-18-26.717757.parquet
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:41d62e0194f09de6784eb158cbaaeceecef9e99da1d8bcefd7657f7512f752c0
+size 51342
diff --git a/tests/reference_details/SmolLM2-1.7B-Instruct-vllm/details_lighteval|agieval:lsat-lr|0_2025-09-19T14-18-26.717757.parquet b/tests/reference_details/SmolLM2-1.7B-Instruct-vllm/details_lighteval|agieval:lsat-lr|0_2025-09-19T14-18-26.717757.parquet
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c1e97a11e44ff4004c34b48709b7d4f2c302ab08413f25c40f4844d7e4c1b8aa
+size 56400
diff --git a/tests/reference_details/SmolLM2-1.7B-Instruct-vllm/details_lighteval|agieval:lsat-rc|0_2025-09-19T14-18-26.717757.parquet b/tests/reference_details/SmolLM2-1.7B-Instruct-vllm/details_lighteval|agieval:lsat-rc|0_2025-09-19T14-18-26.717757.parquet
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:390e7989bf5cd74ef8a85f22a280a63b030018dd17353793d677a8316404caf5
+size 149412
diff --git a/tests/reference_details/SmolLM2-1.7B-Instruct-vllm/details_lighteval|agieval:sat-en-without-passage|0_2025-09-19T14-18-26.717757.parquet b/tests/reference_details/SmolLM2-1.7B-Instruct-vllm/details_lighteval|agieval:sat-en-without-passage|0_2025-09-19T14-18-26.717757.parquet
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:00f59db14783b59ba1607373c2a78c04c29f119ea4daefe6e2ba1b93c84409b5
+size 33324
diff --git a/tests/reference_details/SmolLM2-1.7B-Instruct-vllm/details_lighteval|agieval:sat-en|0_2025-09-19T14-18-26.717757.parquet b/tests/reference_details/SmolLM2-1.7B-Instruct-vllm/details_lighteval|agieval:sat-en|0_2025-09-19T14-18-26.717757.parquet
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3ad107238d2c6cb816fa974c9294c3df62e1ed6dbcedebcf5c14aa075133e4dc
+size 110557
diff --git a/tests/reference_details/SmolLM2-1.7B-Instruct-vllm/details_lighteval|bigbench:causal_judgment|3_2025-09-19T14-18-26.717757.parquet b/tests/reference_details/SmolLM2-1.7B-Instruct-vllm/details_lighteval|bigbench:causal_judgment|3_2025-09-19T14-18-26.717757.parquet
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0ecbb89a9735670fa0b5a0c20619fc1de66a2a791f6c12e1a7ba029c17bba4db
+size 72052
diff --git a/tests/reference_details/SmolLM2-1.7B-Instruct-vllm/details_lighteval|bigbench:date_understanding|3_2025-09-19T14-18-26.717757.parquet b/tests/reference_details/SmolLM2-1.7B-Instruct-vllm/details_lighteval|bigbench:date_understanding|3_2025-09-19T14-18-26.717757.parquet
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cd97d2a2dd9653dd63f7d643e8a8b657af312f5645e4a1af3f2dd28fe9daada2
+size 39423
diff --git a/tests/reference_details/SmolLM2-1.7B-Instruct-vllm/details_lighteval|bigbench:disambiguation_qa|3_2025-09-19T14-18-26.717757.parquet b/tests/reference_details/SmolLM2-1.7B-Instruct-vllm/details_lighteval|bigbench:disambiguation_qa|3_2025-09-19T14-18-26.717757.parquet
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7577f9421170e3ad4b21c88c67b35b7fb896bdf65a070618917e83b16af5c994
+size 38263
diff --git a/tests/reference_details/SmolLM2-1.7B-Instruct-vllm/details_lighteval|bigbench:geometric_shapes|3_2025-09-19T14-18-26.717757.parquet b/tests/reference_details/SmolLM2-1.7B-Instruct-vllm/details_lighteval|bigbench:geometric_shapes|3_2025-09-19T14-18-26.717757.parquet
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1a68770cb413ed6afa04191c575f5bd4d3070d49f2d0018281deff6ea54be2ea
+size 49571
diff --git a/tests/reference_details/SmolLM2-1.7B-Instruct-vllm/details_lighteval|bigbench:logical_deduction_five_objects|3_2025-09-19T14-18-26.717757.parquet b/tests/reference_details/SmolLM2-1.7B-Instruct-vllm/details_lighteval|bigbench:logical_deduction_five_objects|3_2025-09-19T14-18-26.717757.parquet
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:53138809f79be81f1fa604404f4798972677c186972a4b44a596df780531ffc9
+size 47557
diff --git a/tests/reference_details/SmolLM2-1.7B-Instruct-vllm/details_lighteval|bigbench:logical_deduction_seven_objects|3_2025-09-19T14-18-26.717757.parquet b/tests/reference_details/SmolLM2-1.7B-Instruct-vllm/details_lighteval|bigbench:logical_deduction_seven_objects|3_2025-09-19T14-18-26.717757.parquet
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7d51b95d961a2eaf6d968fa0c497cd2018706f1cb1603d689e2b10696a0dff5c
+size 56164
diff --git a/tests/reference_details/SmolLM2-1.7B-Instruct-vllm/details_lighteval|bigbench:movie_recommendation|3_2025-09-19T14-18-26.717757.parquet b/tests/reference_details/SmolLM2-1.7B-Instruct-vllm/details_lighteval|bigbench:movie_recommendation|3_2025-09-19T14-18-26.717757.parquet
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:86d8278d42c12ebe1c899666267d20e165681ec1016d73dcdee26950281065fc
+size 36502
diff --git a/tests/reference_details/SmolLM2-1.7B-Instruct-vllm/details_lighteval|bigbench:navigate|3_2025-09-19T14-18-26.717757.parquet b/tests/reference_details/SmolLM2-1.7B-Instruct-vllm/details_lighteval|bigbench:navigate|3_2025-09-19T14-18-26.717757.parquet
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:03a10a1e6e2bc70196229f24660ec1e8c2d893cf11f10241797dac227984aea1
+size 34537
diff --git a/tests/reference_details/SmolLM2-1.7B-Instruct-vllm/details_lighteval|bigbench:ruin_names|3_2025-09-19T14-18-26.717757.parquet b/tests/reference_details/SmolLM2-1.7B-Instruct-vllm/details_lighteval|bigbench:ruin_names|3_2025-09-19T14-18-26.717757.parquet
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9cf4eac2cbef996c90e580d5d7fbac9cc0336c23e8d660b8e5b4d37eba224220
+size 34510
diff --git a/tests/reference_details/SmolLM2-1.7B-Instruct-vllm/details_lighteval|bigbench:salient_translation_error_detection|3_2025-09-19T14-18-26.717757.parquet b/tests/reference_details/SmolLM2-1.7B-Instruct-vllm/details_lighteval|bigbench:salient_translation_error_detection|3_2025-09-19T14-18-26.717757.parquet
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bf7a2d2588fc351ba2e74a5ede6c112098326cb910b8b8f04b9430abe96a5664
+size 69137
diff --git a/tests/reference_details/SmolLM2-1.7B-Instruct-vllm/details_lighteval|bigbench:snarks|3_2025-09-19T14-18-26.717757.parquet b/tests/reference_details/SmolLM2-1.7B-Instruct-vllm/details_lighteval|bigbench:snarks|3_2025-09-19T14-18-26.717757.parquet
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:76975bfc02c4592c3d5e82a0b50dab43be25d38187f99126b8b305f29a782dbd
+size 37856
diff --git a/tests/reference_details/SmolLM2-1.7B-Instruct-vllm/details_lighteval|bigbench:temporal_sequences|3_2025-09-19T14-18-26.717757.parquet b/tests/reference_details/SmolLM2-1.7B-Instruct-vllm/details_lighteval|bigbench:temporal_sequences|3_2025-09-19T14-18-26.717757.parquet
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0bfba9b70b3aa79f513684e1eebf1ce671eeedbd59862ceea14fcf205c9f490b
+size 50277
diff --git a/tests/reference_details/SmolLM2-1.7B-Instruct-vllm/details_lighteval|bigbench:tracking_shuffled_objects_five_objects|3_2025-09-19T14-18-26.717757.parquet b/tests/reference_details/SmolLM2-1.7B-Instruct-vllm/details_lighteval|bigbench:tracking_shuffled_objects_five_objects|3_2025-09-19T14-18-26.717757.parquet
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0f085820dfa02c1c4d18300485912f1088cf16e746efd6d645075c07733e643c
+size 52495
diff --git a/tests/reference_details/SmolLM2-1.7B-Instruct-vllm/details_lighteval|bigbench:tracking_shuffled_objects_seven_objects|3_2025-09-19T14-18-26.717757.parquet b/tests/reference_details/SmolLM2-1.7B-Instruct-vllm/details_lighteval|bigbench:tracking_shuffled_objects_seven_objects|3_2025-09-19T14-18-26.717757.parquet
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e12c1980dfe5479eb6b376db7221c4529e75dd21626137fcf5921a2136a34ad9
+size 59841
diff --git a/tests/reference_details/SmolLM2-1.7B-Instruct-vllm/details_test|gsm8k|0_2025-09-19T14-18-26.717757.parquet b/tests/reference_details/SmolLM2-1.7B-Instruct-vllm/details_test|gsm8k|0_2025-09-19T14-18-26.717757.parquet
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cf85514ec1ae523022070905f2a3f6ecb4d144314e4e57af04482ed9b310572a
+size 39054
diff --git a/tests/slow_tests/sample_comparison.py b/tests/slow_tests/sample_comparison.py
diff --git a/tests/slow_tests/test_accelerate_model.py b/tests/slow_tests/test_accelerate_model.py
diff --git a/tests/slow_tests/test_accelerate_vlm_model.py b/tests/slow_tests/test_accelerate_vlm_model.py
diff --git a/tests/slow_tests/test_vllm_model.py b/tests/slow_tests/test_vllm_model.py

Original file line number	Diff line number	Diff line change
`@@ -1,2 +1,3 @@`
`1`	`1`	`*.json filter=lfs diff=lfs merge=lfs -text`
`2`	`2`	`tests/unit/metrics/test_cases/*.json -filter -diff -merge text`
	`3`	`+*.parquet filter=lfs diff=lfs merge=lfs -text`
Original file line number	Diff line number	Diff line change
`@@ -0,0 +1,3 @@`
	`1`	`+version https://git-lfs.github.com/spec/v1`
	`2`	`+oid sha256:228e58884c153beb351f47a3d0c5704d9e764844aa6990375855ebdddfe234cb`
	`3`	`+size 11827875`
Original file line number	Diff line number	Diff line change
`@@ -0,0 +1,3 @@`
	`1`	`+version https://git-lfs.github.com/spec/v1`
	`2`	`+oid sha256:1f470cfb11d5901a644ccf28b267829448c511929034ee520a4f870253e3be99`
	`3`	`+size 11540572`
Original file line number	Diff line number	Diff line change
`@@ -0,0 +1,3 @@`
	`1`	`+version https://git-lfs.github.com/spec/v1`
	`2`	`+oid sha256:c2168453c9482af24bcb501f7b1b5f4f83f261e416c40db8e23ec123a13ef27f`
	`3`	`+size 27746331`
Original file line number	Diff line number	Diff line change
`@@ -0,0 +1,3 @@`
	`1`	`+version https://git-lfs.github.com/spec/v1`
	`2`	`+oid sha256:bf8c0061d55f76265ec3d88762e4d806ce2d932909384cd03637413fd5cb89be`
	`3`	`+size 88248`