Reasoning tags removal is true by default (#925)

* rm reasoning tags true by default

* Update src/lighteval/main_vllm.py

Author: clefourrier

src/lighteval/main_accelerate.py

@@ -61,7 +61,11 @@ def accelerate(  # noqa C901
         Optional[str], Option(help="Load responses from details directory.", rich_help_panel=HELP_PANEL_NAME_1)
     ] = None,
     remove_reasoning_tags: Annotated[
-        bool, Option(help="Remove reasoning tags from responses.", rich_help_panel=HELP_PANEL_NAME_1)
+        bool | None,
+        Option(
+            help="Remove reasoning tags from responses (true to remove, false to leave - true by default).",
+            rich_help_panel=HELP_PANEL_NAME_1,
+        ),
     ] = True,
     reasoning_tags: Annotated[
         str | None,

src/lighteval/main_custom.py

@@ -54,7 +54,11 @@ def custom(
         int, Option(help="Number of seeds to use for few-shot evaluation.", rich_help_panel=HELP_PANEL_NAME_1)
     ] = 1,
     remove_reasoning_tags: Annotated[
-        bool, Option(help="Remove reasoning tags from responses.", rich_help_panel=HELP_PANEL_NAME_1)
+        bool | None,
+        Option(
+            help="Remove reasoning tags from responses (true to remove, false to leave - true by default).",
+            rich_help_panel=HELP_PANEL_NAME_1,
+        ),
     ] = True,
     reasoning_tags: Annotated[
         str | None,

src/lighteval/main_endpoint.py

@@ -63,7 +63,11 @@ def inference_endpoint(
         Optional[str], Option(help="Load responses from details directory.", rich_help_panel=HELP_PANEL_NAME_1)
     ] = None,
     remove_reasoning_tags: Annotated[
-        bool, Option(help="Remove reasoning tags from responses.", rich_help_panel=HELP_PANEL_NAME_1)
+        bool | None,
+        Option(
+            help="Remove reasoning tags from responses (true to remove, false to leave - true by default).",
+            rich_help_panel=HELP_PANEL_NAME_1,
+        ),
     ] = True,
     reasoning_tags: Annotated[
         str | None,
@@ -188,7 +192,11 @@ def tgi(
         Optional[str], Option(help="Load responses from details directory.", rich_help_panel=HELP_PANEL_NAME_1)
     ] = None,
     remove_reasoning_tags: Annotated[
-        bool, Option(help="Remove reasoning tags from responses.", rich_help_panel=HELP_PANEL_NAME_1)
+        bool | None,
+        Option(
+            help="Remove reasoning tags from responses (true to remove, false to leave - true by default).",
+            rich_help_panel=HELP_PANEL_NAME_1,
+        ),
     ] = True,
     reasoning_tags: Annotated[
         str | None,
@@ -320,7 +328,11 @@ def litellm(
         Optional[str], Option(help="Load responses from details directory.", rich_help_panel=HELP_PANEL_NAME_1)
     ] = None,
     remove_reasoning_tags: Annotated[
-        bool, Option(help="Remove reasoning tags from responses.", rich_help_panel=HELP_PANEL_NAME_1)
+        bool | None,
+        Option(
+            help="Remove reasoning tags from responses (true to remove, false to leave - true by default).",
+            rich_help_panel=HELP_PANEL_NAME_1,
+        ),
     ] = True,
     reasoning_tags: Annotated[
         str | None,
@@ -486,7 +498,11 @@ def inference_providers(
         ),
     ] = False,
     remove_reasoning_tags: Annotated[
-        bool, Option(help="Remove reasoning tags from responses.", rich_help_panel=HELP_PANEL_NAME_1)
+        bool | None,
+        Option(
+            help="Remove reasoning tags from responses (true to remove, false to leave - true by default).",
+            rich_help_panel=HELP_PANEL_NAME_1,
+        ),
     ] = True,
     reasoning_tags: Annotated[
         str | None,

src/lighteval/main_nanotron.py

@@ -44,7 +44,11 @@ def nanotron(
     ],
     lighteval_config_path: Annotated[str, Option(help="Path to a YAML config to be used for the evaluation.")],
     remove_reasoning_tags: Annotated[
-        bool, Option(help="Remove reasoning tags from responses.", rich_help_panel=HELP_PANEL_NAME_1)
+        bool | None,
+        Option(
+            help="Remove reasoning tags from responses (true to remove, false to leave - true by default).",
+            rich_help_panel=HELP_PANEL_NAME_1,
+        ),
     ] = True,
     reasoning_tags: Annotated[
         str | None,

src/lighteval/main_sglang.py

@@ -54,7 +54,11 @@ def sglang(
         Optional[str], Option(help="Load responses from details directory.", rich_help_panel=HELP_PANEL_NAME_1)
     ] = None,
     remove_reasoning_tags: Annotated[
-        bool, Option(help="Remove reasoning tags from responses.", rich_help_panel=HELP_PANEL_NAME_1)
+        bool | None,
+        Option(
+            help="Remove reasoning tags from responses (true to remove, false to leave - true by default).",
+            rich_help_panel=HELP_PANEL_NAME_1,
+        ),
     ] = True,
     reasoning_tags: Annotated[
         str | None,

src/lighteval/main_vllm.py

@@ -57,8 +57,12 @@ def vllm(
         Optional[str], Option(help="Load responses from details directory.", rich_help_panel=HELP_PANEL_NAME_1)
     ] = None,
     remove_reasoning_tags: Annotated[
-        bool, Option(help="Remove reasoning tags from responses.", rich_help_panel=HELP_PANEL_NAME_1)
-    ] = False,
+        bool | None,
+        Option(
+            help="Remove reasoning tags from responses (true to remove, false to leave - true by default).",
+            rich_help_panel=HELP_PANEL_NAME_1,
+        ),
+    ] = True,
     reasoning_tags: Annotated[
         str | None,
         Option(