raise exception when generation size is more than model length (#571)

* raise exception when generation size is more than model length

* trufflehog

* trufflehog

Author: NathanHB

.github/workflows/trufflehog.yml

@@ -16,3 +16,5 @@ jobs:
         fetch-depth: 0
     - name: Secret Scanning
       uses: trufflesecurity/trufflehog@main
+      with:
+        extra_args: --only-verified

src/lighteval/__main__.py

@@ -34,6 +34,7 @@
 
 
 app = typer.Typer()
+app = typer.Typer(pretty_exceptions_show_locals=False)
 
 logging_config = dict(  # noqa C408
     version=1,

src/lighteval/models/vllm/vllm_model.py

@@ -269,6 +269,11 @@ def greedy_until(
                         f"{context_size + max_new_tokens=} which is greather than {self.max_length=}. Truncating context to {self.max_length - max_new_tokens} tokens."
                     )
                     context_size = self.max_length - max_new_tokens
+                    if context_size < 0:
+                        logger.critical(
+                            f"{context_size=} is less than 0, either reduce the max_new_tokens or increase model max length."
+                        )
+                        raise ValueError("Context size is less than 0.")
                     inputs = [input[-context_size:] for input in inputs]
             else:
                 if context_size > self.max_length: