With transformers >= 4.56 the error message is more explicit and doesn't match exactly

Signed-off-by: Gregory Shtrasberg <[email protected]>

Change condition to check for inclusion

Signed-off-by: Gregory Shtrasberg <[email protected]>

Fix DecodeStream API change

Signed-off-by: Gregory Shtrasberg <[email protected]>

Author: gshtras

vllm/v1/engine/detokenizer.py

@@ -234,7 +234,7 @@ def _protected_step(self, next_token_id: int) -> Optional[str]:
         try:
             token = self.stream.step(self.tokenizer, next_token_id)
         except Exception as e:
-            if str(e) != INVALID_PREFIX_ERR_MSG:
+            if INVALID_PREFIX_ERR_MSG not in str(e):
                 raise e
             # Recover from edge case where tokenizer can produce non-monotonic,
             # invalid UTF-8 output, which breaks the internal state of
@@ -243,7 +243,8 @@ def _protected_step(self, next_token_id: int) -> Optional[str]:
             logger.warning(
                 "Encountered invalid prefix detokenization error"
                 " for request %s, resetting decode stream.", self.request_id)
-            self.stream = DecodeStream(self.skip_special_tokens)
+            self.stream = DecodeStream(
+                skip_special_tokens=self.skip_special_tokens)
             token = self.stream.step(self.tokenizer, next_token_id)
         return token