convert to 8 generation

Tong Li · Tong Li · commit 4702d5784145 · 2025-03-13T16:49:02.000+08:00
diff --git a/applications/ColossalChat/coati/distributed/inference_backend.py b/applications/ColossalChat/coati/distributed/inference_backend.py
@@ -154,7 +154,7 @@ class VLLMInferenceBackend(BaseInferenceBackend):
     )
     FORCE_GENERATE_CONFIG = dict(
         logprobs=0,
-        n=4,
+        n=8,
     )
 
     def __init__(self, model_config: Dict[str, Any], generate_config: Dict[str, Any], tokenizer: PreTrainedTokenizer):

Original file line number	Diff line number	Diff line change
`@@ -154,7 +154,7 @@ class VLLMInferenceBackend(BaseInferenceBackend):`
`154`	`154`	`)`
`155`	`155`	`FORCE_GENERATE_CONFIG = dict(`
`156`	`156`	`logprobs=0,`
`157`		`- n=4,`
	`157`	`+ n=8,`
`158`	`158`	`)`
`159`	`159`
`160`	`160`	`def __init__(self, model_config: Dict[str, Any], generate_config: Dict[str, Any], tokenizer: PreTrainedTokenizer):`