We read every piece of feedback, and take your input very seriously.
To see all available qualifiers, see our documentation.
There was an error while loading. Please reload this page.
1 parent b84ae9e commit bc3cfc3Copy full SHA for bc3cfc3
graphgen/models/llm/local/sglang_wrapper.py
@@ -16,6 +16,7 @@ def __init__(
16
temperature: float = 0.0,
17
top_p: float = 1.0,
18
topk: int = 5,
19
+ tp_size: int = 1,
20
**kwargs: Any,
21
):
22
super().__init__(temperature=temperature, top_p=top_p, **kwargs)
@@ -34,7 +35,7 @@ def __init__(
34
35
self.topk = topk
36
37
# Initialise the offline engine
- self.engine = sgl.Engine(model_path=self.model_path)
38
+ self.engine = sgl.Engine(model_path=self.model_path, tp_size=int(tp_size))
39
40
# Keep helpers for streaming
41
self.async_stream_and_merge = async_stream_and_merge
0 commit comments