We read every piece of feedback, and take your input very seriously.
To see all available qualifiers, see our documentation.
There was an error while loading. Please reload this page.
1 parent 4033442 commit 4a286c5Copy full SHA for 4a286c5
mellea/backends/sglang.py
@@ -17,6 +17,7 @@
17
from collections.abc import Callable
18
from typing import TYPE_CHECKING, Any, Optional
19
20
+import nest_asyncio
21
import sglang as sgl # type:ignore
22
from sglang.utils import async_stream_and_merge # type:ignore
23
from transformers import AutoTokenizer, PreTrainedTokenizerBase
@@ -45,6 +46,8 @@
45
46
from mellea.stdlib.chat import Message
47
from mellea.stdlib.requirement import LLMaJRequirement, Requirement
48
49
+nest_asyncio.apply()
50
+
51
52
class LocalSGLangBackend(FormatterBackend):
53
"""The LocalSGLangBackend uses SGLang's python offline inference interface, and uses a Formatter to convert `Component`s into prompts.
0 commit comments