We read every piece of feedback, and take your input very seriously.
To see all available qualifiers, see our documentation.
There was an error while loading. Please reload this page.
1 parent 1bc6efd commit 9b1cae6Copy full SHA for 9b1cae6
mellea/backends/sglang.py
@@ -15,6 +15,7 @@
15
from collections.abc import Callable
16
from typing import TYPE_CHECKING, Any, Optional
17
18
+import nest_asyncio
19
import sglang as sgl # type:ignore
20
import torch
21
from transformers import AutoTokenizer, PreTrainedTokenizerBase
@@ -41,6 +42,8 @@
41
42
from mellea.stdlib.chat import Message
43
from mellea.stdlib.requirement import LLMaJRequirement, Requirement
44
45
+nest_asyncio.apply()
46
+
47
48
class LocalSGLangBackend(FormatterBackend):
49
"""The LocalSGLangBackend uses SGLang's python offline inference interface, and uses a Formatter to convert `Component`s into prompts.
0 commit comments