feature: add sglang backend

Author: denyszhak

main.go

@@ -15,6 +15,7 @@ import (
 	"github.com/docker/model-runner/pkg/inference"
 	"github.com/docker/model-runner/pkg/inference/backends/llamacpp"
 	"github.com/docker/model-runner/pkg/inference/backends/mlx"
+	"github.com/docker/model-runner/pkg/inference/backends/sglang"
 	"github.com/docker/model-runner/pkg/inference/backends/vllm"
 	"github.com/docker/model-runner/pkg/inference/config"
 	"github.com/docker/model-runner/pkg/inference/memory"
@@ -144,12 +145,23 @@ func main() {
 		log.Fatalf("unable to initialize %s backend: %v", mlx.Name, err)
 	}
 
+	sglangBackend, err := sglang.New(
+		log,
+		modelManager,
+		log.WithFields(logrus.Fields{"component": sglang.Name}),
+		nil,
+	)
+	if err != nil {
+		log.Fatalf("unable to initialize %s backend: %v", sglang.Name, err)
+	}
+
 	scheduler := scheduling.NewScheduler(
 		log,
 		map[string]inference.Backend{
 			llamacpp.Name: llamaCppBackend,
 			vllm.Name:     vllmBackend,
 			mlx.Name:      mlxBackend,
+			sglang.Name:   sglangBackend,
 		},
 		llamaCppBackend,
 		modelManager,

pkg/inference/backend.go

@@ -72,6 +72,10 @@ type Backend interface {
 	// external model management system and false if the backend uses the shared
 	// model manager.
 	UsesExternalModelManagement() bool
+	// UsesTCP returns true if the backend uses TCP for communication instead
+	// of Unix sockets. When true, the scheduler will create a TCP transport
+	// and pass a "host:port" address to Run instead of a Unix socket path.
+	UsesTCP() bool
 	// Install ensures that the backend is installed. It should return a nil
 	// error if installation succeeds or if the backend is already installed.
 	// The provided HTTP client should be used for any HTTP operations.

pkg/inference/backends/llamacpp/llamacpp.go

@@ -89,6 +89,11 @@ func (l *llamaCpp) UsesExternalModelManagement() bool {
 	return false
 }
 
+// UsesTCP implements inference.Backend.UsesTCP.
+func (l *llamaCpp) UsesTCP() bool {
+	return false
+}
+
 // Install implements inference.Backend.Install.
 func (l *llamaCpp) Install(ctx context.Context, httpClient *http.Client) error {
 	l.updatedLlamaCpp = false

pkg/inference/backends/mlx/mlx.go

@@ -65,6 +65,11 @@ func (m *mlx) UsesExternalModelManagement() bool {
 	return false
 }
 
+// UsesTCP implements inference.Backend.UsesTCP.
+func (m *mlx) UsesTCP() bool {
+	return false
+}
+
 // Install implements inference.Backend.Install.
 func (m *mlx) Install(ctx context.Context, httpClient *http.Client) error {
 	if !platform.SupportsMLX() {

pkg/inference/backends/sglang/sglang.go

@@ -0,0 +1,226 @@
+package sglang
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"io/fs"
+	"net/http"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strings"
+
+	"github.com/docker/model-runner/pkg/diskusage"
+	"github.com/docker/model-runner/pkg/inference"
+	"github.com/docker/model-runner/pkg/inference/backends"
+	"github.com/docker/model-runner/pkg/inference/models"
+	"github.com/docker/model-runner/pkg/inference/platform"
+	"github.com/docker/model-runner/pkg/logging"
+)
+
+const (
+	// Name is the backend name.
+	Name      = "sglang"
+	sglangDir = "/opt/sglang-env/bin"
+	sglangVersionFile = "/opt/sglang-env/version"
+)
+
+var (
+	ErrNotImplemented = errors.New("not implemented")
+	ErrSGLangNotFound  = errors.New("sglang package not installed")
+	ErrPythonNotFound  = errors.New("python3 not found in PATH")
+)
+
+// sglang is the SGLang-based backend implementation.
+type sglang struct {
+	// log is the associated logger.
+	log logging.Logger
+	// modelManager is the shared model manager.
+	modelManager *models.Manager
+	// serverLog is the logger to use for the SGLang server process.
+	serverLog logging.Logger
+	// config is the configuration for the SGLang backend.
+	config *Config
+	// status is the state in which the SGLang backend is in.
+	status string
+	// pythonPath is the path to the python3 binary.
+	pythonPath string
+}
+
+// New creates a new SGLang-based backend.
+func New(log logging.Logger, modelManager *models.Manager, serverLog logging.Logger, conf *Config) (inference.Backend, error) {
+	// If no config is provided, use the default configuration
+	if conf == nil {
+		conf = NewDefaultSGLangConfig()
+	}
+
+	return &sglang{
+		log:          log,
+		modelManager: modelManager,
+		serverLog:    serverLog,
+		config:       conf,
+		status:       "not installed",
+	}, nil
+}
+
+// Name implements inference.Backend.Name.
+func (s *sglang) Name() string {
+	return Name
+}
+
+func (s *sglang) UsesExternalModelManagement() bool {
+	return false
+}
+
+// UsesTCP implements inference.Backend.UsesTCP.
+// SGLang only supports TCP, not Unix sockets.
+func (s *sglang) UsesTCP() bool {
+	return true
+}
+
+func (s *sglang) Install(_ context.Context, _ *http.Client) error {
+	if !platform.SupportsSGLang() {
+		return ErrNotImplemented
+	}
+
+	if err := s.initFromDocker(); err == nil {
+		return nil
+	} else if !errors.Is(err, fs.ErrNotExist) {
+		return fmt.Errorf("failed to check SGLang binary: %w", err)
+	}
+
+	return s.initFromHost()
+}
+
+
+func (s *sglang) initFromDocker() error {
+	sglangBinaryPath := s.binaryPath()
+
+	if _, err := os.Stat(sglangBinaryPath); err != nil {
+		return err
+	}
+
+	versionBytes, err := os.ReadFile(sglangVersionFile)
+	if err != nil {
+		s.log.Warnf("could not get sglang version: %v", err)
+		s.status = "running sglang version: unknown"
+		return nil
+	}
+
+	s.status = fmt.Sprintf(
+		"running sglang version: %s",
+		strings.TrimSpace(string(versionBytes)),
+	)
+
+	return nil
+}
+
+
+func (s *sglang) initFromHost() error {
+	pythonPath, err := exec.LookPath("python3")
+	if err != nil {
+		s.status = ErrPythonNotFound.Error()
+		return ErrPythonNotFound
+	}
+
+	s.pythonPath = pythonPath
+
+	if err := exec.Command(pythonPath, "-c", "import sglang").Run(); err != nil {
+		s.status = "sglang package not installed"
+		s.log.Warnf("sglang package not found. Install with: uv pip install sglang[all]")
+		return ErrSGLangNotFound
+	}
+
+	output, err := exec.Command(pythonPath, "-c", "import sglang; print(sglang.__version__)").Output()
+	if err != nil {
+		s.log.Warnf("could not get sglang version: %v", err)
+		s.status = "running sglang version: unknown"
+		return nil
+	}
+
+	s.status = fmt.Sprintf("running sglang version: %s", strings.TrimSpace(string(output)))
+
+	return nil
+}
+
+func (s *sglang) Run(ctx context.Context, socket, model string, modelRef string, mode inference.BackendMode, backendConfig *inference.BackendConfiguration) error {
+	if !platform.SupportsSGLang() {
+		s.log.Warn("sglang backend is not yet supported")
+		return ErrNotImplemented
+	}
+
+	bundle, err := s.modelManager.GetBundle(model)
+	if err != nil {
+		return fmt.Errorf("failed to get model: %w", err)
+	}
+
+	args, err := s.config.GetArgs(bundle, socket, mode, backendConfig)
+	if err != nil {
+		return fmt.Errorf("failed to get SGLang arguments: %w", err)
+	}
+
+	// Add served model name and weight version
+	if model != "" {
+		args = append(args, "--served-model-name", model)
+	}
+	if modelRef != "" {
+		args = append(args, "--weight-version", modelRef)
+	}
+
+	// Determine binary path - use Docker installation if available, otherwise use Python
+	binaryPath := s.binaryPath()
+	sandboxPath := sglangDir
+	if _, err := os.Stat(binaryPath); errors.Is(err, fs.ErrNotExist) {
+		// Use Python installation
+		if s.pythonPath == "" {
+			return fmt.Errorf("sglang: no docker binary at %s and no python runtime configured; did you forget to call Install?", binaryPath)
+		}
+		binaryPath = s.pythonPath
+		sandboxPath = ""
+	}
+
+	return backends.RunBackend(ctx, backends.RunnerConfig{
+		BackendName:     "SGLang",
+		Socket:          socket,
+		BinaryPath:      binaryPath,
+		SandboxPath:     sandboxPath,
+		SandboxConfig:   "",
+		Args:            args,
+		Logger:          s.log,
+		ServerLogWriter: s.serverLog.Writer(),
+	})
+}
+
+func (s *sglang) Status() string {
+	return s.status
+}
+
+func (s *sglang) GetDiskUsage() (int64, error) {
+	// Check if Docker installation exists
+	if _, err := os.Stat(sglangDir); err == nil {
+		size, err := diskusage.Size(sglangDir)
+		if err != nil {
+			return 0, fmt.Errorf("error while getting sglang dir size: %w", err)
+		}
+		return size, nil
+	}
+	// Python installation doesn't have a dedicated installation directory
+	// It's installed via pip in the system Python environment
+	return 0, nil
+}
+
+func (s *sglang) GetRequiredMemoryForModel(_ context.Context, _ string, _ *inference.BackendConfiguration) (inference.RequiredMemory, error) {
+	if !platform.SupportsSGLang() {
+		return inference.RequiredMemory{}, ErrNotImplemented
+	}
+
+	return inference.RequiredMemory{
+		RAM:  1,
+		VRAM: 1,
+	}, nil
+}
+
+func (s *sglang) binaryPath() string {
+	return filepath.Join(sglangDir, "sglang")
+}

pkg/inference/backends/sglang/sglang_config.go

@@ -0,0 +1,85 @@
+package sglang
+
+import (
+	"fmt"
+	"net"
+	"path/filepath"
+	"strconv"
+
+	"github.com/docker/model-runner/pkg/distribution/types"
+	"github.com/docker/model-runner/pkg/inference"
+)
+
+// Config is the configuration for the SGLang backend.
+type Config struct {
+	// Args are the base arguments that are always included.
+	Args []string
+}
+
+// NewDefaultSGLangConfig creates a new SGLangConfig with default values.
+func NewDefaultSGLangConfig() *Config {
+	return &Config{}
+}
+
+// GetArgs implements BackendConfig.GetArgs.
+func (c *Config) GetArgs(bundle types.ModelBundle, socket string, mode inference.BackendMode, config *inference.BackendConfiguration) ([]string, error) {
+	// Start with the arguments from SGLangConfig
+	args := append([]string{}, c.Args...)
+
+	// SGLang uses Python module: python -m sglang.launch_server
+	args = append(args, "-m", "sglang.launch_server")
+
+	// Add model path
+	safetensorsPath := bundle.SafetensorsPath()
+	if safetensorsPath == "" {
+		return nil, fmt.Errorf("safetensors path required by SGLang backend")
+	}
+	modelPath := filepath.Dir(safetensorsPath)
+	args = append(args, "--model-path", modelPath)
+
+	host, port, err := net.SplitHostPort(socket)
+	if err != nil {
+		return nil, fmt.Errorf("failed to parse host:port from %q: %w", socket, err)
+	}
+	args = append(args, "--host", host, "--port", port)
+
+	// Add mode-specific arguments
+	switch mode {
+	case inference.BackendModeCompletion:
+		// Default mode for SGLang
+	case inference.BackendModeEmbedding:
+		args = append(args, "--is-embedding")
+	case inference.BackendModeReranking:
+	default:
+		return nil, fmt.Errorf("unsupported backend mode %q", mode)
+	}
+
+	// Add context-length if specified in model config or backend config
+	if contextLen := GetContextLength(bundle.RuntimeConfig(), config); contextLen != nil {
+		args = append(args, "--context-length", strconv.FormatUint(*contextLen, 10))
+	}
+
+	// Add arguments from backend config
+	if config != nil {
+		args = append(args, config.RuntimeFlags...)
+	}
+
+	return args, nil
+}
+
+// GetContextLength returns the context length (context size) from model config or backend config.
+// Model config takes precedence over backend config.
+// Returns nil if neither is specified (SGLang will auto-derive from model).
+func GetContextLength(modelCfg types.Config, backendCfg *inference.BackendConfiguration) *uint64 {
+	// Model config takes precedence
+	if modelCfg.ContextSize != nil {
+		return modelCfg.ContextSize
+	}
+	// else use backend config
+	if backendCfg != nil && backendCfg.ContextSize > 0 {
+		val := uint64(backendCfg.ContextSize)
+		return &val
+	}
+	// Return nil to let SGLang auto-derive from model config
+	return nil
+}