inference: adjust for lack of logger and paths packages

Signed-off-by: Jacob Howard <[email protected]>

Author: xenoscopic

pkg/inference/backends/llamacpp/download.go

@@ -14,15 +14,15 @@ import (
 	"strings"
 
 	"github.com/docker/model-runner/pkg/internal/dockerhub"
-	"github.com/docker/model-runner/pkg/paths"
+	"github.com/docker/model-runner/pkg/logging"
 )
 
 const (
 	hubNamespace = "docker"
 	hubRepo      = "docker-model-backend-llamacpp"
 )
 
-func ensureLatestLlamaCpp(ctx context.Context, httpClient *http.Client, llamaCppPath string) error {
+func ensureLatestLlamaCpp(ctx context.Context, log logging.Logger, httpClient *http.Client, llamaCppPath string) error {
 	url := fmt.Sprintf("https://hub.docker.com/v2/namespaces/%s/repositories/%s/tags", hubNamespace, hubRepo)
 	resp, err := httpClient.Get(url)
 	if err != nil {
@@ -74,10 +74,13 @@ func ensureLatestLlamaCpp(ctx context.Context, httpClient *http.Client, llamaCpp
 	}
 
 	image := fmt.Sprintf("registry-1.docker.io/%s/%s@%s", hubNamespace, hubRepo, latest)
-	downloadDir := paths.DockerHome(".llamacpp-tmp")
+	downloadDir, err := os.MkdirTemp("", "llamacpp-install")
+	if err != nil {
+		return fmt.Errorf("could not create temporary directory: %w", err)
+	}
 	defer os.RemoveAll(downloadDir)
 
-	if err := extractFromImage(ctx, image, runtime.GOOS, runtime.GOARCH, downloadDir); err != nil {
+	if err := extractFromImage(ctx, log, image, runtime.GOOS, runtime.GOARCH, downloadDir); err != nil {
 		return fmt.Errorf("could not extract image: %w", err)
 	}
 
@@ -97,7 +100,7 @@ func ensureLatestLlamaCpp(ctx context.Context, httpClient *http.Client, llamaCpp
 	}
 
 	log.Infoln("successfully updated llama.cpp binary")
-	log.Infoln("running llama.cpp version:", getLlamaCppVersion(llamaCppPath))
+	log.Infoln("running llama.cpp version:", getLlamaCppVersion(log, llamaCppPath))
 
 	if err := os.WriteFile(currentVersionFile, []byte(latest), 0o644); err != nil {
 		log.Warnf("failed to save llama.cpp version: %v", err)
@@ -106,7 +109,7 @@ func ensureLatestLlamaCpp(ctx context.Context, httpClient *http.Client, llamaCpp
 	return nil
 }
 
-func extractFromImage(ctx context.Context, image, requiredOs, requiredArch, destination string) error {
+func extractFromImage(ctx context.Context, log logging.Logger, image, requiredOs, requiredArch, destination string) error {
 	log.Infof("Extracting image %q to %q", image, destination)
 	tmpDir, err := os.MkdirTemp("", "docker-tar-extract")
 	if err != nil {
@@ -119,7 +122,7 @@ func extractFromImage(ctx context.Context, image, requiredOs, requiredArch, dest
 	return dockerhub.Extract(imageTar, requiredArch, requiredOs, destination)
 }
 
-func getLlamaCppVersion(llamaCpp string) string {
+func getLlamaCppVersion(log logging.Logger, llamaCpp string) string {
 	output, err := exec.Command(llamaCpp, "--version").CombinedOutput()
 	if err != nil {
 		log.Warnf("could not get llama.cpp version: %v", err)

pkg/inference/backends/llamacpp/llamacpp.go

@@ -12,34 +12,49 @@ import (
 
 	"github.com/docker/model-runner/pkg/inference"
 	"github.com/docker/model-runner/pkg/inference/models"
-	"github.com/docker/model-runner/pkg/logger"
-	"github.com/docker/model-runner/pkg/paths"
+	"github.com/docker/model-runner/pkg/logging"
 )
 
 const (
 	// Name is the backend name.
 	Name = "llama.cpp"
-	// componentName is the component name.
-	componentName = "inference-" + Name
 )
 
-var (
-	// log is the log for the backend service.
-	log = logger.Default.WithComponent(componentName)
-	// serveLog is the log for llamaCppProcess
-	serveLog = logger.MakeFileOnly("", componentName)
-)
+// VendoredServerStoragePath returns the parent path of the vendored version of
+// com.docker.llama-server. It can be overridden during init().
+var VendoredServerStoragePath = func() (string, error) {
+	return ".", nil
+}
+
+// UpdatedServerStoragePath returns the parent path of the updated version of
+// com.docker.llama-server. It is also where updates will be stored when
+// downloaded. It can be overridden during init().
+var UpdatedServerStoragePath = func() (string, error) {
+	return ".", nil
+}
 
 // llamaCpp is the llama.cpp-based backend implementation.
 type llamaCpp struct {
+	// log is the associated logger.
+	log logging.Logger
 	// modelManager is the shared model manager.
-	modelManager    *models.Manager
+	modelManager *models.Manager
+	// serverLog is the logger to use for the llama.cpp server process.
+	serverLog       logging.Logger
 	updatedLlamaCpp bool
 }
 
 // New creates a new llama.cpp-based backend.
-func New(modelManager *models.Manager) (inference.Backend, error) {
-	return &llamaCpp{modelManager: modelManager}, nil
+func New(
+	log logging.Logger,
+	modelManager *models.Manager,
+	serverLog logging.Logger,
+) (inference.Backend, error) {
+	return &llamaCpp{
+		log:          log,
+		modelManager: modelManager,
+		serverLog:    serverLog,
+	}, nil
 }
 
 // Name implements inference.Backend.Name.
@@ -67,9 +82,13 @@ func (l *llamaCpp) Install(ctx context.Context, httpClient *http.Client) error {
 	// Internet access and an available docker/docker-model-backend-llamacpp:latest-update on Docker Hub are required.
 	// Even if docker/docker-model-backend-llamacpp:latest-update has been downloaded before, we still require its
 	// digest to be equal to the one on Docker Hub.
-	llamaCppPath := paths.DockerHome("bin", "inference", "com.docker.llama-server")
-	if err := ensureLatestLlamaCpp(ctx, httpClient, llamaCppPath); err != nil {
-		log.Infof("failed to ensure latest llama.cpp: %v\n", err)
+	llamaCppStorage, err := UpdatedServerStoragePath()
+	if err != nil {
+		return fmt.Errorf("unable to determine llama.cpp path: %w", err)
+	}
+	llamaCppPath := filepath.Join(llamaCppStorage, "com.docker.llama-server")
+	if err := ensureLatestLlamaCpp(ctx, l.log, httpClient, llamaCppPath); err != nil {
+		l.log.Infof("failed to ensure latest llama.cpp: %v\n", err)
 		if errors.Is(err, context.Canceled) {
 			return err
 		}
@@ -83,21 +102,24 @@ func (l *llamaCpp) Install(ctx context.Context, httpClient *http.Client) error {
 // Run implements inference.Backend.Run.
 func (l *llamaCpp) Run(ctx context.Context, socket, model string, mode inference.BackendMode) error {
 	modelPath, err := l.modelManager.GetModelPath(model)
-	log.Infof("Model path: %s", modelPath)
+	l.log.Infof("Model path: %s", modelPath)
 	if err != nil {
 		return fmt.Errorf("failed to get model path: %w", err)
 	}
 
 	if err := os.RemoveAll(socket); err != nil {
-		log.Warnln("failed to remove socket file %s: %w", socket, err)
-		log.Warnln("llama.cpp may not be able to start")
+		l.log.Warnln("failed to remove socket file %s: %w", socket, err)
+		l.log.Warnln("llama.cpp may not be able to start")
 	}
 
-	binPath := paths.DockerHome("bin", "inference")
+	binPath, err := UpdatedServerStoragePath()
+	if err != nil {
+		return fmt.Errorf("unable to determine llama.cpp path: %w", err)
+	}
 	if !l.updatedLlamaCpp {
-		binPath, err = paths.InstallPaths.BinResourcesPath()
+		binPath, err = VendoredServerStoragePath()
 		if err != nil {
-			return fmt.Errorf("failed to get llama.cpp path: %w", err)
+			return fmt.Errorf("unable to determine vendored llama.cpp path: %w", err)
 		}
 	}
 	llamaCppArgs := []string{"--model", modelPath, "--jinja"}
@@ -117,9 +139,9 @@ func (l *llamaCpp) Run(ctx context.Context, socket, model string, mode inference
 		// port this backend there.
 		return llamaCppProcess.Process.Signal(os.Interrupt)
 	}
-	serveLogStream := serveLog.Writer()
-	llamaCppProcess.Stdout = serveLogStream
-	llamaCppProcess.Stderr = serveLogStream
+	serverLogStream := l.serverLog.Writer()
+	llamaCppProcess.Stdout = serverLogStream
+	llamaCppProcess.Stderr = serverLogStream
 
 	if err := llamaCppProcess.Start(); err != nil {
 		return fmt.Errorf("unable to start llama.cpp: %w", err)
@@ -128,7 +150,7 @@ func (l *llamaCpp) Run(ctx context.Context, socket, model string, mode inference
 	llamaCppErrors := make(chan error, 1)
 	go func() {
 		llamaCppErr := llamaCppProcess.Wait()
-		serveLogStream.Close()
+		serverLogStream.Close()
 		llamaCppErrors <- llamaCppErr
 		close(llamaCppErrors)
 	}()

pkg/inference/backends/mlx/mlx.go

@@ -7,30 +7,28 @@ import (
 
 	"github.com/docker/model-runner/pkg/inference"
 	"github.com/docker/model-runner/pkg/inference/models"
-	"github.com/docker/model-runner/pkg/logger"
+	"github.com/docker/model-runner/pkg/logging"
 )
 
 const (
 	// Name is the backend name.
 	Name = "mlx"
-	// componentName is the component name.
-	componentName = "inference-" + Name
-)
-
-var (
-	// log is the log for the backend service.
-	log = logger.Default.WithComponent(componentName)
 )
 
 // mlx is the MLX-based backend implementation.
 type mlx struct {
+	// log is the associated logger.
+	log logging.Logger
 	// modelManager is the shared model manager.
 	modelManager *models.Manager
 }
 
 // New creates a new MLX-based backend.
-func New(modelManager *models.Manager) (inference.Backend, error) {
-	return &mlx{modelManager: modelManager}, nil
+func New(log logging.Logger, modelManager *models.Manager) (inference.Backend, error) {
+	return &mlx{
+		log:          log,
+		modelManager: modelManager,
+	}, nil
 }
 
 // Name implements inference.Backend.Name.
@@ -53,6 +51,6 @@ func (m *mlx) Install(ctx context.Context, httpClient *http.Client) error {
 // Run implements inference.Backend.Run.
 func (m *mlx) Run(ctx context.Context, socket, model string, mode inference.BackendMode) error {
 	// TODO: Implement.
-	log.Warn("MLX backend is not yet supported")
+	m.log.Warn("MLX backend is not yet supported")
 	return errors.New("not implemented")
 }

pkg/inference/backends/vllm/vllm.go

@@ -7,30 +7,28 @@ import (
 
 	"github.com/docker/model-runner/pkg/inference"
 	"github.com/docker/model-runner/pkg/inference/models"
-	"github.com/docker/model-runner/pkg/logger"
+	"github.com/docker/model-runner/pkg/logging"
 )
 
 const (
 	// Name is the backend name.
 	Name = "vllm"
-	// componentName is the component name.
-	componentName = "inference-" + Name
-)
-
-var (
-	// log is the log for the backend service.
-	log = logger.Default.WithComponent(componentName)
 )
 
 // vLLM is the vLLM-based backend implementation.
 type vLLM struct {
+	// log is the associated logger.
+	log logging.Logger
 	// modelManager is the shared model manager.
 	modelManager *models.Manager
 }
 
 // New creates a new vLLM-based backend.
-func New(modelManager *models.Manager) (inference.Backend, error) {
-	return &vLLM{modelManager: modelManager}, nil
+func New(log logging.Logger, modelManager *models.Manager) (inference.Backend, error) {
+	return &vLLM{
+		log:          log,
+		modelManager: modelManager,
+	}, nil
 }
 
 // Name implements inference.Backend.Name.
@@ -53,6 +51,6 @@ func (v *vLLM) Install(ctx context.Context, httpClient *http.Client) error {
 // Run implements inference.Backend.Run.
 func (v *vLLM) Run(ctx context.Context, socket, model string, mode inference.BackendMode) error {
 	// TODO: Implement.
-	log.Warn("vLLM backend is not yet supported")
+	v.log.Warn("vLLM backend is not yet supported")
 	return errors.New("not implemented")
 }

pkg/inference/models/manager.go

@@ -11,7 +11,7 @@ import (
 	"github.com/docker/model-distribution/pkg/distribution"
 	"github.com/docker/model-distribution/pkg/types"
 	"github.com/docker/model-runner/pkg/inference"
-	"github.com/docker/model-runner/pkg/logger"
+	"github.com/docker/model-runner/pkg/logging"
 )
 
 const (
@@ -23,7 +23,7 @@ const (
 // Manager manages inference model pulls and storage.
 type Manager struct {
 	// log is the associated logger.
-	log logger.ComponentLogger
+	log logging.Logger
 	// pullTokens is a semaphore used to restrict the maximum number of
 	// concurrent pull requests.
 	pullTokens chan struct{}
@@ -34,7 +34,7 @@ type Manager struct {
 }
 
 // NewManager creates a new model's manager.
-func NewManager(log logger.ComponentLogger, client *distribution.Client) *Manager {
+func NewManager(log logging.Logger, client *distribution.Client) *Manager {
 	// Create the manager.
 	m := &Manager{
 		log:                log,

pkg/inference/scheduling/installer.go

@@ -7,7 +7,7 @@ import (
 	"sync/atomic"
 
 	"github.com/docker/model-runner/pkg/inference"
-	"github.com/docker/model-runner/pkg/logger"
+	"github.com/docker/model-runner/pkg/logging"
 )
 
 var (
@@ -35,7 +35,7 @@ type installStatus struct {
 // installer drives backend installations.
 type installer struct {
 	// log is the associated logger.
-	log logger.ComponentLogger
+	log logging.Logger
 	// backends are the supported inference backends.
 	backends map[string]inference.Backend
 	// httpClient is the HTTP client to use for backend installations.
@@ -48,7 +48,7 @@ type installer struct {
 
 // newInstaller creates a new backend installer.
 func newInstaller(
-	log logger.ComponentLogger,
+	log logging.Logger,
 	backends map[string]inference.Backend,
 	httpClient *http.Client,
 ) *installer {

pkg/inference/scheduling/loader.go

@@ -9,7 +9,7 @@ import (
 
 	"github.com/docker/model-runner/pkg/inference"
 	"github.com/docker/model-runner/pkg/inference/models"
-	"github.com/docker/model-runner/pkg/logger"
+	"github.com/docker/model-runner/pkg/logging"
 )
 
 const (
@@ -48,7 +48,7 @@ type runnerKey struct {
 // been installed successfully.
 type loader struct {
 	// log is the associated logger.
-	log logger.ComponentLogger
+	log logging.Logger
 	// backends are the supported inference backends.
 	backends map[string]inference.Backend
 	// modelManager is the shared model manager.
@@ -86,7 +86,7 @@ type loader struct {
 
 // newLoader creates a new loader.
 func newLoader(
-	log logger.ComponentLogger,
+	log logging.Logger,
 	backends map[string]inference.Backend,
 	modelManager *models.Manager,
 ) *loader {