diff --git a/go.mod b/go.mod index f94d738b6..82fa47f70 100644 --- a/go.mod +++ b/go.mod @@ -5,7 +5,7 @@ go 1.23.7 require ( github.com/containerd/containerd/v2 v2.0.4 github.com/containerd/platforms v1.0.0-rc.1 - github.com/docker/model-distribution v0.0.0-20250627163720-aff34abcf3e0 + github.com/docker/model-distribution v0.0.0-20250710064845-ad2903dc93f7 github.com/google/go-containerregistry v0.20.3 github.com/jaypipes/ghw v0.16.0 github.com/mattn/go-shellwords v1.0.12 diff --git a/go.sum b/go.sum index e851c5df8..7f1f35c0f 100644 --- a/go.sum +++ b/go.sum @@ -38,8 +38,8 @@ github.com/docker/distribution v2.8.3+incompatible h1:AtKxIZ36LoNK51+Z6RpzLpddBi github.com/docker/distribution v2.8.3+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= github.com/docker/docker-credential-helpers v0.8.2 h1:bX3YxiGzFP5sOXWc3bTPEXdEaZSeVMrFgOr3T+zrFAo= github.com/docker/docker-credential-helpers v0.8.2/go.mod h1:P3ci7E3lwkZg6XiHdRKft1KckHiO9a2rNtyFbZ/ry9M= -github.com/docker/model-distribution v0.0.0-20250627163720-aff34abcf3e0 h1:bve4JZI06Admw+NewtPfrpJXsvRnGKTQvBOEICNC1C0= -github.com/docker/model-distribution v0.0.0-20250627163720-aff34abcf3e0/go.mod h1:dThpO9JoG5Px3i+rTluAeZcqLGw8C0qepuEL4gL2o/c= +github.com/docker/model-distribution v0.0.0-20250710064845-ad2903dc93f7 h1:Kkiv0E3LTbj2y3rPRIJtRf89Eahg372RxqvKV14zszo= +github.com/docker/model-distribution v0.0.0-20250710064845-ad2903dc93f7/go.mod h1:dThpO9JoG5Px3i+rTluAeZcqLGw8C0qepuEL4gL2o/c= github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= diff --git a/pkg/inference/backends/llamacpp/llamacpp_config.go b/pkg/inference/backends/llamacpp/llamacpp_config.go index cd4668c5f..5c8822d3a 100644 --- a/pkg/inference/backends/llamacpp/llamacpp_config.go +++ b/pkg/inference/backends/llamacpp/llamacpp_config.go @@ -70,6 +70,12 @@ func (c *Config) GetArgs(model types.Model, socket string, mode inference.Backen args = append(args, config.RuntimeFlags...) } + // Add arguments for Multimodal projector + path, err := model.MMPROJPath() + if path != "" && err == nil { + args = append(args, "--mmproj", path) + } + return args, nil } diff --git a/pkg/inference/backends/llamacpp/llamacpp_config_test.go b/pkg/inference/backends/llamacpp/llamacpp_config_test.go index 6856981dc..4b145ca6b 100644 --- a/pkg/inference/backends/llamacpp/llamacpp_config_test.go +++ b/pkg/inference/backends/llamacpp/llamacpp_config_test.go @@ -1,6 +1,7 @@ package llamacpp import ( + "errors" "runtime" "strconv" "testing" @@ -251,6 +252,10 @@ type fakeModel struct { config types.Config } +func (f *fakeModel) MMPROJPath() (string, error) { + return "", errors.New("not found") +} + func (f *fakeModel) ID() (string, error) { panic("shouldn't be called") } diff --git a/pkg/inference/models/manager.go b/pkg/inference/models/manager.go index c87efa831..5c176f309 100644 --- a/pkg/inference/models/manager.go +++ b/pkg/inference/models/manager.go @@ -333,7 +333,7 @@ func (m *Manager) handleDeleteModel(w http.ResponseWriter, r *http.Request) { } } - if err := m.distributionClient.DeleteModel(r.PathValue("name"), force); err != nil { + if _, err := m.distributionClient.DeleteModel(r.PathValue("name"), force); err != nil { if errors.Is(err, distribution.ErrModelNotFound) { http.Error(w, err.Error(), http.StatusNotFound) return