Skip to content

Commit 7d5932e

Browse files
xenoscopicxenoscopic
committed
Merge branch 'main' into openai-passthrough
2 parents d370bbd + a8437d3 commit 7d5932e

File tree

11 files changed

+356
-123
lines changed

11 files changed

+356
-123
lines changed

‎.github/workflows/dmr-daily-check.yml‎

Lines changed: 148 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,148 @@
1+
name: Daily Docker Model Runner Health Check
2+
3+
permissions:
4+
contents: read
5+
6+
on:
7+
schedule:
8+
# Run daily at 6 AM UTC
9+
- cron: '0 6 * * *'
10+
workflow_dispatch:
11+
inputs:
12+
test_model:
13+
description: 'Model to test with (default: ai/smollm2:360M-Q4_K_M)'
14+
required: false
15+
type: string
16+
default: 'ai/smollm2:360M-Q4_K_M'
17+
18+
jobs:
19+
dmr-health-check:
20+
runs-on: ubuntu-latest
21+
timeout-minutes: 30
22+
23+
steps:
24+
- name: Set up Docker
25+
uses: docker/setup-docker-action@v4
26+
27+
- name: Install docker-model-plugin
28+
run: |
29+
echo "Installing docker-model-plugin..."
30+
# Add Docker's official GPG key:
31+
sudo apt-get update
32+
sudo apt-get install ca-certificates curl
33+
sudo install -m 0755 -d /etc/apt/keyrings
34+
sudo curl -fsSL https://download.docker.com/linux/ubuntu/gpg -o /etc/apt/keyrings/docker.asc
35+
sudo chmod a+r /etc/apt/keyrings/docker.asc
36+
37+
# Add the repository to Apt sources:
38+
echo \
39+
"deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.asc] https://download.docker.com/linux/ubuntu \
40+
$(. /etc/os-release && echo "${UBUNTU_CODENAME:-$VERSION_CODENAME}") stable" | \
41+
sudo tee /etc/apt/sources.list.d/docker.list > /dev/null
42+
sudo apt-get update
43+
sudo apt-get install -y docker-model-plugin
44+
45+
echo "Installation completed successfully"
46+
47+
- name: Test docker model version
48+
run: |
49+
echo "Testing docker model version command..."
50+
sudo docker model version
51+
52+
# Verify the command returns successfully
53+
if [ $? -eq 0 ]; then
54+
echo "✅ docker model version command works correctly"
55+
else
56+
echo "❌ docker model version command failed"
57+
exit 1
58+
fi
59+
60+
- name: Test model pull and run
61+
run: |
62+
MODEL="${{ github.event.inputs.test_model || 'ai/smollm2:360M-Q4_K_M' }}"
63+
echo "Testing with model: $MODEL"
64+
65+
# Test model pull
66+
echo "Pulling model..."
67+
sudo docker model pull "$MODEL"
68+
69+
if [ $? -eq 0 ]; then
70+
echo "✅ Model pull successful"
71+
else
72+
echo "❌ Model pull failed"
73+
exit 1
74+
fi
75+
76+
# Test basic model run (with timeout to avoid hanging)
77+
echo "Testing docker model run..."
78+
timeout 60s sudo docker model run "$MODEL" "Give me a fact about whales." || {
79+
exit_code=$?
80+
if [ $exit_code -eq 124 ]; then
81+
echo "✅ Model run test completed (timed out as expected for non-interactive test)"
82+
else
83+
echo "❌ Model run failed with exit code: $exit_code"
84+
exit 1
85+
fi
86+
}
87+
88+
- name: Test API endpoint
89+
run: |
90+
MODEL="${{ github.event.inputs.test_model || 'ai/smollm2:360M-Q4_K_M' }}"
91+
echo "Testing API endpoint with model: $MODEL"
92+
93+
# Test API call with curl
94+
echo "Testing API call..."
95+
RESPONSE=$(curl -s http://localhost:12434/engines/llama.cpp/v1/chat/completions \
96+
-H "Content-Type: application/json" \
97+
-d "{
98+
\"model\": \"$MODEL\",
99+
\"messages\": [
100+
{
101+
\"role\": \"user\",
102+
\"content\": \"Say hello\"
103+
}
104+
],
105+
\"top_k\": 1,
106+
\"temperature\": 0
107+
}")
108+
109+
if [ $? -eq 0 ]; then
110+
echo "✅ API call successful"
111+
echo "Response received: $RESPONSE"
112+
113+
# Check if response contains "hello" (case-insensitive)
114+
if echo "$RESPONSE" | grep -qi "hello"; then
115+
echo "✅ Response contains 'hello' (case-insensitive)"
116+
else
117+
echo "❌ Response does not contain 'hello'"
118+
echo "Full response: $RESPONSE"
119+
exit 1
120+
fi
121+
else
122+
echo "❌ API call failed"
123+
exit 1
124+
fi
125+
126+
- name: Test model cleanup
127+
run: |
128+
MODEL="${{ github.event.inputs.test_model || 'ai/smollm2:360M-Q4_K_M' }}"
129+
130+
echo "Cleaning up test model..."
131+
sudo docker model rm "$MODEL" || echo "Model removal failed or model not found"
132+
133+
# Verify model was removed
134+
echo "Verifying model cleanup..."
135+
sudo docker model ls
136+
137+
echo "✅ Model cleanup completed"
138+
139+
- name: Report success
140+
if: success()
141+
run: |
142+
echo "🎉 Docker Model Runner daily health check completed successfully!"
143+
echo "All tests passed:"
144+
echo " ✅ docker-model-plugin installation successful"
145+
echo " ✅ docker model version command working"
146+
echo " ✅ Model pull and run operations successful"
147+
echo " ✅ API endpoint operations successful"
148+
echo " ✅ Cleanup operations successful"

‎go.mod‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@ go 1.23.7
55
require (
66
github.com/containerd/containerd/v2 v2.0.4
77
github.com/containerd/platforms v1.0.0-rc.1
8-
github.com/docker/model-distribution v0.0.0-20250627163720-aff34abcf3e0
8+
github.com/docker/model-distribution v0.0.0-20250710123110-a633223e127e
99
github.com/google/go-containerregistry v0.20.3
1010
github.com/jaypipes/ghw v0.16.0
1111
github.com/mattn/go-shellwords v1.0.12

‎go.sum‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -38,8 +38,8 @@ github.com/docker/distribution v2.8.3+incompatible h1:AtKxIZ36LoNK51+Z6RpzLpddBi
3838
github.com/docker/distribution v2.8.3+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w=
3939
github.com/docker/docker-credential-helpers v0.8.2 h1:bX3YxiGzFP5sOXWc3bTPEXdEaZSeVMrFgOr3T+zrFAo=
4040
github.com/docker/docker-credential-helpers v0.8.2/go.mod h1:P3ci7E3lwkZg6XiHdRKft1KckHiO9a2rNtyFbZ/ry9M=
41-
github.com/docker/model-distribution v0.0.0-20250627163720-aff34abcf3e0 h1:bve4JZI06Admw+NewtPfrpJXsvRnGKTQvBOEICNC1C0=
42-
github.com/docker/model-distribution v0.0.0-20250627163720-aff34abcf3e0/go.mod h1:dThpO9JoG5Px3i+rTluAeZcqLGw8C0qepuEL4gL2o/c=
41+
github.com/docker/model-distribution v0.0.0-20250710123110-a633223e127e h1:qBkjP4A20f3RXvtstitIPiStQ4p+bK8xcjosrXLBQZ0=
42+
github.com/docker/model-distribution v0.0.0-20250710123110-a633223e127e/go.mod h1:dThpO9JoG5Px3i+rTluAeZcqLGw8C0qepuEL4gL2o/c=
4343
github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg=
4444
github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U=
4545
github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=

‎pkg/inference/backends/llamacpp/llamacpp_config.go‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -70,6 +70,12 @@ func (c *Config) GetArgs(model types.Model, socket string, mode inference.Backen
7070
args = append(args, config.RuntimeFlags...)
7171
}
7272

73+
// Add arguments for Multimodal projector
74+
path, err := model.MMPROJPath()
75+
if path != "" && err == nil {
76+
args = append(args, "--mmproj", path)
77+
}
78+
7379
return args, nil
7480
}
7581

‎pkg/inference/backends/llamacpp/llamacpp_config_test.go‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
package llamacpp
22

33
import (
4+
"errors"
45
"runtime"
56
"strconv"
67
"testing"
@@ -251,6 +252,10 @@ type fakeModel struct {
251252
config types.Config
252253
}
253254

255+
func (f *fakeModel) MMPROJPath() (string, error) {
256+
return "", errors.New("not found")
257+
}
258+
254259
func (f *fakeModel) ID() (string, error) {
255260
panic("shouldn't be called")
256261
}

‎pkg/inference/models/manager.go‎

Lines changed: 28 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -252,6 +252,27 @@ func (m *Manager) handleGetModel(w http.ResponseWriter, r *http.Request) {
252252
}
253253
}
254254

255+
// ResolveModelID resolves a model reference to a model ID. If resolution fails, it returns the original ref.
256+
func (m *Manager) ResolveModelID(modelRef string) string {
257+
// Sanitize modelRef to prevent log forgery
258+
sanitizedModelRef := strings.ReplaceAll(modelRef, "\n", "")
259+
sanitizedModelRef = strings.ReplaceAll(sanitizedModelRef, "\r", "")
260+
261+
model, err := m.GetModel(sanitizedModelRef)
262+
if err != nil {
263+
m.log.Warnf("Failed to resolve model ref %s to ID: %v", sanitizedModelRef, err)
264+
return sanitizedModelRef
265+
}
266+
267+
modelID, err := model.ID()
268+
if err != nil {
269+
m.log.Warnf("Failed to get model ID for ref %s: %v", sanitizedModelRef, err)
270+
return sanitizedModelRef
271+
}
272+
273+
return modelID
274+
}
275+
255276
func getLocalModel(m *Manager, name string) (*Model, error) {
256277
if m.distributionClient == nil {
257278
return nil, errors.New("model distribution service unavailable")
@@ -329,7 +350,8 @@ func (m *Manager) handleDeleteModel(w http.ResponseWriter, r *http.Request) {
329350
}
330351
}
331352

332-
if err := m.distributionClient.DeleteModel(r.PathValue("name"), force); err != nil {
353+
resp, err := m.distributionClient.DeleteModel(r.PathValue("name"), force)
354+
if err != nil {
333355
if errors.Is(err, distribution.ErrModelNotFound) {
334356
http.Error(w, err.Error(), http.StatusNotFound)
335357
return
@@ -342,6 +364,11 @@ func (m *Manager) handleDeleteModel(w http.ResponseWriter, r *http.Request) {
342364
http.Error(w, err.Error(), http.StatusInternalServerError)
343365
return
344366
}
367+
368+
w.Header().Set("Content-Type", "application/json")
369+
if err := json.NewEncoder(w).Encode(resp); err != nil {
370+
http.Error(w, fmt.Sprintf("error writing response: %v", err), http.StatusInternalServerError)
371+
}
345372
}
346373

347374
// HandleOpenAIGetModels handles GET <inference-prefix>/<backend>/v1/models and

0 commit comments

Comments
 (0)