Skip to content

Commit 6b1cfee

Browse files
xenoscopicxenoscopic
authored
Merge pull request #123 from docker/nicks/chart
charts: add Kubernetes examples
2 parents ecfa5e7 + b42f3a0 commit 6b1cfee

File tree

14 files changed

+983
-0
lines changed

14 files changed

+983
-0
lines changed

‎README.md‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -144,3 +144,11 @@ curl http://localhost:8080/metrics
144144
- **Monitoring integration**: Add the endpoint to your Prometheus configuration
145145

146146
Check [METRICS.md](./METRICS.md) for more details.
147+
148+
## Kubernetes
149+
150+
Experimental support for running in Kubernetes is available
151+
in the form of [a Helm chart and static YAML](charts/docker-model-runner/README).
152+
153+
If you are interested in a specific Kubernetes use-case, please start a
154+
discussion on the issue tracker.

‎charts/docker-model-runner/CONTRIBUTING.md‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
# Contributing
2+
3+
4+
## Using the Makefile
5+
6+
```bash
7+
# Render to plain Kubernetes YAML
8+
make render
9+
10+
# Install the chart
11+
make install
12+
13+
# Upgrade the chart
14+
make upgrade
15+
16+
# Uninstall the chart
17+
make uninstall
18+
```

‎charts/docker-model-runner/Chart.yaml‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
apiVersion: v2
2+
name: docker-model-runner
3+
description: A Helm chart for Docker Model Runner
4+
type: application
5+
version: 0.1.0
6+
appVersion: "latest"

‎charts/docker-model-runner/Makefile‎

Lines changed: 54 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,54 @@
1+
.PHONY: render clean install upgrade uninstall template lint package help
2+
3+
CHART_NAME := docker-model-runner
4+
RELEASE_NAME := docker-model-runner
5+
NAMESPACE := default
6+
7+
render:
8+
@echo "Rendering Helm chart to plain Kubernetes YAML..."
9+
mkdir -p static
10+
helm template $(RELEASE_NAME) . --namespace $(NAMESPACE) > static/docker-model-runner.yaml
11+
helm template $(RELEASE_NAME) . --namespace $(NAMESPACE) --set "nodePort.enabled=true" > static/docker-model-runner-desktop.yaml
12+
helm template $(RELEASE_NAME) . --namespace $(NAMESPACE) --set "modelInit.enabled=true" --set "modelInit.models[0]=ai/smollm2:latest" > static/docker-model-runner-smollm2.yaml
13+
helm template $(RELEASE_NAME) . --namespace $(NAMESPACE) --set "storage.storageClass=gp2" > static/docker-model-runner-eks.yaml
14+
@echo "Rendered YAML saved to static"
15+
16+
clean:
17+
@echo "Cleaning up rendered files..."
18+
rm -fR static
19+
20+
install:
21+
@echo "Installing Helm chart..."
22+
helm install $(RELEASE_NAME) . --namespace $(NAMESPACE) --create-namespace
23+
24+
upgrade:
25+
@echo "Upgrading Helm chart..."
26+
helm upgrade $(RELEASE_NAME) . --namespace $(NAMESPACE)
27+
28+
uninstall:
29+
@echo "Uninstalling Helm chart..."
30+
helm uninstall $(RELEASE_NAME) --namespace $(NAMESPACE)
31+
32+
template:
33+
@echo "Templating Helm chart..."
34+
helm template $(RELEASE_NAME) . --namespace $(NAMESPACE)
35+
36+
lint:
37+
@echo "Linting Helm chart..."
38+
helm lint .
39+
40+
package:
41+
@echo "Packaging Helm chart..."
42+
helm package .
43+
44+
help:
45+
@echo "Available targets:"
46+
@echo " render - Render Helm chart to plain Kubernetes YAML (saves to rendered.yaml)"
47+
@echo " template - Template Helm chart (output to stdout)"
48+
@echo " lint - Lint Helm chart"
49+
@echo " package - Package Helm chart"
50+
@echo " install - Install Helm chart"
51+
@echo " upgrade - Upgrade Helm chart"
52+
@echo " uninstall - Uninstall Helm chart"
53+
@echo " clean - Clean up rendered files"
54+
@echo " help - Show this help message"

‎charts/docker-model-runner/README.md‎

Lines changed: 175 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,175 @@
1+
# Docker Model Runner Kubernetes Support
2+
3+
Manifests for deploying Docker Model Runner on Kubernetes with ephemeral storage, GPU support, and model pre-pulling capabilities.
4+
5+
## Quickstart
6+
7+
### On Docker Desktop
8+
9+
```
10+
kubectl apply -f static/docker-model-runner-desktop.yaml
11+
kubectl wait --for=condition=Available deployment/docker-model-runner --timeout=5m
12+
MODEL_RUNNER_HOST=http://localhost:31245 docker model run ai/smollm2:latest
13+
```
14+
15+
### On any Kubernetes Cluster
16+
17+
```
18+
kubectl apply -f static/docker-model-runner.yaml
19+
kubectl wait --for=condition=Available deployment/docker-model-runner --timeout=5m
20+
kubectl port-forward deployment/docker-model-runner 31245:12434
21+
```
22+
23+
Then:
24+
25+
```
26+
MODEL_RUNNER_HOST=http://localhost:31245 docker model run ai/smollm2:latest
27+
```
28+
29+
## Helm Configuration
30+
31+
### Basic Configuration
32+
33+
Key configuration options in `values.yaml`:
34+
35+
```yaml
36+
# Storage configuration
37+
storage:
38+
size: 100Gi
39+
storageClass: "" # Set this to the storage class of your cloud provider.
40+
41+
# Model pre-pull configuration
42+
modelInit:
43+
enabled: false
44+
models:
45+
- "ai/smollm2:latest"
46+
47+
# GPU configuration
48+
gpu:
49+
enabled: false
50+
vendor: nvidia # or amd
51+
count: 1
52+
53+
# NodePort configuration
54+
nodePort:
55+
enabled: false
56+
port: 31245
57+
```
58+
59+
### GPU Scheduling
60+
61+
To enable GPU scheduling:
62+
63+
```yaml
64+
gpu:
65+
enabled: true
66+
vendor: nvidia # or amd
67+
count: 1
68+
```
69+
70+
This will add the appropriate resource requests/limits:
71+
- NVIDIA: `nvidia.com/gpu`
72+
- AMD: `amd.com/gpu`
73+
74+
### Model Pre-pulling
75+
76+
Configure models to pre-pull during pod initialization:
77+
78+
```yaml
79+
modelInit:
80+
enabled: true
81+
models:
82+
- "ai/smollm2:latest"
83+
- "ai/llama3.2:latest"
84+
- "ai/mistral:latest"
85+
```
86+
87+
## Usage
88+
89+
### Testing the Installation
90+
91+
Once installed, set up a port-forward to access the service:
92+
93+
```bash
94+
kubectl port-forward service/docker-model-runner-nodeport 31245:80
95+
```
96+
97+
Then test the model runner:
98+
99+
```bash
100+
MODEL_RUNNER_HOST=http://localhost:31245 docker model run ai/smollm2:latest
101+
```
102+
103+
### Using with Open WebUI
104+
105+
To use Docker Model Runner with Open WebUI, install the Open WebUI Helm chart:
106+
107+
```bash
108+
# Add the Open WebUI Helm repository
109+
helm repo add open-webui https://helm.openwebui.com/
110+
helm repo update
111+
112+
# Install Open WebUI with auth diabled
113+
# See the open-webui Helm chart for
114+
# connecting to your auth provider.
115+
helm upgrade --install --wait open-webui open-webui/open-webui \
116+
--set ollama.enabled=false \
117+
--set pipelines.enabled=false \
118+
--set extraEnvVars[0].name="WEBUI_AUTH" \
119+
--set-string extraEnvVars[0].value=false \
120+
--set openaiBaseApiUrl="http://docker-model-runner/engines/v1"
121+
```
122+
123+
Access Open WebUI:
124+
125+
```bash
126+
kubectl port-forward service/open-webui 8080:80
127+
```
128+
129+
Then visit http://localhost:8080 in your browser.
130+
131+
## Values Reference
132+
133+
| Parameter | Description | Default |
134+
|-----------|-------------|---------|
135+
| `replicaCount` | Number of replicas | `1` |
136+
| `image.repository` | Docker Model Runner image repository | `docker/model-runner` |
137+
| `image.tag` | Docker Model Runner image tag | `latest` |
138+
| `image.pullPolicy` | Image pull policy | `IfNotPresent` |
139+
| `storage.size` | Ephemeral volume size | `100Gi` |
140+
| `storage.storageClass` | Storage class for ephemeral volume | `""` |
141+
| `modelInit.enabled` | Enable model pre-pulling | `false` |
142+
| `modelInit.models` | List of models to pre-pull | `["ai/smollm2:latest"]` |
143+
| `gpu.enabled` | Enable GPU support | `false` |
144+
| `gpu.vendor` | GPU vendor (nvidia or amd) | `nvidia` |
145+
| `gpu.count` | Number of GPUs to request | `1` |
146+
| `nodePort.enabled` | Enable NodePort service | `false` |
147+
| `nodePort.port` | NodePort port number | `31245` |
148+
149+
## Troubleshooting
150+
151+
### Pod Fails to Start
152+
153+
Check the pod logs:
154+
155+
```bash
156+
kubectl logs -f deployment/docker-model-runner
157+
```
158+
159+
### Model Pre-pull Issues
160+
161+
Check the init container logs:
162+
163+
```bash
164+
kubectl logs -f deployment/docker-model-runner -c model-init
165+
```
166+
167+
### GPU Not Available
168+
169+
Your cluster must use [a GPU scheduling plugin](https://kubernetes.io/docs/tasks/manage-gpus/scheduling-gpus/).
170+
171+
Ensure your cluster has GPU support and the appropriate device plugin installed:
172+
173+
- For NVIDIA GPUs: Install the [NVIDIA device plugin](https://github.com/NVIDIA/k8s-device-plugin)
174+
- For AMD GPUs: Install the [AMD device plugin](https://github.com/ROCm/k8s-device-plugin#deployment)
175+

‎charts/docker-model-runner/static/docker-model-runner-desktop.yaml‎

Lines changed: 114 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,114 @@
1+
---
2+
# Source: docker-model-runner/templates/service.yaml
3+
apiVersion: v1
4+
kind: Service
5+
metadata:
6+
name: docker-model-runner
7+
labels:
8+
helm.sh/chart: docker-model-runner-0.1.0
9+
app.kubernetes.io/name: docker-model-runner
10+
app.kubernetes.io/instance: docker-model-runner
11+
app.kubernetes.io/version: "latest"
12+
app.kubernetes.io/managed-by: Helm
13+
spec:
14+
type: ClusterIP
15+
ports:
16+
- port: 80
17+
targetPort: 12434
18+
protocol: TCP
19+
name: http
20+
selector:
21+
app.kubernetes.io/name: docker-model-runner
22+
app.kubernetes.io/instance: docker-model-runner
23+
---
24+
# Source: docker-model-runner/templates/service.yaml
25+
apiVersion: v1
26+
kind: Service
27+
metadata:
28+
name: docker-model-runner-nodeport
29+
labels:
30+
helm.sh/chart: docker-model-runner-0.1.0
31+
app.kubernetes.io/name: docker-model-runner
32+
app.kubernetes.io/instance: docker-model-runner
33+
app.kubernetes.io/version: "latest"
34+
app.kubernetes.io/managed-by: Helm
35+
spec:
36+
type: NodePort
37+
ports:
38+
- port: 80
39+
targetPort: 12434
40+
nodePort: 31245
41+
protocol: TCP
42+
name: http
43+
selector:
44+
app.kubernetes.io/name: docker-model-runner
45+
app.kubernetes.io/instance: docker-model-runner
46+
---
47+
# Source: docker-model-runner/templates/deployment.yaml
48+
apiVersion: apps/v1
49+
kind: Deployment
50+
metadata:
51+
name: docker-model-runner
52+
labels:
53+
helm.sh/chart: docker-model-runner-0.1.0
54+
app.kubernetes.io/name: docker-model-runner
55+
app.kubernetes.io/instance: docker-model-runner
56+
app.kubernetes.io/version: "latest"
57+
app.kubernetes.io/managed-by: Helm
58+
spec:
59+
replicas: 1
60+
selector:
61+
matchLabels:
62+
app.kubernetes.io/name: docker-model-runner
63+
app.kubernetes.io/instance: docker-model-runner
64+
template:
65+
metadata:
66+
labels:
67+
app.kubernetes.io/name: docker-model-runner
68+
app.kubernetes.io/instance: docker-model-runner
69+
spec:
70+
initContainers:
71+
- name: changeowner
72+
image: busybox
73+
command: ["sh", "-c", "chmod a+rwx /models"]
74+
volumeMounts:
75+
- name: model-storage
76+
mountPath: /models
77+
78+
containers:
79+
- name: model-runner
80+
image: "docker/model-runner:latest"
81+
imagePullPolicy: IfNotPresent
82+
ports:
83+
- containerPort: 12434
84+
env:
85+
- name: DMR_ORIGINS
86+
value: "http://localhost:31245,http://localhost:12434"
87+
volumeMounts:
88+
- name: model-storage
89+
mountPath: /models
90+
securityContext:
91+
allowPrivilegeEscalation: false
92+
readinessProbe:
93+
httpGet:
94+
path: /engines/status
95+
port: 12434
96+
initialDelaySeconds: 5
97+
periodSeconds: 10
98+
failureThreshold: 3
99+
livenessProbe:
100+
httpGet:
101+
path: /engines/status
102+
port: 12434
103+
initialDelaySeconds: 15
104+
periodSeconds: 20
105+
failureThreshold: 3
106+
volumes:
107+
- name: model-storage
108+
ephemeral:
109+
volumeClaimTemplate:
110+
spec:
111+
accessModes: [ "ReadWriteOnce" ]
112+
resources:
113+
requests:
114+
storage: 100Gi

0 commit comments

Comments
 (0)