Skip to content

Commit a70a6c2

Browse files
committed
Add CANN backend support
Signed-off-by: leo-pony <nengjunma@outlook.com>
1 parent bed4533 commit a70a6c2

21 files changed

+130
-24
lines changed

.github/workflows/release.yml

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -148,3 +148,18 @@ jobs:
148148
sbom: true
149149
provenance: mode=max
150150
tags: ${{ steps.tags.outputs.rocm }}
151+
152+
- name: Build CANN image
153+
uses: docker/build-push-action@ca052bb54ab0790a636c9b5f226502c73d547a25
154+
with:
155+
file: Dockerfile
156+
target: final-llamacpp
157+
platforms: linux/arm64, linux/amd64
158+
build-args: |
159+
"LLAMA_SERVER_VERSION=${{ inputs.llamaServerVersion }}"
160+
"LLAMA_SERVER_VARIANT=cann"
161+
"BASE_IMAGE=ascendai/cann:8.2.rc2-910b-ubuntu22.04-py3.11"
162+
push: true
163+
sbom: true
164+
provenance: mode=max
165+
tags: ${{ steps.tags.outputs.cann }}

Dockerfile

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -69,7 +69,7 @@ ENV MODEL_RUNNER_PORT=12434
6969
ENV LLAMA_SERVER_PATH=/app/bin
7070
ENV HOME=/home/modelrunner
7171
ENV MODELS_PATH=/models
72-
ENV LD_LIBRARY_PATH=/app/lib
72+
ENV LD_LIBRARY_PATH=/app/lib:$LD_LIBRARY_PATH
7373

7474
# Label the image so that it's hidden on cloud engines.
7575
LABEL com.docker.desktop.service="model-runner"

README.md

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -224,6 +224,7 @@ Available variants:
224224
- `cpu`: CPU-optimized version
225225
- `cuda`: CUDA-accelerated version for NVIDIA GPUs
226226
- `rocm`: ROCm-accelerated version for AMD GPUs
227+
- `cann`: CANN-accelerated version for Ascend NPUs
227228

228229
The binary path in the image follows this pattern: `/com.docker.llama-server.native.linux.${LLAMA_SERVER_VARIANT}.${TARGETARCH}`
229230

cmd/cli/commands/install-runner.go

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -247,6 +247,8 @@ func runInstallOrStart(cmd *cobra.Command, opts runnerOptions) error {
247247
gpu = gpupkg.GPUSupportCUDA
248248
} else if opts.gpuMode == "rocm" {
249249
gpu = gpupkg.GPUSupportROCm
250+
} else if opts.gpuMode == "cann" {
251+
gpu = gpupkg.GPUSupportCANN
250252
} else if opts.gpuMode != "none" {
251253
return fmt.Errorf("unknown GPU specification: %q", opts.gpuMode)
252254
}
@@ -295,7 +297,7 @@ func newInstallRunner() *cobra.Command {
295297
c.Flags().Uint16Var(&port, "port", 0,
296298
"Docker container port for Docker Model Runner (default: 12434 for Docker Engine, 12435 for Cloud mode)")
297299
c.Flags().StringVar(&host, "host", "127.0.0.1", "Host address to bind Docker Model Runner")
298-
c.Flags().StringVar(&gpuMode, "gpu", "auto", "Specify GPU support (none|auto|cuda|rocm)")
300+
c.Flags().StringVar(&gpuMode, "gpu", "auto", "Specify GPU support (none|auto|cuda|rocm|cann)")
299301
c.Flags().BoolVar(&doNotTrack, "do-not-track", false, "Do not track models usage in Docker Model Runner")
300302
return c
301303
}

cmd/cli/commands/reinstall-runner.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,7 @@ func newReinstallRunner() *cobra.Command {
2828
c.Flags().Uint16Var(&port, "port", 0,
2929
"Docker container port for Docker Model Runner (default: 12434 for Docker Engine, 12435 for Cloud mode)")
3030
c.Flags().StringVar(&host, "host", "127.0.0.1", "Host address to bind Docker Model Runner")
31-
c.Flags().StringVar(&gpuMode, "gpu", "auto", "Specify GPU support (none|auto|cuda)")
31+
c.Flags().StringVar(&gpuMode, "gpu", "auto", "Specify GPU support (none|auto|cuda|rocm|cann)")
3232
c.Flags().BoolVar(&doNotTrack, "do-not-track", false, "Do not track models usage in Docker Model Runner")
3333
return c
3434
}

cmd/cli/commands/restart-runner.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -36,7 +36,7 @@ func newRestartRunner() *cobra.Command {
3636
c.Flags().Uint16Var(&port, "port", 0,
3737
"Docker container port for Docker Model Runner (default: 12434 for Docker Engine, 12435 for Cloud mode)")
3838
c.Flags().StringVar(&host, "host", "127.0.0.1", "Host address to bind Docker Model Runner")
39-
c.Flags().StringVar(&gpuMode, "gpu", "auto", "Specify GPU support (none|auto|cuda)")
39+
c.Flags().StringVar(&gpuMode, "gpu", "auto", "Specify GPU support (none|auto|cuda|rocm|cann)")
4040
c.Flags().BoolVar(&doNotTrack, "do-not-track", false, "Do not track models usage in Docker Model Runner")
4141
return c
4242
}

cmd/cli/commands/start-runner.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,7 @@ func newStartRunner() *cobra.Command {
2424
}
2525
c.Flags().Uint16Var(&port, "port", 0,
2626
"Docker container port for Docker Model Runner (default: 12434 for Docker Engine, 12435 for Cloud mode)")
27-
c.Flags().StringVar(&gpuMode, "gpu", "auto", "Specify GPU support (none|auto|cuda)")
27+
c.Flags().StringVar(&gpuMode, "gpu", "auto", "Specify GPU support (none|auto|cuda|rocm|cann)")
2828
c.Flags().BoolVar(&doNotTrack, "do-not-track", false, "Do not track models usage in Docker Model Runner")
2929
return c
3030
}

cmd/cli/docs/reference/docker_model_install-runner.yaml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,7 @@ options:
1919
- option: gpu
2020
value_type: string
2121
default_value: auto
22-
description: Specify GPU support (none|auto|cuda|rocm)
22+
description: Specify GPU support (none|auto|cuda|rocm|cann)
2323
deprecated: false
2424
hidden: false
2525
experimental: false

cmd/cli/docs/reference/docker_model_reinstall-runner.yaml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,7 @@ options:
1919
- option: gpu
2020
value_type: string
2121
default_value: auto
22-
description: Specify GPU support (none|auto|cuda)
22+
description: Specify GPU support (none|auto|cuda|rocm|cann)
2323
deprecated: false
2424
hidden: false
2525
experimental: false

cmd/cli/docs/reference/docker_model_restart-runner.yaml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,7 @@ options:
2121
- option: gpu
2222
value_type: string
2323
default_value: auto
24-
description: Specify GPU support (none|auto|cuda)
24+
description: Specify GPU support (none|auto|cuda|rocm|cann)
2525
deprecated: false
2626
hidden: false
2727
experimental: false

0 commit comments

Comments
 (0)