Merge pull request #307 from leo-pony/ascend_cann_support

ericcurtin · web-flow · commit a431268a6c9e · 2025-10-30T15:45:28.000Z
Add Ascend NPU CANN backend support
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
@@ -179,3 +179,18 @@ jobs:
           sbom: true
           provenance: mode=max
           tags: ${{ steps.tags.outputs.musa }}
+
+      - name: Build CANN image
+        uses: docker/build-push-action@ca052bb54ab0790a636c9b5f226502c73d547a25
+        with:
+          file: Dockerfile
+          target: final-llamacpp
+          platforms: linux/arm64, linux/amd64
+          build-args: |
+            "LLAMA_SERVER_VERSION=${{ inputs.llamaServerVersion }}"
+            "LLAMA_SERVER_VARIANT=cann"
+            "BASE_IMAGE=ascendai/cann:8.2.rc2-910b-ubuntu22.04-py3.11"
+          push: true
+          sbom: true
+          provenance: mode=max
+          tags: ${{ steps.tags.outputs.cann }}
diff --git a/Dockerfile b/Dockerfile
@@ -69,7 +69,7 @@ ENV MODEL_RUNNER_PORT=12434
 ENV LLAMA_SERVER_PATH=/app/bin
 ENV HOME=/home/modelrunner
 ENV MODELS_PATH=/models
-ENV LD_LIBRARY_PATH=/app/lib
+ENV LD_LIBRARY_PATH=/app/lib:$LD_LIBRARY_PATH
 
 # Label the image so that it's hidden on cloud engines.
 LABEL com.docker.desktop.service="model-runner"
diff --git a/README.md b/README.md
@@ -225,6 +225,7 @@ Available variants:
 - `cuda`: CUDA-accelerated version for NVIDIA GPUs
 - `rocm`: ROCm-accelerated version for AMD GPUs
 - `musa`: MUSA-accelerated version for MTHREADS GPUs
+- `cann`: CANN-accelerated version for Ascend NPUs
 
 The binary path in the image follows this pattern: `/com.docker.llama-server.native.linux.${LLAMA_SERVER_VARIANT}.${TARGETARCH}`
 
diff --git a/cmd/cli/commands/install-runner.go b/cmd/cli/commands/install-runner.go
@@ -253,6 +253,8 @@ func runInstallOrStart(cmd *cobra.Command, opts runnerOptions) error {
 		gpu = gpupkg.GPUSupportROCm
 	} else if opts.gpuMode == "musa" {
 		gpu = gpupkg.GPUSupportMUSA
+	} else if opts.gpuMode == "cann" {
+		gpu = gpupkg.GPUSupportCANN
 	} else if opts.gpuMode != "none" {
 		return fmt.Errorf("unknown GPU specification: %q", opts.gpuMode)
 	}
@@ -313,7 +315,7 @@ func newInstallRunner() *cobra.Command {
 	c.Flags().Uint16Var(&port, "port", 0,
 		"Docker container port for Docker Model Runner (default: 12434 for Docker Engine, 12435 for Cloud mode)")
 	c.Flags().StringVar(&host, "host", "127.0.0.1", "Host address to bind Docker Model Runner")
-	c.Flags().StringVar(&gpuMode, "gpu", "auto", "Specify GPU support (none|auto|cuda|rocm|musa)")
+	c.Flags().StringVar(&gpuMode, "gpu", "auto", "Specify GPU support (none|auto|cuda|rocm|musa|cann)")
 	c.Flags().StringVar(&backend, "backend", "", backendUsage)
 	c.Flags().BoolVar(&doNotTrack, "do-not-track", false, "Do not track models usage in Docker Model Runner")
 	return c
diff --git a/cmd/cli/commands/reinstall-runner.go b/cmd/cli/commands/reinstall-runner.go
@@ -30,7 +30,7 @@ func newReinstallRunner() *cobra.Command {
 	c.Flags().Uint16Var(&port, "port", 0,
 		"Docker container port for Docker Model Runner (default: 12434 for Docker Engine, 12435 for Cloud mode)")
 	c.Flags().StringVar(&host, "host", "127.0.0.1", "Host address to bind Docker Model Runner")
-	c.Flags().StringVar(&gpuMode, "gpu", "auto", "Specify GPU support (none|auto|cuda|musa)")
+	c.Flags().StringVar(&gpuMode, "gpu", "auto", "Specify GPU support (none|auto|cuda|musa|rocm|cann)")
 	c.Flags().StringVar(&backend, "backend", "", backendUsage)
 	c.Flags().BoolVar(&doNotTrack, "do-not-track", false, "Do not track models usage in Docker Model Runner")
 	return c
diff --git a/cmd/cli/commands/restart-runner.go b/cmd/cli/commands/restart-runner.go
@@ -36,7 +36,7 @@ func newRestartRunner() *cobra.Command {
 	c.Flags().Uint16Var(&port, "port", 0,
 		"Docker container port for Docker Model Runner (default: 12434 for Docker Engine, 12435 for Cloud mode)")
 	c.Flags().StringVar(&host, "host", "127.0.0.1", "Host address to bind Docker Model Runner")
-	c.Flags().StringVar(&gpuMode, "gpu", "auto", "Specify GPU support (none|auto|cuda|musa)")
+	c.Flags().StringVar(&gpuMode, "gpu", "auto", "Specify GPU support (none|auto|cuda|musa|rocm|cann)")
 	c.Flags().BoolVar(&doNotTrack, "do-not-track", false, "Do not track models usage in Docker Model Runner")
 	return c
 }
diff --git a/cmd/cli/commands/start-runner.go b/cmd/cli/commands/start-runner.go
@@ -26,7 +26,7 @@ func newStartRunner() *cobra.Command {
 	}
 	c.Flags().Uint16Var(&port, "port", 0,
 		"Docker container port for Docker Model Runner (default: 12434 for Docker Engine, 12435 for Cloud mode)")
-	c.Flags().StringVar(&gpuMode, "gpu", "auto", "Specify GPU support (none|auto|cuda|musa)")
+	c.Flags().StringVar(&gpuMode, "gpu", "auto", "Specify GPU support (none|auto|cuda|musa|rocm|cann)")
 	c.Flags().StringVar(&backend, "backend", "", backendUsage)
 	c.Flags().BoolVar(&doNotTrack, "do-not-track", false, "Do not track models usage in Docker Model Runner")
 	return c
diff --git a/cmd/cli/docs/reference/docker_model_install-runner.yaml b/cmd/cli/docs/reference/docker_model_install-runner.yaml
@@ -28,7 +28,7 @@ options:
     - option: gpu
       value_type: string
       default_value: auto
-      description: Specify GPU support (none|auto|cuda|rocm|musa)
+      description: Specify GPU support (none|auto|cuda|rocm|musa|cann)
       deprecated: false
       hidden: false
       experimental: false
diff --git a/cmd/cli/docs/reference/docker_model_reinstall-runner.yaml b/cmd/cli/docs/reference/docker_model_reinstall-runner.yaml
@@ -28,7 +28,7 @@ options:
     - option: gpu
       value_type: string
       default_value: auto
-      description: Specify GPU support (none|auto|cuda|musa)
+      description: Specify GPU support (none|auto|cuda|musa|rocm|cann)
       deprecated: false
       hidden: false
       experimental: false
diff --git a/cmd/cli/docs/reference/docker_model_restart-runner.yaml b/cmd/cli/docs/reference/docker_model_restart-runner.yaml
@@ -21,7 +21,7 @@ options:
     - option: gpu
       value_type: string
       default_value: auto
-      description: Specify GPU support (none|auto|cuda|musa)
+      description: Specify GPU support (none|auto|cuda|musa|rocm|cann)
       deprecated: false
       hidden: false
       experimental: false
diff --git a/cmd/cli/docs/reference/docker_model_start-runner.yaml b/cmd/cli/docs/reference/docker_model_start-runner.yaml
@@ -30,7 +30,7 @@ options:
     - option: gpu
       value_type: string
       default_value: auto
-      description: Specify GPU support (none|auto|cuda|musa)
+      description: Specify GPU support (none|auto|cuda|musa|rocm|cann)
       deprecated: false
       hidden: false
       experimental: false
diff --git a/cmd/cli/docs/reference/model_install-runner.md b/cmd/cli/docs/reference/model_install-runner.md
@@ -9,7 +9,7 @@ Install Docker Model Runner (Docker Engine only)
 |:-----------------|:---------|:------------|:-------------------------------------------------------------------------------------------------------|
 | `--backend`      | `string` |             | Specify backend (llama.cpp\|vllm). Default: llama.cpp                                                  |
 | `--do-not-track` | `bool`   |             | Do not track models usage in Docker Model Runner                                                       |
-| `--gpu`          | `string` | `auto`      | Specify GPU support (none\|auto\|cuda\|rocm\|musa)                                                     |
+| `--gpu`          | `string` | `auto`      | Specify GPU support (none\|auto\|cuda\|rocm\|musa\|cann)                                               |
 | `--host`         | `string` | `127.0.0.1` | Host address to bind Docker Model Runner                                                               |
 | `--port`         | `uint16` | `0`         | Docker container port for Docker Model Runner (default: 12434 for Docker Engine, 12435 for Cloud mode) |
 
diff --git a/cmd/cli/docs/reference/model_reinstall-runner.md b/cmd/cli/docs/reference/model_reinstall-runner.md
@@ -9,7 +9,7 @@ Reinstall Docker Model Runner (Docker Engine only)
 |:-----------------|:---------|:------------|:-------------------------------------------------------------------------------------------------------|
 | `--backend`      | `string` |             | Specify backend (llama.cpp\|vllm). Default: llama.cpp                                                  |
 | `--do-not-track` | `bool`   |             | Do not track models usage in Docker Model Runner                                                       |
-| `--gpu`          | `string` | `auto`      | Specify GPU support (none\|auto\|cuda\|musa)                                                           |
+| `--gpu`          | `string` | `auto`      | Specify GPU support (none\|auto\|cuda\|musa\|rocm\|cann)                                               |
 | `--host`         | `string` | `127.0.0.1` | Host address to bind Docker Model Runner                                                               |
 | `--port`         | `uint16` | `0`         | Docker container port for Docker Model Runner (default: 12434 for Docker Engine, 12435 for Cloud mode) |
 
diff --git a/cmd/cli/docs/reference/model_restart-runner.md b/cmd/cli/docs/reference/model_restart-runner.md
@@ -8,7 +8,7 @@ Restart Docker Model Runner (Docker Engine only)
 | Name             | Type     | Default     | Description                                                                                            |
 |:-----------------|:---------|:------------|:-------------------------------------------------------------------------------------------------------|
 | `--do-not-track` | `bool`   |             | Do not track models usage in Docker Model Runner                                                       |
-| `--gpu`          | `string` | `auto`      | Specify GPU support (none\|auto\|cuda\|musa)                                                           |
+| `--gpu`          | `string` | `auto`      | Specify GPU support (none\|auto\|cuda\|musa\|rocm\|cann)                                               |
 | `--host`         | `string` | `127.0.0.1` | Host address to bind Docker Model Runner                                                               |
 | `--port`         | `uint16` | `0`         | Docker container port for Docker Model Runner (default: 12434 for Docker Engine, 12435 for Cloud mode) |
 
diff --git a/cmd/cli/docs/reference/model_start-runner.md b/cmd/cli/docs/reference/model_start-runner.md
@@ -9,7 +9,7 @@ Start Docker Model Runner (Docker Engine only)
 |:-----------------|:---------|:--------|:-------------------------------------------------------------------------------------------------------|
 | `--backend`      | `string` |         | Specify backend (llama.cpp\|vllm). Default: llama.cpp                                                  |
 | `--do-not-track` | `bool`   |         | Do not track models usage in Docker Model Runner                                                       |
-| `--gpu`          | `string` | `auto`  | Specify GPU support (none\|auto\|cuda\|musa)                                                           |
+| `--gpu`          | `string` | `auto`  | Specify GPU support (none\|auto\|cuda\|musa\|rocm\|cann)                                               |
 | `--port`         | `uint16` | `0`     | Docker container port for Docker Model Runner (default: 12434 for Docker Engine, 12435 for Cloud mode) |
 
 
diff --git a/cmd/cli/pkg/gpu/gpu.go b/cmd/cli/pkg/gpu/gpu.go
@@ -19,6 +19,8 @@ const (
 	GPUSupportROCm
 	// GPUSupportMUSA indicates MUSA GPU support.
 	GPUSupportMUSA
+	// GPUSupportCANN indicates Ascend NPU support.
+	GPUSupportCANN
 )
 
 // ProbeGPUSupport determines whether or not the Docker engine has GPU support.
@@ -32,7 +34,10 @@ func ProbeGPUSupport(ctx context.Context, dockerClient client.SystemAPIClient) (
 	if hasMTHREADS, err := HasMTHREADSRuntime(ctx, dockerClient); err == nil && hasMTHREADS {
 		return GPUSupportMUSA, nil
 	}
-
+	// Check for CANN runtime first
+	if hasCANN, err := HasCANNRuntime(ctx, dockerClient); err == nil && hasCANN {
+		return GPUSupportCANN, nil
+	}
 	// Then search for nvidia-container-runtime on PATH
 	if _, err := exec.LookPath("nvidia-container-runtime"); err == nil {
 		return GPUSupportCUDA, nil
@@ -80,3 +85,13 @@ func HasMTHREADSRuntime(ctx context.Context, dockerClient client.SystemAPIClient
 	_, hasMTHREADS := info.Runtimes["mthreads"]
 	return hasMTHREADS, nil
 }
+
+// HasCANNRuntime determines whether there is a Ascend CANN runtime available
+func HasCANNRuntime(ctx context.Context, dockerClient client.SystemAPIClient) (bool, error) {
+	info, err := dockerClient.Info(ctx)
+	if err != nil {
+		return false, err
+	}
+	_, hasCANN := info.Runtimes["cann"]
+	return hasCANN, nil
+}
diff --git a/cmd/cli/pkg/standalone/containers.go b/cmd/cli/pkg/standalone/containers.go
@@ -232,6 +232,39 @@ func isRootless(ctx context.Context, dockerClient *client.Client) bool {
 	return false
 }
 
+// Check whether the host Ascend driver path exists. If so, create the corresponding mount configuration.
+func tryGetBindAscendMounts() []mount.Mount {
+	hostPaths := []string{
+		"/usr/local/dcmi",
+		"/usr/local/bin/npu-smi",
+		"/usr/local/Ascend/driver/lib64",
+		"/usr/local/Ascend/driver/version.info",
+	}
+
+	var newMounts []mount.Mount
+	for _, hostPath := range hostPaths {
+		matches, err := filepath.Glob(hostPath)
+		if err != nil {
+			fmt.Errorf("Error checking glob pattern for %s: %v", hostPath, err)
+			continue
+		}
+
+		if len(matches) > 0 {
+			newMount := mount.Mount{
+				Type:     mount.TypeBind,
+				Source:   hostPath,
+				Target:   hostPath,
+				ReadOnly: true,
+			}
+			newMounts = append(newMounts, newMount)
+		} else {
+			fmt.Printf("  [NOT FOUND] Ascend driver path does not exist, skipping: %s\n", hostPath)
+		}
+	}
+
+	return newMounts
+}
+
 // CreateControllerContainer creates and starts a controller container.
 func CreateControllerContainer(ctx context.Context, dockerClient *client.Client, port uint16, host string, environment string, doNotTrack bool, gpu gpupkg.GPUSupport, backend string, modelStorageVolume string, printer StatusPrinter, engineKind types.ModelRunnerEngineKind) error {
 	imageName := controllerImageName(gpu, backend)
@@ -276,6 +309,11 @@ func CreateControllerContainer(ctx context.Context, dockerClient *client.Client,
 			Name: "always",
 		},
 	}
+	ascendMounts := tryGetBindAscendMounts()
+	if len(ascendMounts) > 0 {
+		hostConfig.Mounts = append(hostConfig.Mounts, ascendMounts...)
+	}
+
 	portBindings := []nat.PortBinding{{HostIP: host, HostPort: portStr}}
 	if os.Getenv("_MODEL_RUNNER_TREAT_DESKTOP_AS_MOBY") != "1" {
 		// Don't bind the bridge gateway IP if we're treating Docker Desktop as Moby.
@@ -303,6 +341,10 @@ func CreateControllerContainer(ctx context.Context, dockerClient *client.Client,
 		if ok, err := gpupkg.HasMTHREADSRuntime(ctx, dockerClient); err == nil && ok {
 			hostConfig.Runtime = "mthreads"
 		}
+	} else if gpu == gpupkg.GPUSupportCANN {
+		if ok, err := gpupkg.HasCANNRuntime(ctx, dockerClient); err == nil && ok {
+			hostConfig.Runtime = "cann"
+		}
 	}
 
 	// devicePaths contains glob patterns for common AI accelerator device files.
diff --git a/cmd/cli/pkg/standalone/controller_image.go b/cmd/cli/pkg/standalone/controller_image.go
@@ -40,6 +40,8 @@ func controllerImageVariant(detectedGPU gpupkg.GPUSupport, backend string) strin
 		return "rocm"
 	case gpupkg.GPUSupportMUSA:
 		return "musa"
+	case gpupkg.GPUSupportCANN:
+		return "cann"
 	default:
 		return ""
 	}
diff --git a/llamacpp/native/cann.Dockerfile b/llamacpp/native/cann.Dockerfile
@@ -1,12 +1,14 @@
 # syntax=docker/dockerfile:1
 
-ARG CANN_VERSION=8.0.0-910b
-ARG CANN_IMAGE_VARIANT=openeuler22.03
+ARG CANN_VERSION=8.2.rc2-910b
+ARG CANN_IMAGE_VARIANT=ubuntu22.04
+ARG ASCEND_SOC_TYPE=Ascend910B3
 
-FROM quay.io/ascend/cann:{CANN_VERSION}-{CANN_IMAGE_VARIANT}-py3.10 AS builder
+FROM quay.io/ascend/cann:${CANN_VERSION}-${CANN_IMAGE_VARIANT}-py3.11 AS builder
 
 ARG TARGETARCH
 ARG CANN_IMAGE_VARIANT
+ARG ASCEND_SOC_TYPE
 
 RUN apt-get update && apt-get install -y cmake ninja-build git build-essential curl
 
@@ -25,25 +27,40 @@ RUN echo "-B build \
     -DCMAKE_BUILD_TYPE=Release \
     -DBUILD_SHARED_LIBS=ON \
     -DGGML_BACKEND_DL=ON \
-    -DGGML_CPU_ALL_VARIANTS=ON \
     -DGGML_NATIVE=OFF \
     -DGGML_OPENMP=OFF \
     -DGGML_CANN=ON \
     -DLLAMA_CURL=OFF \
+    -DSOC_TYPE=${ASCEND_SOC_TYPE} \
     -GNinja \
     -S ." > cmake-flags
+
 RUN cmake $(cat cmake-flags)
-RUN cmake --build build --config Release
-RUN cmake --install build --config Release --prefix install
+
+RUN --mount=type=cache,target=/root/.ccache \
+    cann_in_sys_path=/usr/local/Ascend/ascend-toolkit; \
+    cann_in_user_path=$HOME/Ascend/ascend-toolkit; \
+    uname_m=$(uname -m) && \
+    if [ -f "${cann_in_sys_path}/set_env.sh" ]; then \
+        source ${cann_in_sys_path}/set_env.sh; \
+        export LD_LIBRARY_PATH=${cann_in_sys_path}/latest/lib64:${cann_in_sys_path}/latest/${uname_m}-linux/devlib:${LD_LIBRARY_PATH} ; \
+    elif [ -f "${cann_in_user_path}/set_env.sh" ]; then \
+        source "$HOME/Ascend/ascend-toolkit/set_env.sh"; \
+        export LD_LIBRARY_PATH=${cann_in_user_path}/latest/lib64:${cann_in_user_path}/latest/${uname_m}-linux/devlib:${LD_LIBRARY_PATH}; \ 
+    else \
+        echo "No Ascend Toolkit found"; \
+        exit 1; \
+    fi && \
+    cmake --build build --config Release && \
+    cmake --install build --config Release --prefix install
 
 RUN rm install/bin/*.py
 RUN rm -r install/lib/cmake
 RUN rm -r install/lib/pkgconfig
 RUN rm -r install/include
 
-FROM scratch AS final
-
+FROM quay.io/ascend/cann:${CANN_VERSION}-${CANN_IMAGE_VARIANT}-py3.11 AS final
 ARG TARGETARCH
 ARG CANN_VERSION
 
-COPY --from=builder /llama-server/install /com.docker.llama-server.native.linux.cann$CANN_VERSION.$TARGETARCH
+COPY --from=builder /llama-server/install /com.docker.llama-server.native.linux.cann.${TARGETARCH}
diff --git a/scripts/docker-run.sh b/scripts/docker-run.sh
@@ -17,6 +17,10 @@ add_accelerators() {
   render_gid=$(set +o pipefail; command getent group render 2>/dev/null | cut -d: -f3)
   if [[ -n "$render_gid" ]]; then
     args+=("--group-add" "$render_gid")
+    if [ -e "/dev/davinci_manager" ]; then
+      # ascend driver accessing group id is 1000(HwHiAiUser)
+      args+=("--group-add" "$(getent group HwHiAiUser | cut -d: -f3)")
+    fi
   fi
 }
 
@@ -29,6 +33,12 @@ add_optional_args() {
     args+=(-v "$MODELS_PATH:/models" -e MODELS_PATH=/models)
   fi
 
+  for i in /usr/local/dcmi /usr/local/bin/npu-smi /usr/local/Ascend/driver/lib64/ /usr/local/Ascend/driver/version.info /etc/ascend_install.info; do
+    if [ -e "$i" ]; then
+      args+=(-v "$i:$i")
+    fi
+  done
+
   if [ -n "${LLAMA_ARGS-}" ]; then
     args+=(-e "LLAMA_ARGS=$LLAMA_ARGS")
   fi

Original file line number	Diff line number	Diff line change
`@@ -36,7 +36,7 @@ func newRestartRunner() *cobra.Command {`
`36`	`36`	`c.Flags().Uint16Var(&port, "port", 0,`
`37`	`37`	`"Docker container port for Docker Model Runner (default: 12434 for Docker Engine, 12435 for Cloud mode)")`
`38`	`38`	`c.Flags().StringVar(&host, "host", "127.0.0.1", "Host address to bind Docker Model Runner")`
`39`		`- c.Flags().StringVar(&gpuMode, "gpu", "auto", "Specify GPU support (none\|auto\|cuda\|musa)")`
	`39`	`+ c.Flags().StringVar(&gpuMode, "gpu", "auto", "Specify GPU support (none\|auto\|cuda\|musa\|rocm\|cann)")`
`40`	`40`	`c.Flags().BoolVar(&doNotTrack, "do-not-track", false, "Do not track models usage in Docker Model Runner")`
`41`	`41`	`return c`
`42`	`42`	`}`
Original file line number	Diff line number	Diff line change
`@@ -26,7 +26,7 @@ func newStartRunner() *cobra.Command {`
`26`	`26`	`}`
`27`	`27`	`c.Flags().Uint16Var(&port, "port", 0,`
`28`	`28`	`"Docker container port for Docker Model Runner (default: 12434 for Docker Engine, 12435 for Cloud mode)")`
`29`		`- c.Flags().StringVar(&gpuMode, "gpu", "auto", "Specify GPU support (none\|auto\|cuda\|musa)")`
	`29`	`+ c.Flags().StringVar(&gpuMode, "gpu", "auto", "Specify GPU support (none\|auto\|cuda\|musa\|rocm\|cann)")`
`30`	`30`	`c.Flags().StringVar(&backend, "backend", "", backendUsage)`
`31`	`31`	`c.Flags().BoolVar(&doNotTrack, "do-not-track", false, "Do not track models usage in Docker Model Runner")`
`32`	`32`	`return c`