Add CANN backend support

leo-pony · leo-pony · commit a997f1964ac9 · 2025-10-24T17:43:55.000+08:00
Signed-off-by: leo-pony &lt;nengjunma@outlook.com&gt;
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
@@ -146,3 +146,18 @@ jobs:
           sbom: true
           provenance: mode=max
           tags: ${{ steps.tags.outputs.rocm }}
+
+      - name: Build CANN image
+        uses: docker/build-push-action@ca052bb54ab0790a636c9b5f226502c73d547a25
+        with:
+          file: Dockerfile
+          target: final-llamacpp
+          platforms: linux/arm64
+          build-args: |
+            "LLAMA_SERVER_VERSION=${{ inputs.llamaServerVersion }}"
+            "LLAMA_SERVER_VARIANT=cann"
+            "BASE_IMAGE=ascendai/cann:8.2.rc1-910b-ubuntu22.04-py3.11"
+          push: true
+          sbom: true
+          provenance: mode=max
+          tags: ${{ steps.tags.outputs.cann }}
diff --git a/cmd/cli/commands/install-runner.go b/cmd/cli/commands/install-runner.go
@@ -247,6 +247,8 @@ func runInstallOrStart(cmd *cobra.Command, opts runnerOptions) error {
 		gpu = gpupkg.GPUSupportCUDA
 	} else if opts.gpuMode == "rocm" {
 		gpu = gpupkg.GPUSupportROCm
+	} else if opts.gpuMode == "cann" {
+		gpu = gpupkg.GPUSupportCANN
 	} else if opts.gpuMode != "none" {
 		return fmt.Errorf("unknown GPU specification: %q", opts.gpuMode)
 	}
@@ -295,7 +297,7 @@ func newInstallRunner() *cobra.Command {
 	c.Flags().Uint16Var(&port, "port", 0,
 		"Docker container port for Docker Model Runner (default: 12434 for Docker Engine, 12435 for Cloud mode)")
 	c.Flags().StringVar(&host, "host", "127.0.0.1", "Host address to bind Docker Model Runner")
-	c.Flags().StringVar(&gpuMode, "gpu", "auto", "Specify GPU support (none|auto|cuda|rocm)")
+	c.Flags().StringVar(&gpuMode, "gpu", "auto", "Specify GPU support (none|auto|cuda|rocm|cann)")
 	c.Flags().BoolVar(&doNotTrack, "do-not-track", false, "Do not track models usage in Docker Model Runner")
 	return c
 }
diff --git a/cmd/cli/commands/reinstall-runner.go b/cmd/cli/commands/reinstall-runner.go
@@ -28,7 +28,7 @@ func newReinstallRunner() *cobra.Command {
 	c.Flags().Uint16Var(&port, "port", 0,
 		"Docker container port for Docker Model Runner (default: 12434 for Docker Engine, 12435 for Cloud mode)")
 	c.Flags().StringVar(&host, "host", "127.0.0.1", "Host address to bind Docker Model Runner")
-	c.Flags().StringVar(&gpuMode, "gpu", "auto", "Specify GPU support (none|auto|cuda)")
+	c.Flags().StringVar(&gpuMode, "gpu", "auto", "Specify GPU support (none|auto|cuda|rocm|cann)")
 	c.Flags().BoolVar(&doNotTrack, "do-not-track", false, "Do not track models usage in Docker Model Runner")
 	return c
 }
diff --git a/cmd/cli/commands/restart-runner.go b/cmd/cli/commands/restart-runner.go
@@ -36,7 +36,7 @@ func newRestartRunner() *cobra.Command {
 	c.Flags().Uint16Var(&port, "port", 0,
 		"Docker container port for Docker Model Runner (default: 12434 for Docker Engine, 12435 for Cloud mode)")
 	c.Flags().StringVar(&host, "host", "127.0.0.1", "Host address to bind Docker Model Runner")
-	c.Flags().StringVar(&gpuMode, "gpu", "auto", "Specify GPU support (none|auto|cuda)")
+	c.Flags().StringVar(&gpuMode, "gpu", "auto", "Specify GPU support (none|auto|cuda|rocm|cann)")
 	c.Flags().BoolVar(&doNotTrack, "do-not-track", false, "Do not track models usage in Docker Model Runner")
 	return c
 }
diff --git a/cmd/cli/commands/start-runner.go b/cmd/cli/commands/start-runner.go
@@ -24,7 +24,7 @@ func newStartRunner() *cobra.Command {
 	}
 	c.Flags().Uint16Var(&port, "port", 0,
 		"Docker container port for Docker Model Runner (default: 12434 for Docker Engine, 12435 for Cloud mode)")
-	c.Flags().StringVar(&gpuMode, "gpu", "auto", "Specify GPU support (none|auto|cuda)")
+	c.Flags().StringVar(&gpuMode, "gpu", "auto", "Specify GPU support (none|auto|cuda|rocm|cann)")
 	c.Flags().BoolVar(&doNotTrack, "do-not-track", false, "Do not track models usage in Docker Model Runner")
 	return c
 }
diff --git a/cmd/cli/docs/reference/docker_model_install-runner.yaml b/cmd/cli/docs/reference/docker_model_install-runner.yaml
@@ -19,7 +19,7 @@ options:
     - option: gpu
       value_type: string
       default_value: auto
-      description: Specify GPU support (none|auto|cuda|rocm)
+      description: Specify GPU support (none|auto|cuda|rocm|cann)
       deprecated: false
       hidden: false
       experimental: false
diff --git a/cmd/cli/docs/reference/docker_model_reinstall-runner.yaml b/cmd/cli/docs/reference/docker_model_reinstall-runner.yaml
@@ -19,7 +19,7 @@ options:
     - option: gpu
       value_type: string
       default_value: auto
-      description: Specify GPU support (none|auto|cuda)
+      description: Specify GPU support (none|auto|cuda|rocm|cann)
       deprecated: false
       hidden: false
       experimental: false
diff --git a/cmd/cli/docs/reference/docker_model_restart-runner.yaml b/cmd/cli/docs/reference/docker_model_restart-runner.yaml
@@ -21,7 +21,7 @@ options:
     - option: gpu
       value_type: string
       default_value: auto
-      description: Specify GPU support (none|auto|cuda)
+      description: Specify GPU support (none|auto|cuda|rocm|cann)
       deprecated: false
       hidden: false
       experimental: false
diff --git a/cmd/cli/docs/reference/docker_model_start-runner.yaml b/cmd/cli/docs/reference/docker_model_start-runner.yaml
@@ -21,7 +21,7 @@ options:
     - option: gpu
       value_type: string
       default_value: auto
-      description: Specify GPU support (none|auto|cuda)
+      description: Specify GPU support (none|auto|cuda|rocm|cann)
       deprecated: false
       hidden: false
       experimental: false
diff --git a/cmd/cli/docs/reference/model_install-runner.md b/cmd/cli/docs/reference/model_install-runner.md
@@ -8,7 +8,7 @@ Install Docker Model Runner (Docker Engine only)
 | Name             | Type     | Default     | Description                                                                                            |
 |:-----------------|:---------|:------------|:-------------------------------------------------------------------------------------------------------|
 | `--do-not-track` | `bool`   |             | Do not track models usage in Docker Model Runner                                                       |
-| `--gpu`          | `string` | `auto`      | Specify GPU support (none\|auto\|cuda\|rocm)                                                           |
+| `--gpu`          | `string` | `auto`      | Specify GPU support (none\|auto\|cuda\|rocm\|cann)                                                           |
 | `--host`         | `string` | `127.0.0.1` | Host address to bind Docker Model Runner                                                               |
 | `--port`         | `uint16` | `0`         | Docker container port for Docker Model Runner (default: 12434 for Docker Engine, 12435 for Cloud mode) |
 
diff --git a/cmd/cli/docs/reference/model_reinstall-runner.md b/cmd/cli/docs/reference/model_reinstall-runner.md
@@ -8,7 +8,7 @@ Reinstall Docker Model Runner (Docker Engine only)
 | Name             | Type     | Default     | Description                                                                                            |
 |:-----------------|:---------|:------------|:-------------------------------------------------------------------------------------------------------|
 | `--do-not-track` | `bool`   |             | Do not track models usage in Docker Model Runner                                                       |
-| `--gpu`          | `string` | `auto`      | Specify GPU support (none\|auto\|cuda)                                                                 |
+| `--gpu`          | `string` | `auto`      | Specify GPU support (none\|auto\|cuda\|cann)                                                                 |
 | `--host`         | `string` | `127.0.0.1` | Host address to bind Docker Model Runner                                                               |
 | `--port`         | `uint16` | `0`         | Docker container port for Docker Model Runner (default: 12434 for Docker Engine, 12435 for Cloud mode) |
 
diff --git a/cmd/cli/docs/reference/model_restart-runner.md b/cmd/cli/docs/reference/model_restart-runner.md
@@ -8,7 +8,7 @@ Restart Docker Model Runner (Docker Engine only)
 | Name             | Type     | Default     | Description                                                                                            |
 |:-----------------|:---------|:------------|:-------------------------------------------------------------------------------------------------------|
 | `--do-not-track` | `bool`   |             | Do not track models usage in Docker Model Runner                                                       |
-| `--gpu`          | `string` | `auto`      | Specify GPU support (none\|auto\|cuda)                                                                 |
+| `--gpu`          | `string` | `auto`      | Specify GPU support (none\|auto\|cuda\|cann)                                                                 |
 | `--host`         | `string` | `127.0.0.1` | Host address to bind Docker Model Runner                                                               |
 | `--port`         | `uint16` | `0`         | Docker container port for Docker Model Runner (default: 12434 for Docker Engine, 12435 for Cloud mode) |
 
diff --git a/cmd/cli/docs/reference/model_start-runner.md b/cmd/cli/docs/reference/model_start-runner.md
@@ -8,7 +8,7 @@ Start Docker Model Runner (Docker Engine only)
 | Name             | Type     | Default | Description                                                                                            |
 |:-----------------|:---------|:--------|:-------------------------------------------------------------------------------------------------------|
 | `--do-not-track` | `bool`   |         | Do not track models usage in Docker Model Runner                                                       |
-| `--gpu`          | `string` | `auto`  | Specify GPU support (none\|auto\|cuda)                                                                 |
+| `--gpu`          | `string` | `auto`  | Specify GPU support (none\|auto\|cuda\|cann)                                                                 |
 | `--port`         | `uint16` | `0`     | Docker container port for Docker Model Runner (default: 12434 for Docker Engine, 12435 for Cloud mode) |
 
 
diff --git a/cmd/cli/pkg/gpu/gpu.go b/cmd/cli/pkg/gpu/gpu.go
@@ -17,6 +17,8 @@ const (
 	GPUSupportCUDA
 	// GPUSupportROCm indicates ROCm GPU support.
 	GPUSupportROCm
+	// GPUSupportCANN indicates Ascend NPU support.
+	GPUSupportCANN
 )
 
 // ProbeGPUSupport determines whether or not the Docker engine has GPU support.
@@ -25,7 +27,10 @@ func ProbeGPUSupport(ctx context.Context, dockerClient client.SystemAPIClient) (
 	if hasROCm, err := HasROCmRuntime(ctx, dockerClient); err == nil && hasROCm {
 		return GPUSupportROCm, nil
 	}
-
+	// Check for CANN runtime first
+	if hasCANN, err := HasCANNRuntime(ctx, dockerClient); err == nil && hasCANN {
+		return GPUSupportCANN, nil
+	}
 	// Then search for nvidia-container-runtime on PATH
 	if _, err := exec.LookPath("nvidia-container-runtime"); err == nil {
 		return GPUSupportCUDA, nil
@@ -63,3 +68,13 @@ func HasROCmRuntime(ctx context.Context, dockerClient client.SystemAPIClient) (b
 	_, hasROCm := info.Runtimes["rocm"]
 	return hasROCm, nil
 }
+
+// HasCANNRuntime determines whether there is a Ascend CANN runtime available
+func HasCANNRuntime(ctx context.Context, dockerClient client.SystemAPIClient) (bool, error) {
+	info, err := dockerClient.Info(ctx)
+	if err != nil {
+		return false, err
+	}
+	_, hasCANN := info.Runtimes["cann"]
+	return hasCANN, nil
+}
diff --git a/cmd/cli/pkg/standalone/containers.go b/cmd/cli/pkg/standalone/containers.go
@@ -217,6 +217,39 @@ func ensureContainerStarted(ctx context.Context, dockerClient client.ContainerAP
 	return errors.New("timed out")
 }
 
+// Check whether the host Ascend driver path exists. If so, create the corresponding mount configuration.
+func tryGetBindAscendMounts() []mount.Mount {
+	hostPaths := []string{
+		"/usr/local/dcmi",
+		"/usr/local/bin/npu-smi",
+		"/usr/local/Ascend/driver/lib64",
+		"/usr/local/Ascend/driver/version.info",
+	}
+
+	var newMounts []mount.Mount
+	for _, hostPath := range hostPaths {
+		matches, err := filepath.Glob(hostPath)
+		if err != nil {
+			fmt.Errorf("Error checking glob pattern for %s: %v", hostPath, err)
+			continue
+		}
+
+		if len(matches) > 0 {
+			newMount := mount.Mount{
+				Type:     mount.TypeBind,
+				Source:   hostPath,
+				Target:   hostPath,
+				ReadOnly: true,
+			}
+			newMounts = append(newMounts, newMount)
+		} else {
+			fmt.Printf("  [NOT FOUND] Ascend driver path does not exist, skipping: %s\n", hostPath)
+		}
+	}
+
+	return newMounts
+}
+
 // CreateControllerContainer creates and starts a controller container.
 func CreateControllerContainer(ctx context.Context, dockerClient *client.Client, port uint16, host string, environment string, doNotTrack bool, gpu gpupkg.GPUSupport, modelStorageVolume string, printer StatusPrinter, engineKind types.ModelRunnerEngineKind) error {
 	imageName := controllerImageName(gpu)
@@ -261,6 +294,11 @@ func CreateControllerContainer(ctx context.Context, dockerClient *client.Client,
 			Name: "always",
 		},
 	}
+	ascendMounts := tryGetBindAscendMounts()
+	if len(ascendMounts) > 0 {
+		hostConfig.Mounts = append(hostConfig.Mounts, ascendMounts...)
+	}
+
 	portBindings := []nat.PortBinding{{HostIP: host, HostPort: portStr}}
 	if os.Getenv("_MODEL_RUNNER_TREAT_DESKTOP_AS_MOBY") != "1" {
 		// Don't bind the bridge gateway IP if we're treating Docker Desktop as Moby.
@@ -284,6 +322,11 @@ func CreateControllerContainer(ctx context.Context, dockerClient *client.Client,
 			hostConfig.Runtime = "rocm"
 		}
 		// ROCm devices are handled via device paths (/dev/kfd, /dev/dri) which are already added below
+	} else if gpu == gpupkg.GPUSupportCANN {
+		if ok, err := gpupkg.HasCANNRuntime(ctx, dockerClient); err == nil && ok {
+			hostConfig.Runtime = "cann"
+		}
+		// ToDo: Ascend device
 	}
 
 	// devicePaths contains glob patterns for common AI accelerator device files.
diff --git a/cmd/cli/pkg/standalone/controller_image.go b/cmd/cli/pkg/standalone/controller_image.go
@@ -32,6 +32,8 @@ func controllerImageVariant(detectedGPU gpupkg.GPUSupport) string {
 		return "cuda"
 	case gpupkg.GPUSupportROCm:
 		return "rocm"
+	case gpupkg.GPUSupportCANN:
+		return "cann"
 	default:
 		return ""
 	}

Original file line number	Diff line number	Diff line change
`@@ -28,7 +28,7 @@ func newReinstallRunner() *cobra.Command {`
`28`	`28`	`c.Flags().Uint16Var(&port, "port", 0,`
`29`	`29`	`"Docker container port for Docker Model Runner (default: 12434 for Docker Engine, 12435 for Cloud mode)")`
`30`	`30`	`c.Flags().StringVar(&host, "host", "127.0.0.1", "Host address to bind Docker Model Runner")`
`31`		`- c.Flags().StringVar(&gpuMode, "gpu", "auto", "Specify GPU support (none\|auto\|cuda)")`
	`31`	`+ c.Flags().StringVar(&gpuMode, "gpu", "auto", "Specify GPU support (none\|auto\|cuda\|rocm\|cann)")`
`32`	`32`	`c.Flags().BoolVar(&doNotTrack, "do-not-track", false, "Do not track models usage in Docker Model Runner")`
`33`	`33`	`return c`
`34`	`34`	`}`
Original file line number	Diff line number	Diff line change
`@@ -36,7 +36,7 @@ func newRestartRunner() *cobra.Command {`
`36`	`36`	`c.Flags().Uint16Var(&port, "port", 0,`
`37`	`37`	`"Docker container port for Docker Model Runner (default: 12434 for Docker Engine, 12435 for Cloud mode)")`
`38`	`38`	`c.Flags().StringVar(&host, "host", "127.0.0.1", "Host address to bind Docker Model Runner")`
`39`		`- c.Flags().StringVar(&gpuMode, "gpu", "auto", "Specify GPU support (none\|auto\|cuda)")`
	`39`	`+ c.Flags().StringVar(&gpuMode, "gpu", "auto", "Specify GPU support (none\|auto\|cuda\|rocm\|cann)")`
`40`	`40`	`c.Flags().BoolVar(&doNotTrack, "do-not-track", false, "Do not track models usage in Docker Model Runner")`
`41`	`41`	`return c`
`42`	`42`	`}`
Original file line number	Diff line number	Diff line change
`@@ -24,7 +24,7 @@ func newStartRunner() *cobra.Command {`
`24`	`24`	`}`
`25`	`25`	`c.Flags().Uint16Var(&port, "port", 0,`
`26`	`26`	`"Docker container port for Docker Model Runner (default: 12434 for Docker Engine, 12435 for Cloud mode)")`
`27`		`- c.Flags().StringVar(&gpuMode, "gpu", "auto", "Specify GPU support (none\|auto\|cuda)")`
	`27`	`+ c.Flags().StringVar(&gpuMode, "gpu", "auto", "Specify GPU support (none\|auto\|cuda\|rocm\|cann)")`
`28`	`28`	`c.Flags().BoolVar(&doNotTrack, "do-not-track", false, "Do not track models usage in Docker Model Runner")`
`29`	`29`	`return c`
`30`	`30`	`}`