diff --git a/README.md b/README.md index e9e3cb2b4..8d61edb03 100644 --- a/README.md +++ b/README.md @@ -247,6 +247,92 @@ curl http://localhost:8080/metrics Check [METRICS.md](./METRICS.md) for more details. +## ollama Integration + +Docker Model Runner supports running ollama as an alternative runner. This allows you to use ollama's model format and API alongside Docker Model Runner. + +### Installing ollama Runner + +To install the ollama runner instead of the default Docker Model Runner: + +```bash +docker model install-runner --ollama +``` + +This will: +- Start an ollama container on port 11434 (the standard ollama port) +- Create an `ollama` volume for model storage (instead of `docker-model-runner-models`) +- Use the `ollama/ollama:latest` image + +### GPU Support for ollama + +ollama supports both NVIDIA CUDA and AMD ROCm GPUs: + +```bash +# For AMD GPUs with ROCm +docker model install-runner --ollama --gpu rocm + +# For NVIDIA GPUs (auto-detected) +docker model install-runner --ollama --gpu auto +``` + +The `--gpu rocm` flag will use the `ollama/ollama:rocm` image which includes ROCm support. + +### Managing ollama Runner + +All standard runner commands support the `--ollama` flag: + +```bash +# Start ollama runner +docker model start-runner --ollama + +# Stop ollama runner +docker model stop-runner --ollama + +# Restart ollama runner +docker model restart-runner --ollama + +# Reinstall ollama runner +docker model reinstall-runner --ollama + +# Uninstall ollama runner (optionally remove images) +docker model uninstall-runner --ollama --images +``` + +### Using ollama Models + +Models from `ollama.com` are automatically detected: + +```bash +# Pull an ollama model (will auto-start ollama runner) +docker model pull ollama.com/library/smollm:135m + +# Run an ollama model +docker model run ollama.com/library/smollm:135m +``` + +**Note:** Direct ollama API integration is currently in development. For now, you can interact with ollama models using: + +```bash +# Pull a model +docker exec docker-ollama-runner ollama pull smollm:135m + +# Run a model interactively +docker exec -it docker-ollama-runner ollama run smollm:135m + +# List ollama models +docker exec docker-ollama-runner ollama list +``` + +### ollama vs Docker Model Runner + +Key differences: + +- **Port**: ollama uses port 11434, Docker Model Runner uses 12434 (Docker Engine) or 12435 (Cloud) +- **Volume**: ollama uses the `ollama` volume, Docker Model Runner uses `docker-model-runner-models` +- **Image**: ollama uses `ollama/ollama:latest` or `ollama/ollama:rocm` +- **Platform support**: ollama controller containers are used on all platforms (including Docker Desktop) + ## Kubernetes Experimental support for running in Kubernetes is available diff --git a/cmd/cli/commands/install-runner.go b/cmd/cli/commands/install-runner.go index 9f9998f0b..1d1d241b4 100644 --- a/cmd/cli/commands/install-runner.go +++ b/cmd/cli/commands/install-runner.go @@ -72,6 +72,80 @@ func inspectStandaloneRunner(container container.Summary) *standaloneRunner { return result } +// ensureOllamaRunnerAvailable is a utility function that ensures an ollama runner +// is available. Unlike the regular runner, ollama runners are always used via +// controller containers on all platforms. +func ensureOllamaRunnerAvailable(ctx context.Context, printer standalone.StatusPrinter) (*standaloneRunner, error) { + // For ollama, we always use controller containers on all platforms + // If automatic installation has been disabled, then don't do anything. + if os.Getenv("MODEL_RUNNER_NO_AUTO_INSTALL") != "" { + return nil, nil + } + + // Ensure that the output printer is non-nil. + if printer == nil { + printer = standalone.NoopPrinter() + } + + // Create a Docker client for the active context. + dockerClient, err := desktop.DockerClientForContext(dockerCLI, dockerCLI.CurrentContext()) + if err != nil { + return nil, fmt.Errorf("failed to create Docker client: %w", err) + } + + // Check if an ollama runner container exists. + containerID, _, container, err := standalone.FindOllamaControllerContainer(ctx, dockerClient) + if err != nil { + return nil, fmt.Errorf("unable to identify existing ollama runner: %w", err) + } else if containerID != "" { + return inspectStandaloneRunner(container), nil + } + + // Automatically determine GPU support. + gpu, err := gpupkg.ProbeGPUSupport(ctx, dockerClient) + if err != nil { + return nil, fmt.Errorf("unable to probe GPU support: %w", err) + } + + // Ensure that we have an up-to-date copy of the ollama image. + if err := standalone.EnsureOllamaImage(ctx, dockerClient, gpu, "", printer); err != nil { + return nil, fmt.Errorf("unable to pull latest ollama image: %w", err) + } + + // Ensure that we have an ollama storage volume. + modelStorageVolume, err := standalone.EnsureOllamaStorageVolume(ctx, dockerClient, printer) + if err != nil { + return nil, fmt.Errorf("unable to initialize ollama storage: %w", err) + } + + // Create the ollama runner container. + port := uint16(standalone.DefaultOllamaPort) + host := "127.0.0.1" + engineKind := modelRunner.EngineKind() + environment := "moby" + if engineKind == types.ModelRunnerEngineKindCloud { + environment = "cloud" + } + if err := standalone.CreateOllamaControllerContainer(ctx, dockerClient, port, host, environment, false, gpu, "", modelStorageVolume, printer, engineKind); err != nil { + return nil, fmt.Errorf("unable to initialize ollama container: %w", err) + } + + // Poll until we get a response from the ollama runner. + // Note: We reuse the same wait logic, assuming ollama responds similarly + if err := waitForStandaloneRunnerAfterInstall(ctx); err != nil { + return nil, err + } + + // Find the runner container. + containerID, _, container, err = standalone.FindOllamaControllerContainer(ctx, dockerClient) + if err != nil { + return nil, fmt.Errorf("unable to identify existing ollama runner: %w", err) + } else if containerID == "" { + return nil, errors.New("ollama runner not found after installation") + } + return inspectStandaloneRunner(container), nil +} + // ensureStandaloneRunnerAvailable is a utility function that other commands can // use to initialize a default standalone model runner. It is a no-op in // unsupported contexts or if automatic installs have been disabled. @@ -168,6 +242,7 @@ type runnerOptions struct { doNotTrack bool pullImage bool pruneContainers bool + ollama bool } // runInstallOrStart is shared logic for install-runner and start-runner commands @@ -191,7 +266,11 @@ func runInstallOrStart(cmd *cobra.Command, opts runnerOptions) error { // Use "0" as a sentinel default flag value so it's not displayed automatically. // The default values are written in the usage string. // Hence, the user currently won't be able to set the port to 0 in order to get a random available port. - port = standalone.DefaultControllerPortMoby + if opts.ollama { + port = standalone.DefaultOllamaPort + } else { + port = standalone.DefaultControllerPortMoby + } } // HACK: If we're in a Cloud context, then we need to use a // different default port because it conflicts with Docker Desktop's @@ -200,7 +279,7 @@ func runInstallOrStart(cmd *cobra.Command, opts runnerOptions) error { // when context detection happens. So assume that a default value // indicates that we want the Cloud default port. This is less // problematic in Cloud since the UX there is mostly invisible. - if engineKind == types.ModelRunnerEngineKindCloud && + if !opts.ollama && engineKind == types.ModelRunnerEngineKindCloud && port == standalone.DefaultControllerPortMoby { port = standalone.DefaultControllerPortCloud } @@ -219,18 +298,35 @@ func runInstallOrStart(cmd *cobra.Command, opts runnerOptions) error { // If pruning containers (reinstall), remove any existing model runner containers. if opts.pruneContainers { - if err := standalone.PruneControllerContainers(cmd.Context(), dockerClient, false, cmd); err != nil { - return fmt.Errorf("unable to remove model runner container(s): %w", err) + if opts.ollama { + if err := standalone.PruneOllamaControllerContainers(cmd.Context(), dockerClient, false, cmd); err != nil { + return fmt.Errorf("unable to remove ollama runner container(s): %w", err) + } + } else { + if err := standalone.PruneControllerContainers(cmd.Context(), dockerClient, false, cmd); err != nil { + return fmt.Errorf("unable to remove model runner container(s): %w", err) + } } } else { // Check if an active model runner container already exists (install only). - if ctrID, ctrName, _, err := standalone.FindControllerContainer(cmd.Context(), dockerClient); err != nil { + var ctrID, ctrName string + var err error + if opts.ollama { + ctrID, ctrName, _, err = standalone.FindOllamaControllerContainer(cmd.Context(), dockerClient) + } else { + ctrID, ctrName, _, err = standalone.FindControllerContainer(cmd.Context(), dockerClient) + } + if err != nil { return err } else if ctrID != "" { + runnerType := "Model Runner" + if opts.ollama { + runnerType = "ollama runner" + } if ctrName != "" { - cmd.Printf("Model Runner container %s (%s) is already running\n", ctrName, ctrID[:12]) + cmd.Printf("%s container %s (%s) is already running\n", runnerType, ctrName, ctrID[:12]) } else { - cmd.Printf("Model Runner container %s is already running\n", ctrID[:12]) + cmd.Printf("%s container %s is already running\n", runnerType, ctrID[:12]) } return nil } @@ -238,6 +334,7 @@ func runInstallOrStart(cmd *cobra.Command, opts runnerOptions) error { // Determine GPU support. var gpu gpupkg.GPUSupport + var gpuVariant string if opts.gpuMode == "auto" { gpu, err = gpupkg.ProbeGPUSupport(cmd.Context(), dockerClient) if err != nil { @@ -245,25 +342,49 @@ func runInstallOrStart(cmd *cobra.Command, opts runnerOptions) error { } } else if opts.gpuMode == "cuda" { gpu = gpupkg.GPUSupportCUDA + } else if opts.gpuMode == "rocm" { + gpu = gpupkg.GPUSupportROCm + gpuVariant = "rocm" } else if opts.gpuMode != "none" { return fmt.Errorf("unknown GPU specification: %q", opts.gpuMode) } // Ensure that we have an up-to-date copy of the image, if requested. if opts.pullImage { - if err := standalone.EnsureControllerImage(cmd.Context(), dockerClient, gpu, cmd); err != nil { - return fmt.Errorf("unable to pull latest standalone model runner image: %w", err) + if opts.ollama { + if err := standalone.EnsureOllamaImage(cmd.Context(), dockerClient, gpu, gpuVariant, cmd); err != nil { + return fmt.Errorf("unable to pull latest ollama image: %w", err) + } + } else { + if err := standalone.EnsureControllerImage(cmd.Context(), dockerClient, gpu, cmd); err != nil { + return fmt.Errorf("unable to pull latest standalone model runner image: %w", err) + } } } // Ensure that we have a model storage volume. - modelStorageVolume, err := standalone.EnsureModelStorageVolume(cmd.Context(), dockerClient, cmd) - if err != nil { - return fmt.Errorf("unable to initialize standalone model storage: %w", err) + var modelStorageVolume string + if opts.ollama { + modelStorageVolume, err = standalone.EnsureOllamaStorageVolume(cmd.Context(), dockerClient, cmd) + if err != nil { + return fmt.Errorf("unable to initialize ollama storage: %w", err) + } + } else { + modelStorageVolume, err = standalone.EnsureModelStorageVolume(cmd.Context(), dockerClient, cmd) + if err != nil { + return fmt.Errorf("unable to initialize standalone model storage: %w", err) + } } + // Create the model runner container. - if err := standalone.CreateControllerContainer(cmd.Context(), dockerClient, port, opts.host, environment, opts.doNotTrack, gpu, modelStorageVolume, cmd, engineKind); err != nil { - return fmt.Errorf("unable to initialize standalone model runner container: %w", err) + if opts.ollama { + if err := standalone.CreateOllamaControllerContainer(cmd.Context(), dockerClient, port, opts.host, environment, opts.doNotTrack, gpu, gpuVariant, modelStorageVolume, cmd, engineKind); err != nil { + return fmt.Errorf("unable to initialize ollama container: %w", err) + } + } else { + if err := standalone.CreateControllerContainer(cmd.Context(), dockerClient, port, opts.host, environment, opts.doNotTrack, gpu, modelStorageVolume, cmd, engineKind); err != nil { + return fmt.Errorf("unable to initialize standalone model runner container: %w", err) + } } // Poll until we get a response from the model runner. @@ -275,6 +396,7 @@ func newInstallRunner() *cobra.Command { var host string var gpuMode string var doNotTrack bool + var ollama bool c := &cobra.Command{ Use: "install-runner", Short: "Install Docker Model Runner (Docker Engine only)", @@ -286,14 +408,16 @@ func newInstallRunner() *cobra.Command { doNotTrack: doNotTrack, pullImage: true, pruneContainers: false, + ollama: ollama, }) }, ValidArgsFunction: completion.NoComplete, } c.Flags().Uint16Var(&port, "port", 0, - "Docker container port for Docker Model Runner (default: 12434 for Docker Engine, 12435 for Cloud mode)") + "Docker container port for Docker Model Runner (default: 12434 for Docker Engine, 12435 for Cloud mode, 11434 for ollama)") c.Flags().StringVar(&host, "host", "127.0.0.1", "Host address to bind Docker Model Runner") - c.Flags().StringVar(&gpuMode, "gpu", "auto", "Specify GPU support (none|auto|cuda)") + c.Flags().StringVar(&gpuMode, "gpu", "auto", "Specify GPU support (none|auto|cuda|rocm)") c.Flags().BoolVar(&doNotTrack, "do-not-track", false, "Do not track models usage in Docker Model Runner") + c.Flags().BoolVar(&ollama, "ollama", false, "Use ollama runner instead of Docker Model Runner") return c } diff --git a/cmd/cli/commands/install-runner_test.go b/cmd/cli/commands/install-runner_test.go index bce0d8131..974209ccc 100644 --- a/cmd/cli/commands/install-runner_test.go +++ b/cmd/cli/commands/install-runner_test.go @@ -59,7 +59,7 @@ func TestInstallRunnerCommandFlags(t *testing.T) { cmd := newInstallRunner() // Verify all expected flags exist - expectedFlags := []string{"port", "host", "gpu", "do-not-track"} + expectedFlags := []string{"port", "host", "gpu", "do-not-track", "ollama"} for _, flagName := range expectedFlags { if cmd.Flags().Lookup(flagName) == nil { t.Errorf("Expected flag '--%s' not found", flagName) @@ -67,6 +67,66 @@ func TestInstallRunnerCommandFlags(t *testing.T) { } } +func TestInstallRunnerOllamaFlag(t *testing.T) { + cmd := newInstallRunner() + + // Verify the --ollama flag exists + ollamaFlag := cmd.Flags().Lookup("ollama") + if ollamaFlag == nil { + t.Fatal("--ollama flag not found") + } + + // Verify the default value + if ollamaFlag.DefValue != "false" { + t.Errorf("Expected default ollama value to be 'false', got '%s'", ollamaFlag.DefValue) + } + + // Verify the flag type + if ollamaFlag.Value.Type() != "bool" { + t.Errorf("Expected ollama flag type to be 'bool', got '%s'", ollamaFlag.Value.Type()) + } + + // Test setting the flag value + err := cmd.Flags().Set("ollama", "true") + if err != nil { + t.Errorf("Failed to set ollama flag: %v", err) + } + + // Verify the value was set + ollamaValue, err := cmd.Flags().GetBool("ollama") + if err != nil { + t.Errorf("Failed to get ollama flag value: %v", err) + } + if !ollamaValue { + t.Error("Expected ollama value to be true") + } +} + +func TestInstallRunnerGPUFlag(t *testing.T) { + cmd := newInstallRunner() + + // Verify the --gpu flag exists + gpuFlag := cmd.Flags().Lookup("gpu") + if gpuFlag == nil { + t.Fatal("--gpu flag not found") + } + + // Test setting gpu to rocm + err := cmd.Flags().Set("gpu", "rocm") + if err != nil { + t.Errorf("Failed to set gpu flag to 'rocm': %v", err) + } + + // Verify the value was set + gpuValue, err := cmd.Flags().GetString("gpu") + if err != nil { + t.Errorf("Failed to get gpu flag value: %v", err) + } + if gpuValue != "rocm" { + t.Errorf("Expected gpu value to be 'rocm', got '%s'", gpuValue) + } +} + func TestInstallRunnerCommandType(t *testing.T) { cmd := newInstallRunner() diff --git a/cmd/cli/commands/list.go b/cmd/cli/commands/list.go index cf984bfbc..48828d636 100644 --- a/cmd/cli/commands/list.go +++ b/cmd/cli/commands/list.go @@ -82,6 +82,9 @@ func listModels(openai bool, backend string, desktopClient *desktop.Client, quie } return formatter.ToStandardJSON(models) } + // TODO: Add support for listing ollama models from the ollama volume + // This would require querying both the model-runner and ollama daemons + // and merging the results models, err := desktopClient.List() if err != nil { err = handleClientError(err, "Failed to list models") diff --git a/cmd/cli/commands/pull.go b/cmd/cli/commands/pull.go index c311a747c..f6c3edaa6 100644 --- a/cmd/cli/commands/pull.go +++ b/cmd/cli/commands/pull.go @@ -27,10 +27,21 @@ func newPullCmd() *cobra.Command { return nil }, RunE: func(cmd *cobra.Command, args []string) error { + model := args[0] + // Check if this is an ollama model + if isOllamaModel(model) { + // For ollama models, ensure the ollama runner is available + if _, err := ensureOllamaRunnerAvailable(cmd.Context(), cmd); err != nil { + return fmt.Errorf("unable to initialize ollama runner: %w", err) + } + // TODO: Implement ollama-specific pull logic that communicates + // with the ollama daemon on port 11434 + return fmt.Errorf("ollama model pull not yet implemented - please use 'docker exec docker-ollama-runner ollama pull %s'", model) + } if _, err := ensureStandaloneRunnerAvailable(cmd.Context(), cmd); err != nil { return fmt.Errorf("unable to initialize standalone model runner: %w", err) } - return pullModel(cmd, desktopClient, args[0], ignoreRuntimeMemoryCheck) + return pullModel(cmd, desktopClient, model, ignoreRuntimeMemoryCheck) }, ValidArgsFunction: completion.NoComplete, } diff --git a/cmd/cli/commands/reinstall-runner.go b/cmd/cli/commands/reinstall-runner.go index dcec2bc73..670e0dcde 100644 --- a/cmd/cli/commands/reinstall-runner.go +++ b/cmd/cli/commands/reinstall-runner.go @@ -10,6 +10,7 @@ func newReinstallRunner() *cobra.Command { var host string var gpuMode string var doNotTrack bool + var ollama bool c := &cobra.Command{ Use: "reinstall-runner", Short: "Reinstall Docker Model Runner (Docker Engine only)", @@ -21,14 +22,16 @@ func newReinstallRunner() *cobra.Command { doNotTrack: doNotTrack, pullImage: true, pruneContainers: true, + ollama: ollama, }) }, ValidArgsFunction: completion.NoComplete, } c.Flags().Uint16Var(&port, "port", 0, - "Docker container port for Docker Model Runner (default: 12434 for Docker Engine, 12435 for Cloud mode)") + "Docker container port for Docker Model Runner (default: 12434 for Docker Engine, 12435 for Cloud mode, 11434 for ollama)") c.Flags().StringVar(&host, "host", "127.0.0.1", "Host address to bind Docker Model Runner") - c.Flags().StringVar(&gpuMode, "gpu", "auto", "Specify GPU support (none|auto|cuda)") + c.Flags().StringVar(&gpuMode, "gpu", "auto", "Specify GPU support (none|auto|cuda|rocm)") c.Flags().BoolVar(&doNotTrack, "do-not-track", false, "Do not track models usage in Docker Model Runner") + c.Flags().BoolVar(&ollama, "ollama", false, "Use ollama runner instead of Docker Model Runner") return c } diff --git a/cmd/cli/commands/reinstall-runner_test.go b/cmd/cli/commands/reinstall-runner_test.go index a6f160361..9d72636e8 100644 --- a/cmd/cli/commands/reinstall-runner_test.go +++ b/cmd/cli/commands/reinstall-runner_test.go @@ -39,7 +39,7 @@ func TestReinstallRunnerCommandFlags(t *testing.T) { cmd := newReinstallRunner() // Verify all expected flags exist - expectedFlags := []string{"port", "host", "gpu", "do-not-track"} + expectedFlags := []string{"port", "host", "gpu", "do-not-track", "ollama"} for _, flagName := range expectedFlags { if cmd.Flags().Lookup(flagName) == nil { t.Errorf("Expected flag '--%s' not found", flagName) @@ -47,6 +47,31 @@ func TestReinstallRunnerCommandFlags(t *testing.T) { } } +func TestReinstallRunnerOllamaFlag(t *testing.T) { + cmd := newReinstallRunner() + + // Verify the --ollama flag exists and can be set + ollamaFlag := cmd.Flags().Lookup("ollama") + if ollamaFlag == nil { + t.Fatal("--ollama flag not found") + } + + // Test setting the flag value + err := cmd.Flags().Set("ollama", "true") + if err != nil { + t.Errorf("Failed to set ollama flag: %v", err) + } + + // Verify the value was set + ollamaValue, err := cmd.Flags().GetBool("ollama") + if err != nil { + t.Errorf("Failed to get ollama flag value: %v", err) + } + if !ollamaValue { + t.Error("Expected ollama value to be true") + } +} + func TestReinstallRunnerCommandType(t *testing.T) { cmd := newReinstallRunner() diff --git a/cmd/cli/commands/restart-runner.go b/cmd/cli/commands/restart-runner.go index 77ffa1bc0..869516387 100644 --- a/cmd/cli/commands/restart-runner.go +++ b/cmd/cli/commands/restart-runner.go @@ -10,6 +10,7 @@ func newRestartRunner() *cobra.Command { var host string var gpuMode string var doNotTrack bool + var ollama bool c := &cobra.Command{ Use: "restart-runner", Short: "Restart Docker Model Runner (Docker Engine only)", @@ -18,6 +19,7 @@ func newRestartRunner() *cobra.Command { if err := runUninstallOrStop(cmd, cleanupOptions{ models: false, removeImages: false, + ollama: ollama, }); err != nil { return err } @@ -29,14 +31,16 @@ func newRestartRunner() *cobra.Command { gpuMode: gpuMode, doNotTrack: doNotTrack, pullImage: false, + ollama: ollama, }) }, ValidArgsFunction: completion.NoComplete, } c.Flags().Uint16Var(&port, "port", 0, - "Docker container port for Docker Model Runner (default: 12434 for Docker Engine, 12435 for Cloud mode)") + "Docker container port for Docker Model Runner (default: 12434 for Docker Engine, 12435 for Cloud mode, 11434 for ollama)") c.Flags().StringVar(&host, "host", "127.0.0.1", "Host address to bind Docker Model Runner") - c.Flags().StringVar(&gpuMode, "gpu", "auto", "Specify GPU support (none|auto|cuda)") + c.Flags().StringVar(&gpuMode, "gpu", "auto", "Specify GPU support (none|auto|cuda|rocm)") c.Flags().BoolVar(&doNotTrack, "do-not-track", false, "Do not track models usage in Docker Model Runner") + c.Flags().BoolVar(&ollama, "ollama", false, "Restart ollama runner instead of Docker Model Runner") return c } diff --git a/cmd/cli/commands/run.go b/cmd/cli/commands/run.go index 379e609c6..2e1a9aa3a 100644 --- a/cmd/cli/commands/run.go +++ b/cmd/cli/commands/run.go @@ -593,6 +593,17 @@ func newRunCmd() *cobra.Command { } } + // Check if this is an ollama model + if isOllamaModel(model) { + // For ollama models, ensure the ollama runner is available + if _, err := ensureOllamaRunnerAvailable(cmd.Context(), cmd); err != nil { + return fmt.Errorf("unable to initialize ollama runner: %w", err) + } + // TODO: Implement ollama-specific run logic that communicates + // with the ollama daemon on port 11434 + return fmt.Errorf("ollama model run not yet implemented - please use 'docker exec -it docker-ollama-runner ollama run %s'", model) + } + if _, err := ensureStandaloneRunnerAvailable(cmd.Context(), cmd); err != nil { return fmt.Errorf("unable to initialize standalone model runner: %w", err) } diff --git a/cmd/cli/commands/start-runner.go b/cmd/cli/commands/start-runner.go index dc195faca..52b414511 100644 --- a/cmd/cli/commands/start-runner.go +++ b/cmd/cli/commands/start-runner.go @@ -9,6 +9,7 @@ func newStartRunner() *cobra.Command { var port uint16 var gpuMode string var doNotTrack bool + var ollama bool c := &cobra.Command{ Use: "start-runner", Short: "Start Docker Model Runner (Docker Engine only)", @@ -18,13 +19,15 @@ func newStartRunner() *cobra.Command { gpuMode: gpuMode, doNotTrack: doNotTrack, pullImage: false, + ollama: ollama, }) }, ValidArgsFunction: completion.NoComplete, } c.Flags().Uint16Var(&port, "port", 0, - "Docker container port for Docker Model Runner (default: 12434 for Docker Engine, 12435 for Cloud mode)") - c.Flags().StringVar(&gpuMode, "gpu", "auto", "Specify GPU support (none|auto|cuda)") + "Docker container port for Docker Model Runner (default: 12434 for Docker Engine, 12435 for Cloud mode, 11434 for ollama)") + c.Flags().StringVar(&gpuMode, "gpu", "auto", "Specify GPU support (none|auto|cuda|rocm)") c.Flags().BoolVar(&doNotTrack, "do-not-track", false, "Do not track models usage in Docker Model Runner") + c.Flags().BoolVar(&ollama, "ollama", false, "Use ollama runner instead of Docker Model Runner") return c } diff --git a/cmd/cli/commands/stop-runner.go b/cmd/cli/commands/stop-runner.go index b59af9f69..4b7747e17 100644 --- a/cmd/cli/commands/stop-runner.go +++ b/cmd/cli/commands/stop-runner.go @@ -6,7 +6,7 @@ import ( ) func newStopRunner() *cobra.Command { - var models bool + var models, ollama bool c := &cobra.Command{ Use: "stop-runner", Short: "Stop Docker Model Runner (Docker Engine only)", @@ -14,10 +14,12 @@ func newStopRunner() *cobra.Command { return runUninstallOrStop(cmd, cleanupOptions{ models: models, removeImages: false, + ollama: ollama, }) }, ValidArgsFunction: completion.NoComplete, } c.Flags().BoolVar(&models, "models", false, "Remove model storage volume") + c.Flags().BoolVar(&ollama, "ollama", false, "Stop ollama runner instead of Docker Model Runner") return c } diff --git a/cmd/cli/commands/uninstall-runner.go b/cmd/cli/commands/uninstall-runner.go index 8ab5fbd19..7cd627a60 100644 --- a/cmd/cli/commands/uninstall-runner.go +++ b/cmd/cli/commands/uninstall-runner.go @@ -14,6 +14,7 @@ import ( type cleanupOptions struct { models bool removeImages bool + ollama bool } // runUninstallOrStop is shared logic for uninstall-runner and stop-runner commands @@ -38,14 +39,26 @@ func runUninstallOrStop(cmd *cobra.Command, opts cleanupOptions) error { } // Remove any model runner containers. - if err := standalone.PruneControllerContainers(cmd.Context(), dockerClient, false, cmd); err != nil { - return fmt.Errorf("unable to remove model runner container(s): %w", err) + if opts.ollama { + if err := standalone.PruneOllamaControllerContainers(cmd.Context(), dockerClient, false, cmd); err != nil { + return fmt.Errorf("unable to remove ollama runner container(s): %w", err) + } + } else { + if err := standalone.PruneControllerContainers(cmd.Context(), dockerClient, false, cmd); err != nil { + return fmt.Errorf("unable to remove model runner container(s): %w", err) + } } // Remove model runner images, if requested. if opts.removeImages { - if err := standalone.PruneControllerImages(cmd.Context(), dockerClient, cmd); err != nil { - return fmt.Errorf("unable to remove model runner image(s): %w", err) + if opts.ollama { + if err := standalone.PruneOllamaImages(cmd.Context(), dockerClient, cmd); err != nil { + return fmt.Errorf("unable to remove ollama image(s): %w", err) + } + } else { + if err := standalone.PruneControllerImages(cmd.Context(), dockerClient, cmd); err != nil { + return fmt.Errorf("unable to remove model runner image(s): %w", err) + } } } @@ -60,7 +73,7 @@ func runUninstallOrStop(cmd *cobra.Command, opts cleanupOptions) error { } func newUninstallRunner() *cobra.Command { - var models, images bool + var models, images, ollama bool c := &cobra.Command{ Use: "uninstall-runner", Short: "Uninstall Docker Model Runner (Docker Engine only)", @@ -68,11 +81,13 @@ func newUninstallRunner() *cobra.Command { return runUninstallOrStop(cmd, cleanupOptions{ models: models, removeImages: images, + ollama: ollama, }) }, ValidArgsFunction: completion.NoComplete, } c.Flags().BoolVar(&models, "models", false, "Remove model storage volume") - c.Flags().BoolVar(&images, "images", false, "Remove "+standalone.ControllerImage+" images") + c.Flags().BoolVar(&images, "images", false, "Remove runner images") + c.Flags().BoolVar(&ollama, "ollama", false, "Uninstall ollama runner instead of Docker Model Runner") return c } diff --git a/cmd/cli/commands/utils.go b/cmd/cli/commands/utils.go index 46c4b5063..902a5a0f4 100644 --- a/cmd/cli/commands/utils.go +++ b/cmd/cli/commands/utils.go @@ -17,6 +17,11 @@ const ( var notRunningErr = fmt.Errorf("Docker Model Runner is not running. Please start it and try again.\n") +// isOllamaModel checks if a model name is from ollama.com +func isOllamaModel(model string) bool { + return strings.HasPrefix(model, "ollama.com/") +} + func handleClientError(err error, message string) error { if errors.Is(err, desktop.ErrServiceUnavailable) { return notRunningErr diff --git a/cmd/cli/commands/utils_test.go b/cmd/cli/commands/utils_test.go new file mode 100644 index 000000000..04183e9ce --- /dev/null +++ b/cmd/cli/commands/utils_test.go @@ -0,0 +1,58 @@ +package commands + +import ( + "testing" +) + +func TestIsOllamaModel(t *testing.T) { + testCases := []struct { + name string + model string + expected bool + }{ + { + name: "ollama library model", + model: "ollama.com/library/smollm:135m", + expected: true, + }, + { + name: "ollama user model", + model: "ollama.com/user/custom-model:latest", + expected: true, + }, + { + name: "ollama simple", + model: "ollama.com/model", + expected: true, + }, + { + name: "docker hub model", + model: "docker.io/library/llama:latest", + expected: false, + }, + { + name: "huggingface model", + model: "hf.co/TheBloke/Llama-2-7B-GGUF", + expected: false, + }, + { + name: "plain model name", + model: "llama2:7b", + expected: false, + }, + { + name: "empty string", + model: "", + expected: false, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + result := isOllamaModel(tc.model) + if result != tc.expected { + t.Errorf("isOllamaModel(%q) = %v, expected %v", tc.model, result, tc.expected) + } + }) + } +} diff --git a/cmd/cli/pkg/gpu/gpu.go b/cmd/cli/pkg/gpu/gpu.go index b1e1e9fdb..2f32f5685 100644 --- a/cmd/cli/pkg/gpu/gpu.go +++ b/cmd/cli/pkg/gpu/gpu.go @@ -15,6 +15,8 @@ const ( GPUSupportNone GPUSupport = iota // GPUSupportCUDA indicates CUDA GPU support. GPUSupportCUDA + // GPUSupportROCm indicates ROCm GPU support (AMD). + GPUSupportROCm ) // ProbeGPUSupport determines whether or not the Docker engine has GPU support. diff --git a/cmd/cli/pkg/standalone/containers.go b/cmd/cli/pkg/standalone/containers.go index 8e6589e38..b41c734a6 100644 --- a/cmd/cli/pkg/standalone/containers.go +++ b/cmd/cli/pkg/standalone/containers.go @@ -29,6 +29,9 @@ import ( // controllerContainerName is the name to use for the controller container. const controllerContainerName = "docker-model-runner" +// ollamaControllerContainerName is the name to use for the ollama controller container. +const ollamaControllerContainerName = "docker-ollama-runner" + // copyDockerConfigToContainer copies the Docker config file from the host to the container // and sets up proper ownership and permissions for the modelrunner user. // It does nothing for Desktop and Cloud engine kinds. @@ -134,8 +137,20 @@ func execInContainer(ctx context.Context, dockerClient *client.Client, container // returns the ID of the container (if found), the container name (if any), the // full container summary (if found), or any error that occurred. func FindControllerContainer(ctx context.Context, dockerClient client.ContainerAPIClient) (string, string, container.Summary, error) { + return FindControllerContainerByType(ctx, dockerClient, runnerTypeModelRunner) +} + +// FindOllamaControllerContainer searches for a running ollama controller container. +func FindOllamaControllerContainer(ctx context.Context, dockerClient client.ContainerAPIClient) (string, string, container.Summary, error) { + return FindControllerContainerByType(ctx, dockerClient, runnerTypeOllama) +} + +// FindControllerContainerByType searches for a running controller container of a specific type. +// It returns the ID of the container (if found), the container name (if any), the +// full container summary (if found), or any error that occurred. +func FindControllerContainerByType(ctx context.Context, dockerClient client.ContainerAPIClient, runnerType string) (string, string, container.Summary, error) { // Before listing, prune any stopped controller containers. - if err := PruneControllerContainers(ctx, dockerClient, true, NoopPrinter()); err != nil { + if err := PruneControllerContainersByType(ctx, dockerClient, true, NoopPrinter(), runnerType); err != nil { return "", "", container.Summary{}, fmt.Errorf("unable to prune stopped model runner containers: %w", err) } @@ -146,6 +161,7 @@ func FindControllerContainer(ctx context.Context, dockerClient client.ContainerA // middleware only shows these containers if no value is queried. filters.Arg("label", labelDesktopService), filters.Arg("label", labelRole+"="+roleController), + filters.Arg("label", labelRunnerType+"="+runnerType), ), }) if err != nil { @@ -219,7 +235,22 @@ func ensureContainerStarted(ctx context.Context, dockerClient client.ContainerAP // CreateControllerContainer creates and starts a controller container. func CreateControllerContainer(ctx context.Context, dockerClient *client.Client, port uint16, host string, environment string, doNotTrack bool, gpu gpupkg.GPUSupport, modelStorageVolume string, printer StatusPrinter, engineKind types.ModelRunnerEngineKind) error { - imageName := controllerImageName(gpu) + return createControllerContainerInternal(ctx, dockerClient, port, host, environment, doNotTrack, gpu, "", modelStorageVolume, printer, engineKind, runnerTypeModelRunner, controllerContainerName, "/models") +} + +// CreateOllamaControllerContainer creates and starts an ollama controller container. +func CreateOllamaControllerContainer(ctx context.Context, dockerClient *client.Client, port uint16, host string, environment string, doNotTrack bool, gpu gpupkg.GPUSupport, gpuVariant string, modelStorageVolume string, printer StatusPrinter, engineKind types.ModelRunnerEngineKind) error { + return createControllerContainerInternal(ctx, dockerClient, port, host, environment, doNotTrack, gpu, gpuVariant, modelStorageVolume, printer, engineKind, runnerTypeOllama, ollamaControllerContainerName, "/root/.ollama") +} + +// createControllerContainerInternal creates and starts a controller container of a specific type. +func createControllerContainerInternal(ctx context.Context, dockerClient *client.Client, port uint16, host string, environment string, doNotTrack bool, gpu gpupkg.GPUSupport, gpuVariant string, modelStorageVolume string, printer StatusPrinter, engineKind types.ModelRunnerEngineKind, runnerType string, containerName string, mountTarget string) error { + var imageName string + if runnerType == runnerTypeOllama { + imageName = ollamaImageName(gpu, gpuVariant) + } else { + imageName = controllerImageName(gpu) + } // Set up the container configuration. portStr := strconv.Itoa(int(port)) @@ -239,6 +270,7 @@ func CreateControllerContainer(ctx context.Context, dockerClient *client.Client, Labels: map[string]string{ labelDesktopService: serviceModelRunner, labelRole: roleController, + labelRunnerType: runnerType, }, } hostConfig := &container.HostConfig{ @@ -246,7 +278,7 @@ func CreateControllerContainer(ctx context.Context, dockerClient *client.Client, { Type: mount.TypeVolume, Source: modelStorageVolume, - Target: "/models", + Target: mountTarget, }, }, RestartPolicy: container.RestartPolicy{ @@ -318,19 +350,19 @@ func CreateControllerContainer(ctx context.Context, dockerClient *client.Client, // pass silently and simply work in conjunction with any concurrent // installers to start the container. // TODO: Remove strings.Contains check once we ensure it's not necessary. - resp, err := dockerClient.ContainerCreate(ctx, config, hostConfig, nil, nil, controllerContainerName) + resp, err := dockerClient.ContainerCreate(ctx, config, hostConfig, nil, nil, containerName) if err != nil && !(errdefs.IsConflict(err) || strings.Contains(err.Error(), "is already in use by container")) { - return fmt.Errorf("failed to create container %s: %w", controllerContainerName, err) + return fmt.Errorf("failed to create container %s: %w", containerName, err) } created := err == nil // Start the container. - printer.Printf("Starting model runner container %s...\n", controllerContainerName) - if err := ensureContainerStarted(ctx, dockerClient, controllerContainerName); err != nil { + printer.Printf("Starting model runner container %s...\n", containerName) + if err := ensureContainerStarted(ctx, dockerClient, containerName); err != nil { if created { _ = dockerClient.ContainerRemove(ctx, resp.ID, container.RemoveOptions{Force: true}) } - return fmt.Errorf("failed to start container %s: %w", controllerContainerName, err) + return fmt.Errorf("failed to start container %s: %w", containerName, err) } // Copy Docker config file if it exists and we're the container creator. @@ -346,6 +378,16 @@ func CreateControllerContainer(ctx context.Context, dockerClient *client.Client, // PruneControllerContainers stops and removes any model runner controller // containers. func PruneControllerContainers(ctx context.Context, dockerClient client.ContainerAPIClient, skipRunning bool, printer StatusPrinter) error { + return PruneControllerContainersByType(ctx, dockerClient, skipRunning, printer, runnerTypeModelRunner) +} + +// PruneOllamaControllerContainers stops and removes any ollama controller containers. +func PruneOllamaControllerContainers(ctx context.Context, dockerClient client.ContainerAPIClient, skipRunning bool, printer StatusPrinter) error { + return PruneControllerContainersByType(ctx, dockerClient, skipRunning, printer, runnerTypeOllama) +} + +// PruneControllerContainersByType stops and removes controller containers of a specific type. +func PruneControllerContainersByType(ctx context.Context, dockerClient client.ContainerAPIClient, skipRunning bool, printer StatusPrinter, runnerType string) error { // Identify all controller containers. containers, err := dockerClient.ContainerList(ctx, container.ListOptions{ All: true, @@ -354,6 +396,7 @@ func PruneControllerContainers(ctx context.Context, dockerClient client.Containe // middleware only shows these containers if no value is queried. filters.Arg("label", labelDesktopService), filters.Arg("label", labelRole+"="+roleController), + filters.Arg("label", labelRunnerType+"="+runnerType), ), }) if err != nil { diff --git a/cmd/cli/pkg/standalone/controller_image.go b/cmd/cli/pkg/standalone/controller_image.go index 2962e97b7..eb48eb76e 100644 --- a/cmd/cli/pkg/standalone/controller_image.go +++ b/cmd/cli/pkg/standalone/controller_image.go @@ -11,6 +11,10 @@ const ( ControllerImage = "docker/model-runner" // defaultControllerImageVersion is the image version used for the controller container defaultControllerImageVersion = "latest" + // OllamaImage is the image used for the ollama controller container. + OllamaImage = "ollama/ollama" + // defaultOllamaImageVersion is the image version used for the ollama controller container + defaultOllamaImageVersion = "latest" ) func controllerImageVersion() string { @@ -46,3 +50,31 @@ func fmtControllerImageName(repo, version, variant string) string { func controllerImageName(detectedGPU gpupkg.GPUSupport) string { return fmtControllerImageName(ControllerImage, controllerImageVersion(), controllerImageVariant(detectedGPU)) } + +func ollamaImageVersion() string { + if version, ok := os.LookupEnv("OLLAMA_CONTROLLER_VERSION"); ok && version != "" { + return version + } + return defaultOllamaImageVersion +} + +func ollamaImageVariant(detectedGPU gpupkg.GPUSupport) string { + if variant, ok := os.LookupEnv("OLLAMA_CONTROLLER_VARIANT"); ok { + return variant + } + // For ollama, we have "rocm" variant for AMD GPUs + // Note: CUDA GPUs use the base "latest" image + if detectedGPU == gpupkg.GPUSupportCUDA { + return "" // ollama/ollama:latest works for CUDA + } + return "" +} + +func ollamaImageName(detectedGPU gpupkg.GPUSupport, gpuVariant string) string { + variant := ollamaImageVariant(detectedGPU) + // Allow explicit override with gpuVariant parameter (e.g., "rocm") + if gpuVariant == "rocm" { + variant = "rocm" + } + return fmtControllerImageName(OllamaImage, ollamaImageVersion(), variant) +} diff --git a/cmd/cli/pkg/standalone/images.go b/cmd/cli/pkg/standalone/images.go index 48897b248..21ed54fcd 100644 --- a/cmd/cli/pkg/standalone/images.go +++ b/cmd/cli/pkg/standalone/images.go @@ -15,7 +15,17 @@ import ( // EnsureControllerImage ensures that the controller container image is pulled. func EnsureControllerImage(ctx context.Context, dockerClient client.ImageAPIClient, gpu gpupkg.GPUSupport, printer StatusPrinter) error { imageName := controllerImageName(gpu) + return ensureImage(ctx, dockerClient, imageName, printer) +} + +// EnsureOllamaImage ensures that the ollama container image is pulled. +func EnsureOllamaImage(ctx context.Context, dockerClient client.ImageAPIClient, gpu gpupkg.GPUSupport, gpuVariant string, printer StatusPrinter) error { + imageName := ollamaImageName(gpu, gpuVariant) + return ensureImage(ctx, dockerClient, imageName, printer) +} +// ensureImage pulls a container image if needed. +func ensureImage(ctx context.Context, dockerClient client.ImageAPIClient, imageName string, printer StatusPrinter) error { // Perform the pull. out, err := dockerClient.ImagePull(ctx, imageName, image.PullOptions{}) if err != nil { @@ -59,3 +69,19 @@ func PruneControllerImages(ctx context.Context, dockerClient client.ImageAPIClie } return nil } + +// PruneOllamaImages removes any unused ollama container images. +func PruneOllamaImages(ctx context.Context, dockerClient client.ImageAPIClient, printer StatusPrinter) error { + // Remove the standard ollama image, if present. + imageNameBase := fmtControllerImageName(OllamaImage, ollamaImageVersion(), "") + if _, err := dockerClient.ImageRemove(ctx, imageNameBase, image.RemoveOptions{}); err == nil { + printer.Println("Removed image", imageNameBase) + } + + // Remove the ROCm ollama image, if present. + imageNameROCm := fmtControllerImageName(OllamaImage, ollamaImageVersion(), "rocm") + if _, err := dockerClient.ImageRemove(ctx, imageNameROCm, image.RemoveOptions{}); err == nil { + printer.Println("Removed image", imageNameROCm) + } + return nil +} diff --git a/cmd/cli/pkg/standalone/labels.go b/cmd/cli/pkg/standalone/labels.go index a9260bad1..4939a8bee 100644 --- a/cmd/cli/pkg/standalone/labels.go +++ b/cmd/cli/pkg/standalone/labels.go @@ -26,4 +26,14 @@ const ( // roleModelStorage is the role label value used to identify the model // runner model storage volume. roleModelStorage = "model-storage" + + // labelRunnerType is the label used to identify the type of runner + // (e.g., "model-runner" or "ollama"). + labelRunnerType = "com.docker.model-runner.type" + + // runnerTypeModelRunner is the runner type label value for model-runner. + runnerTypeModelRunner = "model-runner" + + // runnerTypeOllama is the runner type label value for ollama. + runnerTypeOllama = "ollama" ) diff --git a/cmd/cli/pkg/standalone/ports.go b/cmd/cli/pkg/standalone/ports.go index eea85732c..c21c5629e 100644 --- a/cmd/cli/pkg/standalone/ports.go +++ b/cmd/cli/pkg/standalone/ports.go @@ -7,4 +7,7 @@ const ( // DefaultControllerPortCloud is the default TCP port on which the // standalone controller will listen for requests in Cloud environments. DefaultControllerPortCloud = 12435 + // DefaultOllamaPort is the default TCP port on which ollama + // controller will listen for requests. + DefaultOllamaPort = 11434 ) diff --git a/cmd/cli/pkg/standalone/volumes.go b/cmd/cli/pkg/standalone/volumes.go index ce5ddc06e..9b4b54323 100644 --- a/cmd/cli/pkg/standalone/volumes.go +++ b/cmd/cli/pkg/standalone/volumes.go @@ -12,14 +12,30 @@ import ( // modelStorageVolumeName is the name to use for the model storage volume. const modelStorageVolumeName = "docker-model-runner-models" +// ollamaStorageVolumeName is the name to use for the ollama storage volume. +const ollamaStorageVolumeName = "ollama" + // EnsureModelStorageVolume ensures that a model storage volume exists, creating // it if necessary. It returns the name of the storage volume or any error that // occurred. func EnsureModelStorageVolume(ctx context.Context, dockerClient client.VolumeAPIClient, printer StatusPrinter) (string, error) { + return ensureStorageVolume(ctx, dockerClient, printer, modelStorageVolumeName, runnerTypeModelRunner) +} + +// EnsureOllamaStorageVolume ensures that an ollama storage volume exists, creating +// it if necessary. It returns the name of the storage volume or any error that +// occurred. +func EnsureOllamaStorageVolume(ctx context.Context, dockerClient client.VolumeAPIClient, printer StatusPrinter) (string, error) { + return ensureStorageVolume(ctx, dockerClient, printer, ollamaStorageVolumeName, runnerTypeOllama) +} + +// ensureStorageVolume ensures that a storage volume exists for the given runner type. +func ensureStorageVolume(ctx context.Context, dockerClient client.VolumeAPIClient, printer StatusPrinter, volumeName, runnerType string) (string, error) { // Try to identify the storage volume. volumes, err := dockerClient.VolumeList(ctx, volume.ListOptions{ Filters: filters.NewArgs( filters.Arg("label", labelRole+"="+roleModelStorage), + filters.Arg("label", labelRunnerType+"="+runnerType), ), }) if err != nil { @@ -33,18 +49,19 @@ func EnsureModelStorageVolume(ctx context.Context, dockerClient client.VolumeAPI } // Create the volume. - printer.Printf("Creating model storage volume %s...\n", modelStorageVolumeName) - volume, err := dockerClient.VolumeCreate(ctx, volume.CreateOptions{ - Name: modelStorageVolumeName, + printer.Printf("Creating model storage volume %s...\n", volumeName) + vol, err := dockerClient.VolumeCreate(ctx, volume.CreateOptions{ + Name: volumeName, Labels: map[string]string{ labelDesktopService: serviceModelRunner, labelRole: roleModelStorage, + labelRunnerType: runnerType, }, }) if err != nil { return "", fmt.Errorf("unable to create volume: %w", err) } - return volume.Name, nil + return vol.Name, nil } // PruneModelStorageVolumes removes any unused model storage volume(s).