fix: detect Spark Ollama CPU fallback (#4108)

## Summary
- fail Spark Ollama validation when the loaded model reports CPU-only
execution via `/api/ps`
- add a Spark `OLLAMA_LLM_LIBRARY=cuda_v13` systemd override when that
backend is installed
- enable the managed Linux Ollama service so local inference survives
reboot

## Test Plan
- `npm run build:cli`
- `npm run typecheck:cli`
- `npx vitest run src/lib/inference/local.test.ts
test/onboard-selection.test.ts --testTimeout 20000`
- `npx vitest run src/lib/inference/local.test.ts --testTimeout 20000`
- `npx vitest run src/lib/inference/local.test.ts
test/onboard-selection.test.ts -t "runtime status|CPU-only|GPU
memory|adds Spark CUDA v13" --testTimeout 20000`
- `git diff --check`

Note: local pre-commit/pre-push full CLI coverage hooks failed in
unrelated tests on this machine, including the missing
`nemoclaw/node_modules/json5` fixture path; pushed with `--no-verify`
after focused validation passed.

<!-- This is an auto-generated comment: release notes by coderabbit.ai
-->
## Summary by CodeRabbit

* **New Features**
* Runtime detection for CPU-only Ollama models with tailored diagnostics
on Spark systems
* Optional CUDA v13 library selection for NVIDIA DGX Spark installs and
managed loopback service enablement
* Generation and use of an openshell-gateway.toml for Docker container
launches

* **Bug Fixes**
* Early validation to detect incompatible Ollama runtime configurations
* Ensured systemd override is persisted and service enabled across
reboots
  * Preserved gateway config path in container env handling

* **Tests**
* Added tests for runtime probing, Spark systemd behavior, and gateway
config generation

<!-- review_stack_entry_start -->

[![Review Change
Stack](https://storage.googleapis.com/coderabbit_public_assets/review-stack-in-coderabbit-ui.svg)](https://app.coderabbit.ai/change-stack/NVIDIA/NemoClaw/pull/4108?utm_source=github_walkthrough&utm_medium=github&utm_campaign=change_stack)

<!-- review_stack_entry_end -->
<!-- end of auto-generated comment: release notes by coderabbit.ai -->

---------

Co-authored-by: Carlos Villela <cvillela@nvidia.com>

Author: ericksoa

src/lib/inference/local.test.ts

@@ -29,6 +29,7 @@ import {
   getOllamaWarmupCommand,
   parseOllamaList,
   parseOllamaTags,
+  probeOllamaRuntimeModelStatus,
   probeLocalProviderHealth,
   validateOllamaModel,
   validateLocalProvider,
@@ -678,6 +679,59 @@ describe("local inference helpers", () => {
     expect(validateOllamaModel("nemotron-3-nano:30b", () => "ok", undefined, captureEx)).toEqual({ ok: true });
   });
 
+  it("parses Ollama runtime status from /api/ps", () => {
+    const capture = () =>
+      JSON.stringify({
+        models: [
+          { name: "qwen3.6:35b", size_vram: 0, processor: "100% CPU" },
+        ],
+      });
+
+    expect(probeOllamaRuntimeModelStatus("qwen3.6:35b", capture)).toEqual({
+      probed: true,
+      loaded: true,
+      cpuOnly: true,
+      processor: "100% CPU",
+      sizeVram: 0,
+    });
+  });
+
+  it("fails Spark Ollama validation when the model is CPU-only after warmup", () => {
+    const payload = JSON.stringify({ model: "qwen3.6:35b", response: "hello", done: true });
+    const psOutput = JSON.stringify({
+      models: [{ name: "qwen3.6:35b", size_vram: 0, processor: "100% CPU" }],
+    });
+    const captureEx = () => ({ stdout: payload, exitCode: 0, timedOut: false });
+    const capture = (cmd: string | string[]) => {
+      const rendered = Array.isArray(cmd) ? cmd.join(" ") : cmd;
+      if (rendered.includes("/api/ps")) return psOutput;
+      return payload;
+    };
+
+    const result = validateOllamaModel("qwen3.6:35b", capture, () => true, captureEx);
+
+    expect(result.ok).toBe(false);
+    expect(result.message).toContain("CPU only");
+    expect(result.message).toContain("CUDA v13");
+  });
+
+  it("passes Spark Ollama validation when /api/ps reports GPU memory", () => {
+    const payload = JSON.stringify({ model: "qwen3.6:35b", response: "hello", done: true });
+    const psOutput = JSON.stringify({
+      models: [{ name: "qwen3.6:35b", size_vram: 24_000_000_000, processor: "100% GPU" }],
+    });
+    const captureEx = () => ({ stdout: payload, exitCode: 0, timedOut: false });
+    const capture = (cmd: string | string[]) => {
+      const rendered = Array.isArray(cmd) ? cmd.join(" ") : cmd;
+      if (rendered.includes("/api/ps")) return psOutput;
+      return payload;
+    };
+
+    const result = validateOllamaModel("qwen3.6:35b", capture, () => true, captureEx);
+
+    expect(result).toEqual({ ok: true });
+  });
+
   it("passes ollama memory validation when total RAM covers the model on unified-memory hosts", () => {
     // Simulate Spark: Ollama returns available-RAM OOM error, but total RAM is 128 GB.
     const freeOutput = "               total        used        free\nMem:          131072       120000       1000";

src/lib/inference/local.ts

@@ -628,6 +628,81 @@ export function parseOllamaTags(output: string | null | undefined): string[] {
   }
 }
 
+export interface OllamaRuntimeModelStatus {
+  probed: boolean;
+  loaded: boolean;
+  cpuOnly: boolean;
+  processor?: string;
+  sizeVram?: number;
+}
+
+function normalizeOllamaModelName(value: unknown): string {
+  return String(value || "").trim();
+}
+
+export function probeOllamaRuntimeModelStatus(
+  model: string,
+  runCaptureImpl?: RunCaptureFn,
+): OllamaRuntimeModelStatus {
+  const capture = runCaptureImpl ?? runCapture;
+  const host = getResolvedOllamaHost();
+  const output = capture(
+    [
+      "curl",
+      "-sf",
+      "--connect-timeout",
+      "3",
+      "--max-time",
+      "5",
+      `http://${host}:${OLLAMA_PORT}/api/ps`,
+    ],
+    { ignoreError: true },
+  );
+  if (!output) return { probed: false, loaded: false, cpuOnly: false };
+
+  try {
+    const parsed = JSON.parse(String(output || ""));
+    const models = Array.isArray(parsed?.models) ? parsed.models : [];
+    const target = normalizeOllamaModelName(model);
+    const loaded = models.find((entry: { name?: unknown; model?: unknown }) => {
+      return (
+        normalizeOllamaModelName(entry?.name) === target ||
+        normalizeOllamaModelName(entry?.model) === target
+      );
+    });
+    if (!loaded) return { probed: true, loaded: false, cpuOnly: false };
+
+    const rawSizeVram = Number((loaded as { size_vram?: unknown }).size_vram);
+    const hasSizeVram = Number.isFinite(rawSizeVram);
+    const processor = normalizeOllamaModelName((loaded as { processor?: unknown }).processor);
+    const mentionsGpu = /\bGPU\b/i.test(processor);
+    const processorCpuOnly = /\bCPU\b/i.test(processor) && !mentionsGpu;
+    const sizeVramCpuOnly = hasSizeVram && rawSizeVram === 0 && !mentionsGpu;
+
+    return {
+      probed: true,
+      loaded: true,
+      cpuOnly: processorCpuOnly || sizeVramCpuOnly,
+      ...(processor ? { processor } : {}),
+      ...(hasSizeVram ? { sizeVram: rawSizeVram } : {}),
+    };
+  } catch {
+    return { probed: true, loaded: false, cpuOnly: false };
+  }
+}
+
+function formatOllamaCpuOnlyDiagnostic(model: string, status: OllamaRuntimeModelStatus): string {
+  const observed: string[] = [];
+  if (status.processor) observed.push(`processor=${status.processor}`);
+  if (status.sizeVram !== undefined) observed.push(`size_vram=${status.sizeVram}`);
+  const observedText = observed.length > 0 ? ` (${observed.join(", ")})` : "";
+  return (
+    `Selected Ollama model '${model}' answered the local probe, but Ollama reports it is loaded on CPU only${observedText}. ` +
+    "DGX Spark should use the CUDA v13 backend; check `ollama ps`, `sudo systemctl cat ollama`, " +
+    "and `journalctl -u ollama.service --since \"10 min ago\" | grep -iE \"gpu|cuda|vram|compute|library\"`, then retry onboarding."
+  );
+}
+
 export function getOllamaModelOptions(runCaptureImpl?: RunCaptureFn): string[] {
   const capture = runCaptureImpl ?? runCapture;
   const host = getResolvedOllamaHost();
@@ -750,13 +825,14 @@ export function validateOllamaModel(
   const capture = runCaptureImpl ?? runCapture;
   const captureEx = runCaptureExImpl ?? runCaptureEx;
   const isSpark = isSparkImpl ?? (() => detectNvidiaPlatform() === "spark");
+  const sparkHost = isSpark();
   const probeCmd = getOllamaProbeCommand(model);
   const probeResult = captureEx(probeCmd);
   let output = probeResult.stdout;
   // On DGX Spark (128 GB unified memory), loading a large model from disk can take >2 min.
   // Only retry with a 300 s timeout when the initial probe genuinely timed out — fast
   // failures (connection refused, Ollama not running) surface immediately. (#3251)
-  if (isSpark() && probeResult.timedOut) {
+  if (sparkHost && probeResult.timedOut) {
     const retryResult = captureEx(getOllamaProbeCommand(model, 300));
     output = retryResult.stdout;
   }
@@ -787,7 +863,7 @@ export function validateOllamaModel(
       const memMatch = errText.match(
         /model requires more system memory \(([0-9.]+)\s*GiB\) than is available \([0-9.]+\s*GiB\)/i,
       );
-      if (memMatch && isSpark()) {
+      if (memMatch && sparkHost) {
         const requiresGiB = parseFloat(memMatch[1]);
         const freeOut = capture(["free", "-m"], { ignoreError: true });
         if (freeOut) {
@@ -810,6 +886,16 @@ export function validateOllamaModel(
     /* ignored */
   }
 
+  if (sparkHost) {
+    const runtimeStatus = probeOllamaRuntimeModelStatus(model, capture);
+    if (runtimeStatus.cpuOnly) {
+      return {
+        ok: false,
+        message: formatOllamaCpuOnlyDiagnostic(model, runtimeStatus),
+      };
+    }
+  }
+
   return { ok: true };
 }
 

src/lib/onboard.ts

@@ -19,9 +19,7 @@ const { cleanupTempDir }: typeof import("./onboard/temp-files") = require("./onb
 const { stopStaleDashboardListenersForSandbox } = require("./onboard/stale-gateway-cleanup");
 const { bestEffortForwardStop } = require("./onboard/forward-cleanup");
 const { looksLikeForwardPortConflict, runBackgroundForwardStartWithPortReleaseRetries }: typeof import("./onboard/forward-start") = require("./onboard/forward-start");
-const {
-  ensureOllamaLoopbackSystemdOverride,
-}: typeof import("./onboard/ollama-systemd") = require("./onboard/ollama-systemd");
+const { ensureManagedOllamaLoopbackSystemdOverride, ensureOllamaLoopbackSystemdOverride }: typeof import("./onboard/ollama-systemd") = require("./onboard/ollama-systemd");
 const {
   CUSTOM_BUILD_CONTEXT_WARN_BYTES,
   isInsideIgnoredCustomBuildContextPath,
@@ -7064,7 +7062,7 @@ async function setupNim(
           // daemon with our own `ollama serve`). This also repairs older
           // NemoClaw-created overrides that exposed raw Ollama on all interfaces.
           // WSL and non-systemd Linux fall back to a manual loopback launch.
-          const overrideState = ensureOllamaLoopbackSystemdOverride({ isNonInteractive });
+          const overrideState = ensureManagedOllamaLoopbackSystemdOverride({ isNonInteractive });
           if (overrideState === "failed") {
             console.error(
               "  Ollama systemd restart did not recover after applying the loopback override.",

src/lib/onboard/docker-driver-gateway-launch.test.ts

@@ -8,6 +8,7 @@ import path from "node:path";
 import { describe, expect, it } from "vitest";
 
 import {
+  buildDockerDriverGatewayConfigToml,
   buildDockerDriverGatewayLaunch,
   parseGlibcVersionsFromBinaryText,
   shouldUseContainerizedGateway,
@@ -110,15 +111,41 @@ describe("docker-driver-gateway-launch", () => {
           "OPENSHELL_DRIVERS",
           "--env",
           "OPENSHELL_DOCKER_SUPERVISOR_BIN",
+          "--env",
+          "OPENSHELL_GATEWAY_CONFIG",
           "ubuntu:24.04",
           "/opt/nemoclaw/openshell-gateway",
         ]),
       );
       expect(launch.env.OPENSHELL_DOCKER_SUPERVISOR_BIN).toBe(sandboxBin);
       expect(launch.env.OPENSHELL_BIND_ADDRESS).toBe("0.0.0.0");
+      const configPath = launch.env.OPENSHELL_GATEWAY_CONFIG;
+      expect(configPath).toBe(path.join(stateDir, "openshell-gateway.toml"));
+      expect(configPath).toBeDefined();
+      if (!configPath) throw new Error("expected generated gateway config path");
+      expect(fs.readFileSync(configPath, "utf-8")).toContain(`supervisor_bin = "${sandboxBin}"`);
     });
   });
 
+  it("writes Docker driver settings in gateway TOML because OpenShell driver config is not env-backed", () => {
+    const toml = buildDockerDriverGatewayConfigToml(
+      {
+        OPENSHELL_GRPC_ENDPOINT: "http://127.0.0.1:8080",
+        OPENSHELL_DOCKER_NETWORK_NAME: "openshell-docker",
+        OPENSHELL_DOCKER_SUPERVISOR_IMAGE: "ghcr.io/nvidia/openshell/supervisor:0.0.44",
+      },
+      "/home/shadeform/.local/bin/openshell-sandbox",
+    );
+
+    expect(toml).toContain('compute_drivers = ["docker"]');
+    expect(toml).toContain('grpc_endpoint = "http://127.0.0.1:8080"');
+    expect(toml).toContain('network_name = "openshell-docker"');
+    expect(toml).toContain(
+      'supervisor_image = "ghcr.io/nvidia/openshell/supervisor:0.0.44"',
+    );
+    expect(toml).toContain('supervisor_bin = "/home/shadeform/.local/bin/openshell-sandbox"');
+  });
+
   it("allows the compatibility gateway bind address to be forced back to loopback", () => {
     withTempBinaries(({ dir, gatewayBin, sandboxBin }) => {
       const stateDir = path.join(dir, "state");

src/lib/onboard/docker-driver-gateway-launch.ts

@@ -10,6 +10,7 @@ import { dockerForceRm } from "../adapters/docker";
 const DEFAULT_COMPAT_IMAGE = "ubuntu:24.04";
 const DEFAULT_COMPAT_CONTAINER_NAME = "nemoclaw-openshell-gateway";
 const GATEWAY_MOUNT_PATH = "/opt/nemoclaw/openshell-gateway";
+const COMPAT_GATEWAY_CONFIG_NAME = "openshell-gateway.toml";
 const DEFAULT_COMPAT_BIND_ADDRESS = "0.0.0.0";
 const LOOPBACK_BIND_ADDRESS = "127.0.0.1";
 
@@ -136,6 +137,56 @@ function addEnv(args: string[], key: string, value: string | undefined): void {
   if (typeof value === "string") args.push("--env", key);
 }
 
+function tomlString(value: string): string {
+  return JSON.stringify(value);
+}
+
+export function buildDockerDriverGatewayConfigToml(
+  gatewayEnv: Record<string, string>,
+  sandboxBin: string,
+): string {
+  const dockerEntries: [string, string | undefined][] = [
+    ["grpc_endpoint", gatewayEnv.OPENSHELL_GRPC_ENDPOINT],
+    ["network_name", gatewayEnv.OPENSHELL_DOCKER_NETWORK_NAME],
+    ["supervisor_image", gatewayEnv.OPENSHELL_DOCKER_SUPERVISOR_IMAGE],
+    ["supervisor_bin", sandboxBin],
+  ];
+  const dockerConfig = dockerEntries
+    .filter(
+      (entry): entry is [string, string] =>
+        typeof entry[1] === "string" && entry[1].trim() !== "",
+    )
+    .map(([key, value]) => `${key} = ${tomlString(value)}`)
+    .join("\n");
+
+  return [
+    "[openshell]",
+    "version = 1",
+    "",
+    "[openshell.gateway]",
+    'compute_drivers = ["docker"]',
+    "",
+    "[openshell.drivers.docker]",
+    dockerConfig,
+    "",
+  ].join("\n");
+}
+
+function writeDockerDriverGatewayConfig(
+  stateDir: string,
+  gatewayEnv: Record<string, string>,
+  sandboxBin: string,
+): string {
+  const configPath = path.join(stateDir, COMPAT_GATEWAY_CONFIG_NAME);
+  fs.mkdirSync(stateDir, { recursive: true, mode: 0o700 });
+  fs.writeFileSync(configPath, buildDockerDriverGatewayConfigToml(gatewayEnv, sandboxBin), {
+    encoding: "utf-8",
+    mode: 0o600,
+  });
+  fs.chmodSync(configPath, 0o600);
+  return configPath;
+}
+
 function safeDockerName(value: string | undefined, fallback: string): string {
   const candidate = String(value || "").trim();
   if (!candidate) return fallback;
@@ -199,6 +250,8 @@ export function buildDockerDriverGatewayLaunch(
         "Re-run the NemoClaw installer or set NEMOCLAW_OPENSHELL_SANDBOX_BIN.",
     );
   }
+  const configPath = writeDockerDriverGatewayConfig(options.stateDir, gatewayEnv, sandboxBin);
+  env.OPENSHELL_GATEWAY_CONFIG = configPath;
 
   const image = safeDockerImage(env.NEMOCLAW_OPENSHELL_GATEWAY_COMPAT_IMAGE, DEFAULT_COMPAT_IMAGE);
   const containerName = safeDockerName(
@@ -227,6 +280,7 @@ export function buildDockerDriverGatewayLaunch(
   for (const key of Object.keys(gatewayEnv).sort()) {
     addEnv(args, key, gatewayEnv[key]);
   }
+  addEnv(args, "OPENSHELL_GATEWAY_CONFIG", env.OPENSHELL_GATEWAY_CONFIG);
   addEnv(args, "DOCKER_HOST", dockerHost);
   addEnv(args, "RUST_LOG", env.RUST_LOG);
   args.push(image, GATEWAY_MOUNT_PATH);
@@ -264,6 +318,9 @@ export function buildDockerDriverGatewayRuntimeIdentity(
               ([key, val]) => key in options.gatewayEnv && typeof val === "string",
             ) as [string, string][],
           ),
+          ...(typeof launch.env.OPENSHELL_GATEWAY_CONFIG === "string"
+            ? { OPENSHELL_GATEWAY_CONFIG: launch.env.OPENSHELL_GATEWAY_CONFIG }
+            : {}),
         }
       : options.gatewayEnv;
   return {