project-runner: implement overlay so that rootfs changes persist

Author: williamstein

src/packages/project-runner/package.json

@@ -12,6 +12,7 @@
     "depcheck": "pnpx depcheck",
     "clean": "rm -rf node_modules dist",
     "start": "pnpm exec cocalc-project-runner",
+    "build-all": "pnpm build && pnpm build-tarball && pnpm build-sea",
     "build-tarball": "cd sea && ./build-tarball.sh",
     "build-sea": "cd sea && ./build-sea.sh"
   },

src/packages/project-runner/run/overlay.ts

@@ -0,0 +1,154 @@
+import { join } from "path";
+import { data } from "@cocalc/backend/data";
+import { exists } from "@cocalc/backend/misc/async-utils-node";
+import { executeCode } from "@cocalc/backend/execute-code";
+import { mkdir, rm, writeFile } from "fs/promises";
+import { type Configuration } from "./types";
+import { reuseInFlight } from "@cocalc/util/reuse-in-flight";
+import { replace_all } from "@cocalc/util/misc";
+
+const DEFAULT_IMAGE = "ubuntu:25.04";
+
+const IMAGE_CACHE =
+  process.env.COCALC_IMAGE_CACHE ?? join(data, "cache", "images");
+const PROJECT_ROOTS =
+  process.env.COCALC_PROJECT_ROOTS ?? join(data, "cache", "project-roots");
+
+export const extractBaseImage = reuseInFlight(async (image: string) => {
+  const baseImagePath = join(IMAGE_CACHE, image);
+  const okFile = baseImagePath + ".ok";
+  if (await exists(okFile)) {
+    // already exist
+    return baseImagePath;
+  }
+  // pull it -- this takes most of the time.
+  // It is also important to do this before the unshare below,
+  // since doing it inside the unshare hits namespace issues.
+  await executeCode({
+    timeout: 60 * 60, // in seconds
+    err_on_exit: true,
+    command: "podman",
+    args: [
+      // ignore_chown_errors=true is needed since otherwise we
+      // have to make changes to the host system to allow more
+      // uid's, etc. for complicated images (e.g., sage);
+      // this is fine since we run everything as root anyways.
+      "--storage-opt",
+      "ignore_chown_errors=true",
+      "pull",
+      image,
+    ],
+  });
+  // TODO: an optimization on COW filesystem if we pull one image
+  // then pull another with a different tag, would be to start by
+  // initializing the target path using COW, then 'rsync ... --delete'
+  // to transform it to the result.  This could MASSIVELY save space.
+
+  // extract the image
+  try {
+    await executeCode({
+      timeout: 60 * 60, // timeout in seconds
+      err_on_exit: true,
+      command: "podman",
+      args: [
+        "unshare",
+        "bash",
+        "-c",
+        `
+  set -ev
+  mnt="$(podman image mount ${image})"
+  echo "mounted at: $mnt"
+  mkdir -p "${baseImagePath}"
+  rsync -aHx --numeric-ids --delete "$mnt"/ "${baseImagePath}"/
+  podman image unmount ${image}
+`,
+      ],
+    });
+  } catch (err) {
+    // fail -- clean up the mess (hopefully)
+    try {
+      await rm(baseImagePath, { force: true, recursive: true, maxRetries: 3 });
+      await executeCode({ command: "podman", args: ["image", "rm", image] });
+    } catch {}
+    throw err;
+  }
+  // success!
+  await writeFile(okFile, "");
+  // remove the image to save space, in case it isn't used by
+  // anything else.  we will not need it again, since we already
+  // have a copy of it.
+  await executeCode({ command: "podman", args: ["image", "rm", image] });
+  return baseImagePath;
+});
+
+function getMergedPath(project_id) {
+  return join(PROJECT_ROOTS, project_id);
+}
+
+function getPaths({ home, image, project_id }) {
+  const userOverlays = join(home, ".overlay", image);
+  const upper = join(userOverlays, "upper");
+  const workdir = join(userOverlays, "workdir");
+  const merged = getMergedPath(project_id);
+  return { upper, workdir, merged };
+}
+
+function getImage(config) {
+  return config?.image ?? DEFAULT_IMAGE;
+}
+
+export async function mount({
+  project_id,
+  home,
+  config,
+}: {
+  project_id: string;
+  home: string;
+  config?: Configuration;
+}) {
+  const image = getImage(config);
+  const lower = await extractBaseImage(image);
+  const { upper, workdir, merged } = getPaths({ home, image, project_id });
+  await mkdir(upper, { recursive: true });
+  await mkdir(workdir, { recursive: true });
+  await mkdir(merged, { recursive: true });
+
+  await mountOverlayFs({ lower, upper, workdir, merged });
+
+  return merged;
+}
+
+/*
+This would go in sudo for the user to allow just this:
+
+    wstein ALL=(ALL) NOPASSWD: /bin/mount -t overlay *, /bin/umount *
+*/
+
+export async function unmount(project_id: string) {
+  const mountpoint = getMergedPath(project_id);
+  await executeCode({
+    err_on_exit: true,
+    command: "sudo",
+    args: ["umount", mountpoint],
+  });
+}
+
+function escape(path) {
+  return replace_all(path, ":", `\\:`);
+}
+
+async function mountOverlayFs({ upper, workdir, merged, lower }) {
+  await executeCode({
+    err_on_exit: true,
+    command: "sudo",
+    args: [
+      "mount",
+      "-t",
+      "overlay",
+      "overlay",
+      "-o",
+      `lowerdir=${escape(lower)},upperdir=${escape(upper)},workdir=${escape(workdir)}`,
+      merged,
+    ],
+  });
+}

src/packages/project-runner/run/podman.ts

@@ -15,14 +15,13 @@ import { getCoCalcMounts, COCALC_SRC } from "./mounts";
 import { mountHome, setQuota } from "./filesystem";
 import { executeCode } from "@cocalc/backend/execute-code";
 import { join } from "path";
+import * as rootFilesystem from "./overlay";
 
 const logger = getLogger("project-runner:podman");
 const children: { [project_id: string]: any } = {};
 
 const GRACE_PERIOD = 2000;
 
-const DEFAULT_IMAGE = "ubuntu:25.04";
-
 export async function start({
   project_id,
   config,
@@ -40,6 +39,7 @@ export async function start({
   }
 
   const home = await mountHome(project_id);
+  const rootfs = await rootFilesystem.mount({ project_id, home, config });
   await mkdir(home, { recursive: true });
   await ensureConfFilesExists(home);
   const env = getEnvironment({
@@ -60,6 +60,7 @@ export async function start({
   }
 
   const args: string[] = ["run", "--rm", "--network=host", "--user=0:0"];
+
   const cmd = "podman";
   const script = join(COCALC_SRC, "/packages/project/bin/cocalc-project.js");
 
@@ -75,7 +76,7 @@ export async function start({
     args.push("-e", `${name}=${env[name]}`);
   }
 
-  args.push(config?.image ?? DEFAULT_IMAGE);
+  args.push("--rootfs", rootfs);
   args.push(nodePath);
   args.push(script, "--init", "project_init.sh");
 
@@ -110,6 +111,7 @@ export async function stop({ project_id }) {
       ]);
     } catch {}
     delete children[project_id];
+    await rootFilesystem.unmount(project_id);
   }
 }