work in progress working on SEA distributable binary of project runner

Author: williamstein

.gitignore

@@ -183,4 +183,5 @@ g
 *.pkg
 *.zip
 
-src/packages/lite/build
+src/packages/lite/build
+src/packages/project-runner/build

src/packages/lite/sea/build-sea.sh

@@ -52,7 +52,7 @@ case "$OS" in
     ;;
 esac
 
-rm cocalc.js cocalc-lite.tar.xz
+rm cocalc.js cocalc-lite.tar.xz sea-prep.blob
 
 mkdir -p ../build/sea
 mv $TARGET ../build/sea

src/packages/lite/sea/cocalc-template.js

@@ -9,10 +9,11 @@ const os = require("node:os");
 const path = require("node:path");
 const { spawnSync } = require("node:child_process");
 
-// Choose where to extract (version this if you’ll update assets)
+// Choose where to extract
 const destDir = path.join(
   process.env.XDG_CACHE_HOME || path.join(os.homedir(), ".cache"),
-  "cocalc-lite",
+  "cocalc",
+  "lite",
   VERSION,
 );
 
@@ -35,12 +36,12 @@ if (!fs.existsSync(stamp)) {
   );
 
   if (child.error) {
-    console.error("Failed to run tar:", r.error);
+    console.error("Failed to run tar:", child.error);
     process.exit(1);
   }
   if (child.status !== 0) {
-    console.error(`tar exited with code ${r.status}`);
-    process.exit(r.status);
+    console.error(`tar exited with code ${child.status}`);
+    process.exit(child.status);
   }
 
   console.log("Assets ready at:", destDir);
@@ -62,9 +63,7 @@ process.argv = [process.execPath, script, ...process.argv.slice(2)];
 
 // make sure PATH (and any other env) includes your extracted tools
 process.env.PATH =
-  path.join(destDir, "cocalc-lite/lite/bin") +
-  path.delimiter +
-  process.env.PATH;
+  path.join(destDir, "lite/bin") + path.delimiter + process.env.PATH;
 
 process.env.AUTH_TOKEN = "random";
 

src/packages/project-runner/run/env.ts

@@ -3,7 +3,7 @@ import { join } from "node:path";
 import base_path from "@cocalc/backend/base-path";
 
 export function dataPath(HOME: string): string {
-  return join(HOME, ".cache", "cocalc");
+  return join(HOME, ".cache", "cocalc", "project");
 }
 
 // see also packages/project/secret-token.ts
@@ -91,6 +91,7 @@ export function getEnvironment({
       COCALC_USERNAME: USER,
       USER,
       COCALC_EXTRA_ENV: extra_env,
+      // probably want to be more careful with PATH
       PATH: `${HOME}/bin:${HOME}/.local/bin:${process.env.PATH}`,
       CONAT_SERVER: conatServer,
       COCALC_SECRET_TOKEN: secretTokenPath(HOME),

src/packages/project-runner/run/index.ts

@@ -34,7 +34,7 @@ import { mkdir } from "fs/promises";
 import { exists } from "@cocalc/backend/misc/async-utils-node";
 import { spawn } from "node:child_process";
 import { type Configuration } from "./types";
-export { type Configuration};
+export { type Configuration };
 import { limits } from "./limits";
 import {
   client as createFileClient,
@@ -116,7 +116,7 @@ async function start({
   if (!isValidUUID(project_id)) {
     throw Error("start: project_id must be valid");
   }
-  logger.debug("start", { project_id, config });
+  logger.debug("start", { project_id, config: { ...config, secret: "xxx" } });
   if (children[project_id] != null && children[project_id].exitCode == null) {
     logger.debug("start -- already running");
     return;
@@ -138,6 +138,7 @@ async function start({
     env: config?.env,
     HOME: home,
   });
+  env.PATH = dirname(process.argv[0]) + ":" + (env.PATH ?? "");
   await setupDataPath(home);
   if (config?.secret) {
     await writeSecretToken(home, config.secret);

src/packages/project-runner/sea/build-sea.sh

@@ -0,0 +1,62 @@
+#!/usr/bin/env bash
+set -Eeuo pipefail
+
+# --- config ---
+NAME="cocalc-project-runner"
+
+export VERSION="$npm_package_version"
+FUSE="NODE_SEA_FUSE_fce680ab2cc467b6e072b8b5df1996b2"   # must match your sea-config.json
+MACHINE="$(uname -m)"
+OS="$(uname -s | tr '[:upper:]' '[:lower:]')"
+
+TARGET="./$NAME-$VERSION-$MACHINE-$OS"
+
+NODE_BIN="$(command -v node)"
+
+echo "Building CoCalc Project Runner SEA for $OS"
+
+# 1) Stage the node runtime we’ll inject into
+cp "$NODE_BIN" "$TARGET"
+chmod u+w "$TARGET"
+
+cp ../build/tarball/$NAME-$VERSION-$MACHINE-$OS.tar.xz $NAME.tar.xz
+envsubst < cocalc-template.js > cocalc.js
+
+# 2) Bundle app into a SEA blob
+#    This writes ./sea-prep.blob using your sea-config.json
+node --experimental-sea-config sea-config.json
+
+# 3) Platform-specific injection and signing
+case "$OS" in
+  darwin)
+    # Remove existing signature before mutation (ok if it fails on already-unsigned copy)
+    codesign --remove-signature "$TARGET" || true
+
+    # Inject the SEA blob into the Mach-O binary, specifying the segment name for macOS
+    npx -y postject "$TARGET" NODE_SEA_BLOB ./sea-prep.blob \
+      --sentinel-fuse "$FUSE" \
+      --macho-segment-name NODE_SEA
+
+    # Re-sign ad-hoc so macOS will run it
+    codesign --force --sign - "$TARGET"
+    ;;
+
+  linux)
+    # Inject into the ELF binary (no Mach-O segment flag on Linux)
+    npx -y postject "$TARGET" NODE_SEA_BLOB ./sea-prep.blob \
+      --sentinel-fuse "$FUSE"
+    ;;
+
+  *)
+    echo "Unsupported OS: $OS" >&2
+    exit 2
+    ;;
+esac
+
+rm cocalc.js $NAME.tar.xz sea-prep.blob
+
+mkdir -p ../build/sea
+mv $TARGET ../build/sea
+
+
+echo "Done. Built $TARGET"

src/packages/project-runner/sea/build-tarball.sh

@@ -90,10 +90,8 @@ rm -rf better-sqlite3@*/node_modules/better-sqlite3/deps/sqlite3
 
 cd "$SRC"/..
 rm -rf *.md .github .git docs
-mv src/packages/* .
-rm -rf src
 # remove rustic -- only needed for file server
-rm -f backend/node_modules/.bin/rustic
+rm -f src/packages/backend/node_modules/.bin/rustic
 
 cd $TMP
 mkdir -p $BIN/../build/tarball

src/packages/project-runner/sea/cocalc-template.js

@@ -0,0 +1,78 @@
+// cocalc.js
+const VERSION = "${VERSION}";
+
+const { getRawAsset } = require("node:sea");
+const fs = require("node:fs");
+const os = require("node:os");
+const path = require("node:path");
+const { spawnSync } = require("node:child_process");
+
+const destDir = path.join(
+  process.env.XDG_CACHE_HOME || path.join(os.homedir(), ".cache"),
+  "cocalc",
+  "project-runner",
+  VERSION,
+);
+
+const stamp = path.join(destDir, ".ok");
+if (!fs.existsSync(stamp)) {
+  console.log("Unpacking...");
+  // Read the SEA asset into a Buffer
+  const ab = getRawAsset("cocalc-project-runner.tar.xz"); // ArrayBuffer (no copy)
+  const buf = Buffer.from(new Uint8Array(ab)); // turn into Node Buffer
+
+  fs.mkdirSync(destDir, { recursive: true });
+
+  const child = spawnSync(
+    "tar",
+    ["-Jxf", "-", "-C", destDir, "--strip-components=1"],
+    {
+      input: buf,
+      stdio: ["pipe", "inherit", "inherit"],
+    },
+  );
+
+  if (child.error) {
+    console.error("Failed to run tar:", child.error);
+    process.exit(1);
+  }
+  if (child.status !== 0) {
+    console.error(`tar exited with code ${child.status}`);
+    process.exit(child.status);
+  }
+
+  console.log("Assets ready at:", destDir);
+  fs.writeFileSync(stamp, "");
+}
+
+const Module = require("node:module");
+
+const looksLikeScript = process.argv[2] && !process.argv[2].startsWith("-");
+
+if (looksLikeScript) {
+  process.argv = [process.execPath, ...process.argv.slice(2)];
+} else {
+  console.log("Starting CoCalc Project Runner");
+
+  const script = path.join(destDir, "src/packages/project-runner/bin/start.js");
+
+  if (!fs.existsSync(script)) {
+    console.error("missing start.js at", script);
+    process.exit(1);
+  }
+
+  // set up argv and cwd as if launched directly
+  process.chdir(path.dirname(script));
+  process.argv = [process.execPath, script, ...process.argv.slice(2)];
+
+  // make sure PATH (and any other env) includes your extracted tools
+  process.env.PATH =
+    path.join(destDir, "src/packages/project-runner/bin/") +
+    path.delimiter +
+    process.env.PATH;
+
+  process.env.AUTH_TOKEN = "random";
+}
+
+// run like “node start.js”
+Module.runMain(); // loads process.argv[1] as the main script

src/packages/project-runner/sea/sea-config.json

@@ -0,0 +1,9 @@
+{
+  "main": "cocalc.js",
+  "output": "sea-prep.blob",
+  "disableExperimentalSEAWarning": true,
+  "useSnapshot": false,
+  "assets": {
+    "cocalc-project-runner.tar.xz": "cocalc-project-runner.tar.xz"
+  }
+}