Skip to content

Commit 27ba0a5

Browse files
williamsteinwilliamstein
committed
refactor project runner code to make it easier to have multiple runtimes
- but probably just use podman
1 parent 4a4be47 commit 27ba0a5

File tree

6 files changed

+310
-259
lines changed

6 files changed

+310
-259
lines changed

‎src/packages/project-runner/bin/start.js‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,6 @@ const { init } = require("@cocalc/project-runner/run");
44

55
(async () => {
66
console.log("Starting...");
7-
await init();
7+
await init({runtime:'nsjail'});
88
console.log("CoCalc Project Runner ready");
99
})();

‎src/packages/project-runner/run/filesystem.ts‎

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,34 @@
1+
import {
2+
client as createFileClient,
3+
type Fileserver,
4+
} from "@cocalc/conat/files/file-server";
5+
import { type Client as ConatClient } from "@cocalc/conat/core/client";
6+
7+
//import getLogger from "@cocalc/backend/logger";
8+
9+
// const logger = getLogger("project-runner:filesystem");
10+
11+
let client: ConatClient | null = null;
12+
export function init(opts: { client: ConatClient }) {
13+
client = opts.client;
14+
}
15+
16+
let fsclient: Fileserver | null = null;
17+
function getFsClient() {
18+
if (client == null) {
19+
throw Error("client not initialized");
20+
}
21+
fsclient ??= createFileClient({ client });
22+
return fsclient;
23+
}
24+
25+
export async function setQuota(project_id: string, size: number | string) {
26+
const c = getFsClient();
27+
await c.setQuota({ project_id, size });
28+
}
29+
30+
export async function mountHome(project_id: string): Promise<string> {
31+
const c = getFsClient();
32+
const { path } = await c.mount({ project_id });
33+
return path;
34+
}

‎src/packages/project-runner/run/index.ts‎

Lines changed: 22 additions & 255 deletions
Original file line numberDiff line numberDiff line change
@@ -1,253 +1,41 @@
11
/*
22
Project run server.
33
4-
It may be necessary to do this to enable the user running this
5-
code to use nsjail:
6-
7-
sudo sysctl -w kernel.apparmor_restrict_unprivileged_unconfined=0 && sudo sysctl -w kernel.apparmor_restrict_unprivileged_userns=0
8-
9-
10-
See https://github.com/google/nsjail/issues/236#issuecomment-2267096267
11-
12-
To make permanent:
13-
14-
echo -e "kernel.apparmor_restrict_unprivileged_unconfined=0\nkernel.apparmor_restrict_unprivileged_userns=0" | sudo tee /etc/sysctl.d/99-custom.conf && sudo sysctl --system
15-
16-
---
17-
184
DEV -- see packages/server/conat/project/run.ts
195
206
*/
217

228
import { type Client as ConatClient } from "@cocalc/conat/core/client";
239
import { conat } from "@cocalc/backend/conat";
2410
import { server as projectRunnerServer } from "@cocalc/conat/project/runner/run";
25-
import { isValidUUID } from "@cocalc/util/misc";
2611
import { reuseInFlight } from "@cocalc/util/reuse-in-flight";
27-
import getLogger from "@cocalc/backend/logger";
28-
import { root } from "@cocalc/backend/data";
29-
import { basename, dirname, join } from "node:path";
30-
import { userInfo } from "node:os";
31-
import { ensureConfFilesExists, setupDataPath, writeSecretToken } from "./util";
32-
import { getEnvironment } from "./env";
33-
import { mkdir } from "fs/promises";
34-
import { exists } from "@cocalc/backend/misc/async-utils-node";
35-
import { spawn } from "node:child_process";
3612
import { type Configuration } from "./types";
3713
export { type Configuration };
38-
import { limits } from "./limits";
39-
import {
40-
client as createFileClient,
41-
type Fileserver,
42-
} from "@cocalc/conat/files/file-server";
43-
import { nsjail } from "@cocalc/backend/sandbox/install";
44-
import { once } from "@cocalc/util/async-utils";
45-
46-
// for development it may be useful to just disabling using nsjail namespaces
47-
// entirely -- change this to true to do so.
48-
const DISABLE_NSJAIL = false;
49-
50-
const DEFAULT_UID = 2001;
51-
52-
// how long from SIGTERM until SIGKILL
53-
const GRACE_PERIOD = 3000;
54-
55-
const logger = getLogger("project-runner");
56-
57-
const children: { [project_id: string]: any } = {};
58-
59-
const MOUNTS = {
60-
"-R": ["/etc", "/var", "/bin", "/lib", "/usr", "/lib64", "/run"],
61-
"-B": ["/dev"],
62-
};
63-
64-
let nodePath = process.execPath;
65-
async function initMounts() {
66-
for (const type in MOUNTS) {
67-
const v: string[] = [];
68-
for (const path of MOUNTS[type]) {
69-
if (await exists(path)) {
70-
v.push(path);
71-
}
72-
}
73-
MOUNTS[type] = v;
74-
}
75-
MOUNTS["-R"].push(`${dirname(root)}:/cocalc`);
76-
77-
// also if this node is install via nvm, we make exactly this
78-
// version of node's install available
79-
if (!process.execPath.startsWith("/usr/")) {
80-
// not already in an obvious system-wide place we included above
81-
// IMPORTANT: take care not to put the binary next to sensitive info!
82-
MOUNTS["-R"].push(`${dirname(process.execPath)}:/cocalc/bin`);
83-
nodePath = join("/cocalc/bin", basename(process.execPath));
84-
}
85-
}
86-
87-
let fsclient: Fileserver | null = null;
88-
function getFsClient() {
89-
if (client == null) {
90-
throw Error("not initialized");
91-
}
92-
fsclient ??= createFileClient({ client });
93-
return fsclient;
94-
}
95-
96-
async function setQuota(project_id: string, size: number | string) {
97-
const c = getFsClient();
98-
await c.setQuota({ project_id, size });
99-
}
100-
101-
async function mountHome(project_id: string): Promise<string> {
102-
const c = getFsClient();
103-
const { path } = await c.mount({ project_id });
104-
return path;
105-
}
106-
107-
async function start({
108-
project_id,
109-
config,
110-
}: {
111-
project_id: string;
112-
config?: Configuration;
113-
}) {
114-
if (!isValidUUID(project_id)) {
115-
throw Error("start: project_id must be valid");
116-
}
117-
logger.debug("start", { project_id, config: { ...config, secret: "xxx" } });
118-
if (children[project_id] != null && children[project_id].exitCode == null) {
119-
logger.debug("start -- already running");
120-
return;
121-
}
122-
let uid, gid;
123-
if (userInfo().uid) {
124-
// server running as non-root user -- single user mode
125-
uid = gid = userInfo().uid;
126-
} else {
127-
// server is running as root -- multiuser mode
128-
uid = gid = DEFAULT_UID;
129-
}
130-
131-
const home = await mountHome(project_id);
132-
await mkdir(home, { recursive: true });
133-
await ensureConfFilesExists(home);
134-
const env = getEnvironment({
135-
project_id,
136-
env: config?.env,
137-
HOME: home,
138-
});
139-
await setupDataPath(home);
140-
if (config?.secret) {
141-
await writeSecretToken(home, config.secret);
142-
}
143-
144-
if (config?.disk) {
145-
// TODO: maybe this should be done in parallel with other things
146-
// to make startup time slightly faster (?) -- could also be incorporated
147-
// into mount.
148-
await setQuota(project_id, config.disk);
149-
}
150-
151-
let script: string,
152-
cmd: string,
153-
args: string[] = [];
154-
if (DISABLE_NSJAIL) {
155-
// DANGEROUS: no safety at all here!
156-
// This may be useful in some environments, especially for debugging.
157-
cmd = process.execPath;
158-
script = join(root, "packages/project/bin/cocalc-project.js");
159-
} else {
160-
script = "/cocalc/src/packages/project/bin/cocalc-project.js";
161-
args.push(
162-
"-q", // not too verbose
163-
"-Mo", // run a command once
164-
"--disable_clone_newnet", // [ ] TODO: for now we have the full host network
165-
"--keep_env", // this just keeps env
166-
"--keep_caps", // [ ] TODO: maybe NOT needed!
167-
"--skip_setsid", // evidently needed for terminal signals (e.g., ctrl+z); dangerous. [ ] TODO -- really needed?
168-
);
169-
170-
args.push("--hostname", `project-${env.COCALC_PROJECT_ID}`);
171-
172-
if (uid != null && gid != null) {
173-
args.push("-u", `${uid}`, "-g", `${gid}`);
174-
}
175-
176-
for (const type in MOUNTS) {
177-
for (const path of MOUNTS[type]) {
178-
args.push(type, path);
179-
}
180-
}
181-
// need a /tmp directory
182-
args.push("-m", "none:/tmp:tmpfs:size=500000000");
183-
184-
args.push("-B", `${home}:${env.HOME}`);
185-
args.push(...limits(config));
186-
args.push("--");
187-
args.push(nodePath);
188-
cmd = nsjail;
189-
}
190-
191-
args.push(script, "--init", "project_init.sh");
192-
193-
//logEnv(env);
194-
// console.log(`${cmd} ${args.join(" ")}`);
195-
logger.debug(`${cmd} ${args.join(" ")}`);
196-
const child = spawn(cmd, args, {
197-
env,
198-
uid,
199-
gid: uid,
200-
});
201-
children[project_id] = child;
202-
203-
child.stdout.on("data", (chunk: Buffer) => {
204-
logger.debug(`project_id=${project_id}.stdout: `, chunk.toString());
205-
});
206-
child.stderr.on("data", (chunk: Buffer) => {
207-
logger.debug(`project_id=${project_id}.stderr: `, chunk.toString());
208-
});
209-
}
14+
import { init as initFilesystem } from "./filesystem";
15+
import getLogger from "@cocalc/backend/logger";
16+
import * as nsjail from "./nsjail";
17+
import { init as initMounts } from "./mounts";
21018

211-
async function stop({ project_id }) {
212-
if (!isValidUUID(project_id)) {
213-
throw Error("stop: project_id must be valid");
214-
}
215-
logger.debug("stop", { project_id });
216-
const child = children[project_id];
217-
if (child != null && child.exitCode == null) {
218-
const exit = once(child, "exit", GRACE_PERIOD);
219-
child.kill("SIGTERM");
220-
try {
221-
await exit;
222-
} catch {
223-
const exit2 = once(child, "exit");
224-
child.kill("SIGKILL");
225-
await exit2;
226-
}
227-
delete children[project_id];
228-
}
229-
}
230-
231-
async function status({ project_id }) {
232-
if (!isValidUUID(project_id)) {
233-
throw Error("status: project_id must be valid");
234-
}
235-
logger.debug("status", { project_id });
236-
let state;
237-
if (children[project_id] == null || children[project_id].exitCode) {
238-
state = "opened";
239-
} else {
240-
state = "running";
241-
}
242-
// [ ] TODO: ip -- need to figure out the networking story for running projects
243-
// The following will only work on a single machine with global network address space
244-
return { state, ip: "127.0.0.1" };
245-
}
19+
const logger = getLogger("project-runner:run");
24620

24721
let client: ConatClient | null = null;
248-
export async function init(opts: { client?: ConatClient } = {}) {
22+
export async function init(
23+
opts: { client?: ConatClient; runtime?: "nsjail" | "podman" } = {},
24+
) {
25+
logger.debug("init", opts.runtime);
26+
let runtime;
27+
switch (opts.runtime) {
28+
case "nsjail":
29+
runtime = nsjail;
30+
break;
31+
default:
32+
throw Error(`runtime '${opts.runtime}' not implemented`);
33+
}
24934
client = opts.client ?? conat();
35+
initFilesystem({ client });
25036
await initMounts();
37+
38+
const { start, stop, status } = runtime;
25139
return await projectRunnerServer({
25240
client,
25341
start: reuseInFlight(start),
@@ -256,27 +44,6 @@ export async function init(opts: { client?: ConatClient } = {}) {
25644
});
25745
}
25846

259-
export function killAllProjects() {
260-
for (const project_id in children) {
261-
logger.debug(`killing project_id=${project_id}`);
262-
children[project_id]?.kill("SIGKILL");
263-
delete children[project_id];
264-
}
47+
export function close() {
48+
nsjail.close();
26549
}
266-
267-
// important to killAllProjects, because it kills all
268-
// the processes that were spawned
269-
process.once("exit", killAllProjects);
270-
["SIGINT", "SIGTERM", "SIGQUIT"].forEach((sig) => {
271-
process.once(sig, () => {
272-
process.exit();
273-
});
274-
});
275-
276-
// function logEnv(env) {
277-
// let s = "export ";
278-
// for (const key in env) {
279-
// s += `${key}="${env[key]}" `;
280-
// }
281-
// console.log(s);
282-
// }

‎src/packages/project-runner/run/mounts.ts‎

Lines changed: 46 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,46 @@
1+
import getLogger from "@cocalc/backend/logger";
2+
import { basename, dirname, join } from "node:path";
3+
import { root } from "@cocalc/backend/data";
4+
import { exists } from "@cocalc/backend/misc/async-utils-node";
5+
6+
const logger = getLogger("project-runner:mounts");
7+
8+
const MOUNTS = {
9+
"-R": ["/etc", "/var", "/bin", "/lib", "/usr", "/lib64", "/run"],
10+
"-B": ["/dev"],
11+
};
12+
13+
export let nodePath = process.execPath;
14+
let initialized = false;
15+
export async function init() {
16+
if (initialized) {
17+
return;
18+
}
19+
logger.debug("init");
20+
initialized = true;
21+
for (const type in MOUNTS) {
22+
const v: string[] = [];
23+
for (const path of MOUNTS[type]) {
24+
if (await exists(path)) {
25+
v.push(path);
26+
}
27+
}
28+
MOUNTS[type] = v;
29+
}
30+
MOUNTS["-R"].push(`${dirname(root)}:/cocalc`);
31+
32+
// also if this node is install via nvm, we make exactly this
33+
// version of node's install available
34+
if (!process.execPath.startsWith("/usr/")) {
35+
// not already in an obvious system-wide place we included above
36+
// IMPORTANT: take care not to put the binary next to sensitive info!
37+
MOUNTS["-R"].push(`${dirname(process.execPath)}:/cocalc/bin`);
38+
nodePath = join("/cocalc/bin", basename(process.execPath));
39+
}
40+
logger.debug(MOUNTS);
41+
}
42+
43+
export async function getMounts() {
44+
await init();
45+
return MOUNTS;
46+
}

0 commit comments

Comments
 (0)