working on adding conat fileserver support

Author: williamstein

src/packages/conat/files/fs.ts

@@ -0,0 +1,134 @@
+import { type Client } from "@cocalc/conat/core/client";
+import { conat } from "@cocalc/conat/client";
+
+export interface Filesystem {
+  appendFile: (path: string, data: string | Buffer, encoding?) => Promise<void>;
+  chmod: (path: string, mode: string | number) => Promise<void>;
+  copyFile: (src: string, dest: string) => Promise<void>;
+  cp: (src: string, dest: string, options?) => Promise<void>;
+  exists: (path: string) => Promise<void>;
+  link: (existingPath: string, newPath: string) => Promise<void>;
+  mkdir: (path: string, options?) => Promise<void>;
+  readFile: (path: string, encoding?: any) => Promise<string | Buffer>;
+  readdir: (path: string) => Promise<string[]>;
+  realpath: (path: string) => Promise<string>;
+  rename: (oldPath: string, newPath: string) => Promise<void>;
+  rm: (path: string, options?) => Promise<void>;
+  rmdir: (path: string, options?) => Promise<void>;
+  stat: (path: string) => Promise<Stats>;
+  symlink: (target: string, path: string) => Promise<void>;
+  truncate: (path: string, len?: number) => Promise<void>;
+  unlink: (path: string) => Promise<void>;
+  utimes: (
+    path: string,
+    atime: number | string | Date,
+    mtime: number | string | Date,
+  ) => Promise<void>;
+  writeFile: (path: string, data: string | Buffer) => Promise<void>;
+}
+
+export interface Stats {
+  dev: number;
+  ino: number;
+  mode: number;
+  nlink: number;
+  uid: number;
+  gid: number;
+  rdev: number;
+  size: number;
+  blksize: number;
+  blocks: number;
+  atimeMs: number;
+  mtimeMs: number;
+  ctimeMs: number;
+  birthtimeMs: number;
+  atime: Date;
+  mtime: Date;
+  ctime: Date;
+  birthtime: Date;
+}
+
+interface Options {
+  service: string;
+  client?: Client;
+  fs: (subject?: string) => Promise<Filesystem>;
+}
+
+export async function fsServer({ service, fs, client }: Options) {
+  return await (client ?? conat()).service<Filesystem & { subject?: string }>(
+    `${service}.*`,
+    {
+      async appendFile(path: string, data: string | Buffer, encoding?) {
+        await (await fs(this.subject)).appendFile(path, data, encoding);
+      },
+      async chmod(path: string, mode: string | number) {
+        await (await fs(this.subject)).chmod(path, mode);
+      },
+      async copyFile(src: string, dest: string) {
+        await (await fs(this.subject)).copyFile(src, dest);
+      },
+      async cp(src: string, dest: string, options?) {
+        await (await fs(this.subject)).cp(src, dest, options);
+      },
+      async exists(path: string) {
+        await (await fs(this.subject)).exists(path);
+      },
+      async link(existingPath: string, newPath: string) {
+        await (await fs(this.subject)).link(existingPath, newPath);
+      },
+      async mkdir(path: string, options?) {
+        await (await fs(this.subject)).mkdir(path, options);
+      },
+      async readFile(path: string, encoding?) {
+        return await (await fs(this.subject)).readFile(path, encoding);
+      },
+      async readdir(path: string) {
+        return await (await fs(this.subject)).readdir(path);
+      },
+      async realpath(path: string) {
+        return await (await fs(this.subject)).realpath(path);
+      },
+      async rename(oldPath: string, newPath: string) {
+        return await (await fs(this.subject)).rename(oldPath, newPath);
+      },
+      async rm(path: string, options?) {
+        return await (await fs(this.subject)).rm(path, options);
+      },
+      async rmdir(path: string, options?) {
+        return await (await fs(this.subject)).rmdir(path, options);
+      },
+      async stat(path: string): Promise<Stats> {
+        return await (await fs(this.subject)).stat(path);
+      },
+      async symlink(target: string, path: string) {
+        return await (await fs(this.subject)).symlink(target, path);
+      },
+      async truncate(path: string, len?: number) {
+        return await (await fs(this.subject)).truncate(path, len);
+      },
+      async unlink(path: string) {
+        return await (await fs(this.subject)).unlink(path);
+      },
+      async utimes(
+        path: string,
+        atime: number | string | Date,
+        mtime: number | string | Date,
+      ) {
+        return await (await fs(this.subject)).utimes(path, atime, mtime);
+      },
+      async writeFile(path: string, data: string | Buffer) {
+        return await (await fs(this.subject)).writeFile(path, data);
+      },
+    },
+  );
+}
+
+export function fsClient({
+  client,
+  subject,
+}: {
+  client?: Client;
+  subject: string;
+}) {
+  return (client ?? conat()).call<Filesystem>(subject);
+}

src/packages/file-server/btrfs/subvolume-bup.ts

@@ -46,7 +46,7 @@ export class SubvolumeBup {
         `createBackup: creating ${BUP_SNAPSHOT} to get a consistent backup`,
       );
       await this.subvolume.snapshots.create(BUP_SNAPSHOT);
-      const target = this.subvolume.normalize(
+      const target = this.subvolume.fs.safeAbsPath(
         this.subvolume.snapshots.path(BUP_SNAPSHOT),
       );
 
@@ -133,7 +133,9 @@ export class SubvolumeBup {
       return v;
     }
 
-    path = normalize(path);
+    path = this.subvolume.fs
+      .safeAbsPath(path)
+      .slice(this.subvolume.path.length);
     const { stdout } = await sudo({
       command: "bup",
       args: [

src/packages/file-server/btrfs/subvolume.ts

@@ -4,12 +4,12 @@ A subvolume
 
 import { type Filesystem, DEFAULT_SUBVOLUME_SIZE } from "./filesystem";
 import refCache from "@cocalc/util/refcache";
-import { sudo } from "./util";
-import { join, normalize } from "path";
+import { isdir, sudo } from "./util";
+import { join } from "path";
 import { SubvolumeBup } from "./subvolume-bup";
 import { SubvolumeSnapshots } from "./subvolume-snapshots";
 import { SubvolumeQuota } from "./subvolume-quota";
-import { SandboxedFilesystem } from "../fs";
+import { SandboxedFilesystem } from "../fs/sandbox";
 import { exists } from "@cocalc/backend/misc/async-utils-node";
 
 import getLogger from "@cocalc/backend/logger";
@@ -77,11 +77,35 @@ export class Subvolume {
     });
   };
 
-  // this should provide a path that is guaranteed to be
-  // inside this.path on the filesystem or throw error
-  // [ ] TODO: not sure if the code here is sufficient!!
-  normalize = (path: string) => {
-    return join(this.path, normalize(path));
+  rsync = async ({
+    src,
+    target,
+    timeout = 5 * 60 * 1000,
+  }: {
+    src: string;
+    target: string;
+    timeout?: number;
+  }): Promise<{ stdout: string; stderr: string; exit_code: number }> => {
+    let srcPath = this.fs.safeAbsPath(src);
+    let targetPath = this.fs.safeAbsPath(target);
+    if (src.endsWith("/")) {
+      srcPath += "/";
+    }
+    if (target.endsWith("/")) {
+      targetPath += "/";
+    }
+    if (!srcPath.endsWith("/") && (await isdir(srcPath))) {
+      srcPath += "/";
+      if (!targetPath.endsWith("/")) {
+        targetPath += "/";
+      }
+    }
+    return await sudo({
+      command: "rsync",
+      args: [srcPath, targetPath],
+      err_on_exit: false,
+      timeout: timeout / 1000,
+    });
   };
 }
 

src/packages/file-server/conat/local-path.ts

@@ -0,0 +1,44 @@
+import { fsServer } from "@cocalc/conat/files/fs";
+import { conat } from "@cocalc/backend/conat";
+import { SandboxedFilesystem } from "@cocalc/file-server/fs/sandbox";
+import { mkdir } from "fs/promises";
+import { join } from "path";
+import { isValidUUID } from "@cocalc/util/misc";
+
+export function localPathFileserver({
+  service,
+  path,
+}: {
+  service: string;
+  path: string;
+}) {
+  const client = conat();
+  const server = fsServer({
+    service,
+    client,
+    fs: async (subject: string) => {
+      const project_id = getProjectId(subject);
+      const p = join(path, project_id);
+      try {
+        await mkdir(p);
+      } catch {}
+      return new SandboxedFilesystem(p);
+    },
+  });
+  return server;
+}
+
+function getProjectId(subject: string) {
+  const v = subject.split(".");
+  if (v.length != 2) {
+    throw Error("subject must have 2 segments");
+  }
+  if (!v[1].startsWith("project-")) {
+    throw Error("second segment of subject must start with 'project-'");
+  }
+  const project_id = v[1].slice("project-".length);
+  if (!isValidUUID(project_id)) {
+    throw Error("not a valid project id");
+  }
+  return project_id;
+}

src/packages/file-server/conat/test/local-path.test.ts

@@ -0,0 +1,3 @@
+describe("use the simple fileserver", () => {
+  it("does nothing", async () => {});
+});

src/packages/file-server/fs/sandbox.test.ts

@@ -0,0 +1,59 @@
+import { SandboxedFilesystem } from "./sandbox";
+import { mkdtemp, mkdir, rm } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { join } from "path";
+
+let tempDir;
+beforeAll(async () => {
+  tempDir = await mkdtemp(join(tmpdir(), "cocalc"));
+});
+
+describe("test using the filesystem sandbox to do a few standard things", () => {
+  let fs;
+  it("creates and reads file", async () => {
+    await mkdir(join(tempDir, "test-1"));
+    fs = new SandboxedFilesystem(join(tempDir, "test-1"));
+    await fs.writeFile("a", "hi");
+    const r = await fs.readFile("a", "utf8");
+    expect(r).toEqual("hi");
+  });
+
+  it("truncate file", async () => {
+    await fs.writeFile("b", "hello");
+    await fs.truncate("b", 4);
+    const r = await fs.readFile("b", "utf8");
+    expect(r).toEqual("hell");
+  });
+});
+
+describe("make various attempts to break out of the sandbox", () => {
+  let fs;
+  it("creates sandbox", async () => {
+    await mkdir(join(tempDir, "test-2"));
+    fs = new SandboxedFilesystem(join(tempDir, "test-2"));
+    await fs.writeFile("x", "hi");
+  });
+
+  it("obvious first attempt to escape fails", async () => {
+    const v = await fs.readdir("..");
+    expect(v).toEqual(["x"]);
+  });
+
+  it("obvious first attempt to escape fails", async () => {
+    const v = await fs.readdir("a/../..");
+    expect(v).toEqual(["x"]);
+  });
+
+  it("another attempt", async () => {
+    await fs.copyFile("/x", "/tmp");
+    const v = await fs.readdir("a/../..");
+    expect(v).toEqual(["tmp", "x"]);
+
+    const r = await fs.readFile("tmp", "utf8");
+    expect(r).toEqual("hi");
+  });
+});
+
+afterAll(async () => {
+  await rm(tempDir, { force: true, recursive: true });
+});