btrfs: add more filesystem support

Author: williamstein

src/packages/file-server/btrfs/subvolume.ts

@@ -4,13 +4,33 @@ A subvolume
 
 import { type Filesystem, DEFAULT_SUBVOLUME_SIZE } from "./filesystem";
 import refCache from "@cocalc/util/refcache";
-import { readFile, writeFile, unlink } from "node:fs/promises";
+import {
+  appendFile,
+  chmod,
+  cp,
+  copyFile,
+  link,
+  readFile,
+  realpath,
+  rename,
+  rm,
+  rmdir,
+  mkdir,
+  stat,
+  symlink,
+  truncate,
+  writeFile,
+  unlink,
+  utimes,
+  watch,
+} from "node:fs/promises";
 import { exists, isdir, listdir, mkdirp, sudo } from "./util";
 import { join, normalize } from "path";
 import { updateRollingSnapshots, type SnapshotCounts } from "./snapshots";
 import { DirectoryListingEntry } from "@cocalc/util/types";
 import getListing from "@cocalc/backend/get-listing";
 import getLogger from "@cocalc/backend/logger";
+import { exists as pathExists } from "@cocalc/backend/misc/async-utils-node";
 
 export const SNAPSHOTS = ".snapshots";
 const SEND_SNAPSHOT_PREFIX = "send-";
@@ -104,10 +124,73 @@ export class Subvolume {
     return await writeFile(this.normalize(path), data);
   };
 
+  appendFile = async (path: string, data: string | Buffer, encoding?) => {
+    path = normalize(path);
+    return await appendFile(this.normalize(path), data, encoding);
+  };
+
   unlink = async (path: string) => {
     await unlink(this.normalize(path));
   };
 
+  stat = async (path: string) => {
+    return await stat(this.normalize(path));
+  };
+
+  exists = async (path: string) => {
+    return await pathExists(this.normalize(path));
+  };
+
+  // hard link
+  link = async (existingPath: string, newPath: string) => {
+    return await link(this.normalize(existingPath), this.normalize(newPath));
+  };
+
+  symlink = async (target: string, path: string) => {
+    return await symlink(this.normalize(target), this.normalize(path));
+  };
+
+  realpath = async (path: string) => {
+    const x = await realpath(this.normalize(path));
+    return x.slice(this.path.length + 1);
+  };
+
+  rename = async (oldPath: string, newPath: string) => {
+    await rename(this.normalize(oldPath), this.normalize(newPath));
+  };
+
+  utimes = async (
+    path: string,
+    atime: number | string | Date,
+    mtime: number | string | Date,
+  ) => {
+    await utimes(this.normalize(path), atime, mtime);
+  };
+
+  watch = (filename: string, options?) => {
+    return watch(this.normalize(filename), options);
+  };
+
+  truncate = async (path: string, len?: number) => {
+    await truncate(this.normalize(path), len);
+  };
+
+  copyFile = async (src: string, dest: string) => {
+    await copyFile(this.normalize(src), this.normalize(dest));
+  };
+
+  cp = async (src: string, dest: string, options?) => {
+    await cp(this.normalize(src), this.normalize(dest), options);
+  };
+
+  chmod = async (path: string, mode: string | number) => {
+    await chmod(this.normalize(path), mode);
+  };
+
+  mkdir = async (path: string, options?) => {
+    await mkdir(this.normalize(path), options);
+  };
+
   rsync = async ({
     src,
     target,
@@ -135,6 +218,14 @@ export class Subvolume {
     });
   };
 
+  rmdir = async (path: string, options?) => {
+    await rmdir(this.normalize(path), options);
+  };
+
+  rm = async (path: string, options?) => {
+    await rm(this.normalize(path), options);
+  };
+
   /////////////
   // QUOTA
   /////////////

src/packages/file-server/btrfs/test/filesystem-stress.test.ts

@@ -1,6 +1,4 @@
 import { before, after, fs } from "./setup";
-import { writeFile } from "fs/promises";
-import { join } from "path";
 
 beforeAll(before);
 
@@ -45,7 +43,7 @@ describe("stress operations with subvolumes", () => {
   it("write a file to each volume", async () => {
     for (const name of await fs.list()) {
       const vol = await fs.subvolume(name);
-      await writeFile(join(vol.path, "a.txt"), "hi");
+      await vol.writeFile("a.txt", "hi");
     }
   });
 

src/packages/file-server/btrfs/test/filesystem.test.ts

@@ -1,7 +1,5 @@
 import { before, after, fs } from "./setup";
 import { isValidUUID } from "@cocalc/util/misc";
-import { readFile, writeFile } from "fs/promises";
-import { join } from "path";
 
 beforeAll(before);
 
@@ -63,16 +61,16 @@ describe("operations with subvolumes", () => {
   it("rsync an actual file", async () => {
     const sagemath = await fs.subvolume("sagemath");
     const cython = await fs.subvolume("cython");
-    await writeFile(join(sagemath.path, "README.md"), "hi");
+    await sagemath.writeFile("README.md", "hi");
     await fs.rsync({ src: "sagemath", target: "cython" });
-    const copy = await readFile(join(cython.path, "README.md"), "utf8");
+    const copy = await cython.readFile("README.md", "utf8");
     expect(copy).toEqual("hi");
   });
 
   it("clone a subvolume with contents", async () => {
     await fs.cloneSubvolume("cython", "pyrex");
     const pyrex = await fs.subvolume("pyrex");
-    const clone = await readFile(join(pyrex.path, "README.md"), "utf8");
+    const clone = await pyrex.readFile("README.md", "utf8");
     expect(clone).toEqual("hi");
   });
 });

src/packages/file-server/btrfs/test/setup.ts

@@ -35,7 +35,8 @@ async function ensureMoreLoops() {
 export async function before() {
   try {
     const command = `umount ${join(tmpdir(), TEMP_PREFIX)}*/mnt`;
-    // attempt to unmount any mounts left from previous runs
+    // attempt to unmount any mounts left from previous runs.
+    // TODO: this could impact runs in parallel
     await sudo({ command, bash: true });
   } catch {}
   await ensureMoreLoops();

src/packages/file-server/btrfs/test/subvolume.test.ts

@@ -49,6 +49,129 @@ describe("setting and getting quota of a subvolume", () => {
   });
 });
 
+describe("the filesystem operations", () => {
+  let vol;
+
+  it("creates a volume and get empty listing", async () => {
+    vol = await fs.subvolume("fs");
+    expect(await vol.ls("")).toEqual([]);
+  });
+
+  it("error listing non-existent path", async () => {
+    vol = await fs.subvolume("fs");
+    expect(async () => {
+      await vol.ls("no-such-path");
+    }).rejects.toThrow("ENOENT");
+  });
+
+  it("creates a text file to it", async () => {
+    await vol.writeFile("a.txt", "hello");
+    const ls = await vol.ls("");
+    expect(ls).toEqual([{ name: "a.txt", mtime: ls[0].mtime, size: 5 }]);
+  });
+
+  it("read the file we just created as utf8", async () => {
+    expect(await vol.readFile("a.txt", "utf8")).toEqual("hello");
+  });
+
+  it("read the file we just created as a binary buffer", async () => {
+    expect(await vol.readFile("a.txt")).toEqual(Buffer.from("hello"));
+  });
+
+  it("stat the file we just created", async () => {
+    const s = await vol.stat("a.txt");
+    expect(s.size).toBe(5);
+    expect(Math.abs(s.mtimeMs - Date.now())).toBeLessThan(60_000);
+  });
+
+  let origStat;
+  it("snapshot filesystem and see file is in snapshot", async () => {
+    await vol.createSnapshot("snap");
+    const s = await vol.ls(vol.snapshotPath("snap"));
+    expect(s).toEqual([{ name: "a.txt", mtime: s[0].mtime, size: 5 }]);
+
+    const stat = await vol.stat("a.txt");
+    origStat = stat;
+    expect(stat.mtimeMs / 1000).toBeCloseTo(s[0].mtime);
+  });
+
+  it("unlink (delete) our file", async () => {
+    await vol.unlink("a.txt");
+    expect(await vol.ls("")).toEqual([]);
+  });
+
+  it("snapshot still exists", async () => {
+    expect(await vol.exists(vol.snapshotPath("snap", "a.txt")));
+  });
+
+  it("copy file from snapshot and note it has the same mode as before (so much nicer than what happens with zfs)", async () => {
+    await vol.copyFile(vol.snapshotPath("snap", "a.txt"), "a.txt");
+    const stat = await vol.stat("a.txt");
+    expect(stat.mode).toEqual(origStat.mode);
+  });
+
+  it("create and copy a folder", async () => {
+    await vol.mkdir("my-folder");
+    await vol.writeFile("my-folder/foo.txt", "foo");
+    await vol.cp("my-folder", "folder2", { recursive: true });
+    expect(await vol.readFile("folder2/foo.txt", "utf8")).toEqual("foo");
+  });
+
+  it("append to a file", async () => {
+    await vol.writeFile("b.txt", "hell");
+    await vol.appendFile("b.txt", "-o");
+    expect(await vol.readFile("b.txt", "utf8")).toEqual("hell-o");
+  });
+
+  it("make a file readonly, then change it back", async () => {
+    await vol.writeFile("c.txt", "hi");
+    await vol.chmod("c.txt", "444");
+    expect(async () => {
+      await vol.appendFile("c.txt", " there");
+    }).rejects.toThrow("EACCES");
+    await vol.chmod("c.txt", "666");
+    await vol.appendFile("c.txt", " there");
+  });
+
+  it("realpath of a symlink", async () => {
+    await vol.writeFile("real.txt", "i am real");
+    await vol.symlink("real.txt", "link.txt");
+    expect(await vol.realpath("link.txt")).toBe("real.txt");
+  });
+
+  it("watch for changes", async () => {
+    await vol.writeFile("w.txt", "hi");
+    const ac = new AbortController();
+    const { signal } = ac;
+    const watcher = vol.watch("w.txt", { signal });
+    vol.appendFile("w.txt", " there");
+    const { value, done } = await watcher.next();
+    expect(done).toBe(false);
+    expect(value).toEqual({ eventType: "change", filename: "w.txt" });
+    ac.abort();
+
+    expect(async () => {
+      await watcher.next();
+    }).rejects.toThrow("aborted");
+  });
+
+  it("rename a file", async () => {
+    await vol.writeFile("old", "hi");
+    await vol.rename("old", "new");
+    expect(await vol.readFile("new", "utf8")).toEqual("hi");
+  });
+
+  it("create and remove a directory", async () => {
+    await vol.mkdir("path");
+    await vol.rmdir("path");
+  });
+
+  it("create a directory recursively and remove", async () => {
+    await vol.mkdir("path/to/stuff", { recursive: true });
+    await vol.rm("path", { recursive: true });
+  });
+});
+
 describe("test snapshots", () => {
   let vol;
   it("creates a volume and write a file to it", async () => {