btrfs: adding more testing and fs operations support

Author: williamstein

src/packages/backend/get-listing.ts

@@ -4,7 +4,7 @@
  */
 
 /*
-Server directory listing through the HTTP server and Websocket API.
+This is used by backends to serve directory listings to clients:
 
 {files:[..., {size:?,name:?,mtime:?,isdir:?}]}
 

src/packages/file-server/btrfs/subvolume.ts

@@ -4,10 +4,12 @@ A subvolume
 
 import { type Filesystem, DEFAULT_SUBVOLUME_SIZE } from "./filesystem";
 import refCache from "@cocalc/util/refcache";
-import { exists, listdir, mkdirp, sudo } from "./util";
+import { readFile, writeFile, unlink } from "node:fs/promises";
+import { exists, isdir, listdir, mkdirp, sudo } from "./util";
 import { join, normalize } from "path";
 import { updateRollingSnapshots, type SnapshotCounts } from "./snapshots";
-//import { human_readable_size } from "@cocalc/util/misc";
+import { DirectoryListingEntry } from "@cocalc/util/types";
+import getListing from "@cocalc/backend/get-listing";
 import getLogger from "@cocalc/backend/logger";
 
 export const SNAPSHOTS = ".snapshots";
@@ -23,10 +25,11 @@ interface Options {
 }
 
 export class Subvolume {
-  private filesystem: Filesystem;
   public readonly name: string;
-  public readonly path: string;
-  public readonly snapshotsDir: string;
+
+  private filesystem: Filesystem;
+  private readonly path: string;
+  private readonly snapshotsDir: string;
 
   constructor({ filesystem, name }: Options) {
     this.filesystem = filesystem;
@@ -70,6 +73,72 @@ export class Subvolume {
     });
   };
 
+  // this should provide a path that is guaranteed to be
+  // inside this.path on the filesystem or throw error
+  // [ ] TODO: not sure if the code here is sufficient!!
+  private normalize = (path: string) => {
+    return join(this.path, normalize(path));
+  };
+
+  /////////////
+  // Files
+  /////////////
+  ls = async (
+    path: string,
+    { hidden, limit }: { hidden?: boolean; limit?: number } = {},
+  ): Promise<DirectoryListingEntry[]> => {
+    path = normalize(path);
+    return await getListing(this.normalize(path), hidden, {
+      limit,
+      home: "/",
+    });
+  };
+
+  readFile = async (path: string, encoding?: any): Promise<string | Buffer> => {
+    path = normalize(path);
+    return await readFile(this.normalize(path), encoding);
+  };
+
+  writeFile = async (path: string, data: string | Buffer) => {
+    path = normalize(path);
+    return await writeFile(this.normalize(path), data);
+  };
+
+  unlink = async (path: string) => {
+    await unlink(this.normalize(path));
+  };
+
+  rsync = async ({
+    src,
+    target,
+    args = ["-axH"],
+    timeout = 5 * 60 * 1000,
+  }: {
+    src: string;
+    target: string;
+    args?: string[];
+    timeout?: number;
+  }): Promise<{ stdout: string; stderr: string; exit_code: number }> => {
+    let srcPath = this.normalize(src);
+    let targetPath = this.normalize(target);
+    if (!srcPath.endsWith("/") && (await isdir(srcPath))) {
+      srcPath += "/";
+      if (!targetPath.endsWith("/")) {
+        targetPath += "/";
+      }
+    }
+    return await sudo({
+      command: "rsync",
+      args: [...args, srcPath, targetPath],
+      err_on_exit: false,
+      timeout: timeout / 1000,
+    });
+  };
+
+  /////////////
+  // QUOTA
+  /////////////
+
   private quotaInfo = async () => {
     const { stdout } = await sudo({
       verbose: false,
@@ -150,6 +219,13 @@ export class Subvolume {
     return { used, free, size };
   };
 
+  /////////////
+  // SNAPSHOTS
+  /////////////
+  snapshotPath = (snapshot: string, ...segments) => {
+    return join(SNAPSHOTS, snapshot, ...segments);
+  };
+
   private makeSnapshotsDir = async () => {
     if (await exists(this.snapshotsDir)) {
       return;
@@ -233,6 +309,13 @@ export class Subvolume {
     return snapGen < pathGen;
   };
 
+  /////////////
+  // BACKUPS
+  // There is a single global dedup'd backup archive stored in the btrfs filesystem.
+  // Obviously, admins should rsync this regularly to a separate location as a genuine
+  // backup strategy.
+  /////////////
+
   // create a new bup backup
   createBupBackup = async ({
     // timeout used for bup index and bup save commands
@@ -304,7 +387,7 @@ export class Subvolume {
     const i = path.indexOf("/"); // remove the commit name
     await sudo({
       command: "rm",
-      args: ["-rf", join(this.path, path.slice(i + 1))],
+      args: ["-rf", this.normalize(path.slice(i + 1))],
     });
     await sudo({
       command: "bup",
@@ -320,16 +403,7 @@ export class Subvolume {
     });
   };
 
-  bupLs = async (
-    path: string,
-  ): Promise<
-    {
-      path: string;
-      size: number;
-      timestamp: number;
-      isdir: boolean;
-    }[]
-  > => {
+  bupLs = async (path: string): Promise<DirectoryListingEntry[]> => {
     path = normalize(path);
     const { stdout } = await sudo({
       command: "bup",
@@ -343,30 +417,25 @@ export class Subvolume {
         join(`/${this.name}`, path),
       ],
     });
-    const v: {
-      path: string;
-      size: number;
-      timestamp: number;
-      isdir: boolean;
-    }[] = [];
+    const v: DirectoryListingEntry[] = [];
     for (const x of stdout.split("\n")) {
       // [-rw-------","6b851643360e435eb87ef9a6ab64a8b1/6b851643360e435eb87ef9a6ab64a8b1","5","2025-07-15","06:12","a.txt"]
       const w = x.split(/\s+/);
       if (w.length >= 6) {
-        let isdir, path;
+        let isdir, name;
         if (w[5].endsWith("@") || w[5].endsWith("=") || w[5].endsWith("|")) {
           w[5] = w[5].slice(0, -1);
         }
         if (w[5].endsWith("/")) {
           isdir = true;
-          path = w[5].slice(0, -1);
+          name = w[5].slice(0, -1);
         } else {
-          path = w[5];
+          name = w[5];
           isdir = false;
         }
         const size = parseInt(w[2]);
-        const timestamp = new Date(w[3] + "T" + w[4]).valueOf();
-        v.push({ path, size, timestamp, isdir });
+        const mtime = new Date(w[3] + "T" + w[4]).valueOf() / 1000;
+        v.push({ name, size, mtime, isdir });
       }
     }
     return v;
@@ -392,6 +461,18 @@ export class Subvolume {
     });
   };
 
+  /////////////
+  // BTRFS send/recv
+  // Not used.  Instead we will rely on bup (and snapshots of the underlying disk) for backups, since:
+  //  - much easier to check they are valid
+  //  - decoupled from any btrfs issues
+  //  - not tied to any specific filesystem at all
+  //  - easier to offsite via incremntal rsync
+  //  - much more space efficient with *global* dedup and compression
+  //  - bup is really just git, which is very proven
+  // The drawback is speed.
+  /////////////
+
   // this was just a quick proof of concept -- I don't like it.  Should switch to using
   // timestamps and a lock.
   // To recover these, doing recv for each in order does work.  Then you have to

src/packages/file-server/btrfs/test/subvolume-stress.test.ts

@@ -1,39 +1,84 @@
 import { before, after, fs } from "./setup";
-import { writeFile } from "fs/promises";
+import { mkdir, writeFile } from "fs/promises";
 import { join } from "path";
 
+const DEBUG = false;
+const log = DEBUG ? console.log : (..._args) => {};
+
+const numSnapshots = 25;
+const numFiles = 1000;
+
 beforeAll(before);
-//const log = console.log;
-const log = (..._args) => {};
 
-describe("stress test creating many snapshots", () => {
+describe(`stress test creating ${numSnapshots} snapshots`, () => {
   let vol;
   it("creates a volume and write a file to it", async () => {
     vol = await fs.subvolume("stress");
   });
 
-  const count = 25;
-  it(`create file and snapshot the volume ${count} times`, async () => {
+  it(`create file and snapshot the volume ${numSnapshots} times`, async () => {
     const snaps: string[] = [];
     const start = Date.now();
-    for (let i = 0; i < count; i++) {
+    for (let i = 0; i < numSnapshots; i++) {
       await writeFile(join(vol.path, `${i}.txt`), "world");
       await vol.createSnapshot(`snap${i}`);
       snaps.push(`snap${i}`);
     }
     log(
-      `created ${Math.round((count / (Date.now() - start)) * 1000)} snapshots per second in serial`,
+      `created ${Math.round((numSnapshots / (Date.now() - start)) * 1000)} snapshots per second in serial`,
     );
     snaps.sort();
     expect(await vol.snapshots()).toEqual(snaps);
   });
 
-  it(`delete our ${count} snapshots`, async () => {
-    for (let i = 0; i < count; i++) {
+  it(`delete our ${numSnapshots} snapshots`, async () => {
+    for (let i = 0; i < numSnapshots; i++) {
       await vol.deleteSnapshot(`snap${i}`);
     }
     expect(await vol.snapshots()).toEqual([]);
   });
 });
 
+describe(`create ${numFiles} files`, () => {
+  let vol;
+  it("creates a volume", async () => {
+    vol = await fs.subvolume("many-files");
+  });
+
+  it(`creates ${numFiles} files`, async () => {
+    const names: string[] = [];
+    const start = Date.now();
+    for (let i = 0; i < numFiles; i++) {
+      await writeFile(join(vol.path, `${i}`), "world");
+      names.push(`${i}`);
+    }
+    log(
+      `created ${Math.round((numFiles / (Date.now() - start)) * 1000)} files per second in serial`,
+    );
+    const v = await vol.ls("");
+    const w = v.map(({ name }) => name);
+    expect(w.sort()).toEqual(names.sort());
+  });
+
+  it(`creates ${numFiles} files in parallel`, async () => {
+    await mkdir(join(vol.path, "p"));
+    const names: string[] = [];
+    const start = Date.now();
+    const z: any[] = [];
+    for (let i = 0; i < numFiles; i++) {
+      z.push(writeFile(join(vol.path, `p/${i}`), "world"));
+      names.push(`${i}`);
+    }
+    await Promise.all(z);
+    log(
+      `created ${Math.round((numFiles / (Date.now() - start)) * 1000)} files per second in parallel`,
+    );
+    const t0 = Date.now();
+    const v = await vol.ls("p");
+    log("get listing of files took", Date.now() - t0, "ms");
+    const w = v.map(({ name }) => name);
+    expect(w.sort()).toEqual(names.sort());
+  });
+});
+
 afterAll(after);