add more btrfs tests

Author: williamstein

src/packages/file-server/btrfs/subvolume.ts

@@ -4,25 +4,17 @@ A subvolume
 
 import { type Filesystem, DEFAULT_SUBVOLUME_SIZE } from "./filesystem";
 import refCache from "@cocalc/util/refcache";
-import {
-  exists,
-  listdir,
-  mkdirp,
-  sudo,
-} from "./util";
+import { exists, listdir, mkdirp, sudo } from "./util";
 import { join } from "path";
 import { updateRollingSnapshots, type SnapshotCounts } from "./snapshots";
-import { human_readable_size } from "@cocalc/util/misc";
+//import { human_readable_size } from "@cocalc/util/misc";
+import getLogger from "@cocalc/backend/logger";
 
 export const SNAPSHOTS = ".snapshots";
 const SEND_SNAPSHOT_PREFIX = "send-";
-
 const BUP_SNAPSHOT = "temp-bup-snapshot";
-
 const PAD = 4;
 
-import getLogger from "@cocalc/backend/logger";
-
 const logger = getLogger("file-server:storage-btrfs:subvolume");
 
 interface Options {
@@ -88,6 +80,20 @@ export class Subvolume {
     return x["qgroup-show"][0];
   };
 
+  quota = async (): Promise<{
+    size: number;
+    used: number;
+  }> => {
+    let { max_referenced: size, referenced: used } = await this.quotaInfo();
+    if (size == "none") {
+      size = null;
+    }
+    return {
+      used,
+      size,
+    };
+  };
+
   size = async (size: string | number) => {
     if (!size) {
       throw Error("size must be specified");
@@ -98,23 +104,50 @@ export class Subvolume {
     });
   };
 
+  du = async () => {
+    return await sudo({
+      command: "btrfs",
+      args: ["filesystem", "du", "-s", this.path],
+    });
+  };
+
   usage = async (): Promise<{
+    // used and free in bytes
+    used: number;
+    free: number;
     size: number;
-    usage: number;
-    human: { size: string; usage: string };
   }> => {
-    let { max_referenced: size, referenced: usage } = await this.quotaInfo();
-    if (size == "none") {
-      size = null;
+    const { stdout } = await sudo({
+      command: "btrfs",
+      args: ["filesystem", "usage", "-b", this.path],
+    });
+    let used: number = -1;
+    let free: number = -1;
+    let size: number = -1;
+    for (const x of stdout.split("\n")) {
+      if (used == -1) {
+        const i = x.indexOf("Used:");
+        if (i != -1) {
+          used = parseInt(x.split(":")[1].trim());
+          continue;
+        }
+      }
+      if (free == -1) {
+        const i = x.indexOf("Free (statfs, df):");
+        if (i != -1) {
+          free = parseInt(x.split(":")[1].trim());
+          continue;
+        }
+      }
+      if (size == -1) {
+        const i = x.indexOf("Device size:");
+        if (i != -1) {
+          size = parseInt(x.split(":")[1].trim());
+          continue;
+        }
+      }
     }
-    return {
-      usage,
-      size,
-      human: {
-        usage: human_readable_size(usage),
-        size: size != null ? human_readable_size(size) : size,
-      },
-    };
+    return { used, free, size };
   };
 
   private makeSnapshotsDir = async () => {

src/packages/file-server/btrfs/test/filesystem-stress.test.ts

@@ -0,0 +1,97 @@
+import { before, after, fs } from "./setup";
+import { writeFile } from "fs/promises";
+import { join } from "path";
+
+beforeAll(before);
+
+//const log = console.log;
+const log = (..._args) => {};
+
+describe("stress operations with subvolumes", () => {
+  const count1 = 10;
+  it(`create ${count1} subvolumes in serial`, async () => {
+    const t = Date.now();
+    for (let i = 0; i < count1; i++) {
+      await fs.subvolume(`${i}`);
+    }
+    log(
+      `created ${Math.round((count1 / (Date.now() - t)) * 1000)} subvolumes per second serial`,
+    );
+  });
+
+  it("list them and confirm", async () => {
+    const v = await fs.list();
+    expect(v.length).toBe(count1);
+  });
+
+  let count2 = 10;
+  it(`create ${count2} subvolumes in parallel`, async () => {
+    const v: any[] = [];
+    const t = Date.now();
+    for (let i = 0; i < count2; i++) {
+      v.push(fs.subvolume(`p-${i}`));
+    }
+    await Promise.all(v);
+    log(
+      `created ${Math.round((count2 / (Date.now() - t)) * 1000)} subvolumes per second in parallel`,
+    );
+  });
+
+  it("list them and confirm", async () => {
+    const v = await fs.list();
+    expect(v.length).toBe(count1 + count2);
+  });
+
+  it("write a file to each volume", async () => {
+    for (const name of await fs.list()) {
+      const vol = await fs.subvolume(name);
+      await writeFile(join(vol.path, "a.txt"), "hi");
+    }
+  });
+
+  it("clone the first group in serial", async () => {
+    const t = Date.now();
+    for (let i = 0; i < count1; i++) {
+      await fs.cloneSubvolume(`${i}`, `clone-of-${i}`);
+    }
+    log(
+      `cloned ${Math.round((count1 / (Date.now() - t)) * 1000)} subvolumes per second serial`,
+    );
+  });
+
+  it("clone the second group in parallel", async () => {
+    const t = Date.now();
+    const v: any[] = [];
+    for (let i = 0; i < count2; i++) {
+      v.push(fs.cloneSubvolume(`p-${i}`, `clone-of-p-${i}`));
+    }
+    await Promise.all(v);
+    log(
+      `cloned ${Math.round((count2 / (Date.now() - t)) * 1000)} subvolumes per second parallel`,
+    );
+  });
+
+  it("delete the first batch serial", async () => {
+    const t = Date.now();
+    for (let i = 0; i < count1; i++) {
+      await fs.deleteSubvolume(`${i}`);
+    }
+    log(
+      `deleted ${Math.round((count1 / (Date.now() - t)) * 1000)} subvolumes per second serial`,
+    );
+  });
+
+  it("delete the second batch in parallel", async () => {
+    const v: any[] = [];
+    const t = Date.now();
+    for (let i = 0; i < count2; i++) {
+      v.push(fs.deleteSubvolume(`p-${i}`));
+    }
+    await Promise.all(v);
+    log(
+      `deleted ${Math.round((count2 / (Date.now() - t)) * 1000)} subvolumes per second in parallel`,
+    );
+  });
+});
+
+afterAll(after);

src/packages/file-server/btrfs/test/setup.ts

@@ -6,6 +6,7 @@ import process from "node:process";
 import { chmod, mkdtemp, mkdir, rm } from "node:fs/promises";
 import { tmpdir } from "node:os";
 import { join } from "path";
+import { until } from "@cocalc/util/async-utils";
 
 export let fs: Filesystem;
 let tempDir;
@@ -26,8 +27,13 @@ export async function before() {
 }
 
 export async function after() {
-  try {
-    await fs.unmount();
-  } catch {}
+  await until(async () => {
+    try {
+      await fs.unmount();
+      return true;
+    } catch {
+      return false;
+    }
+  });
   await rm(tempDir, { force: true, recursive: true });
 }

src/packages/file-server/btrfs/test/subvolume.test.ts

@@ -0,0 +1,45 @@
+import { before, after, fs } from "./setup";
+import { writeFile } from "fs/promises";
+import { join } from "path";
+import { delay } from "awaiting";
+import { wait } from "@cocalc/backend/conat/test/util";
+import { randomBytes } from "crypto";
+
+beforeAll(before);
+
+jest.setTimeout(20000);
+describe("setting and getting quota of a subvolume", () => {
+  let vol;
+  it("set the quota of a subvolume to 5 M", async () => {
+    vol = await fs.subvolume("q");
+    await vol.size("5M");
+
+    const { size, used } = await vol.quota();
+    expect(size).toBe(5 * 1024 * 1024);
+    expect(used).toBe(0);
+  });
+
+  it("write a file and check usage goes up", async () => {
+    const buf = randomBytes(4 * 1024 * 1024);
+    await writeFile(join(vol.path, "buf"), buf);
+    await wait({
+      until: async () => {
+        await delay(1000);
+        const { used } = await vol.usage();
+        return used > 0;
+      },
+    });
+    const { used } = await vol.usage();
+    expect(used).toBeGreaterThan(0);
+  });
+
+  it("fail to write a 50MB file (due to quota)", async () => {
+    const buf2 = randomBytes(50 * 1024 * 1024);
+    const b = join(vol.path, "buf2");
+    expect(async () => {
+      await writeFile(b, buf2);
+    }).rejects.toThrow("write");
+  });
+});
+
+afterAll(after);

src/packages/file-server/btrfs/util.ts

@@ -0,0 +1,71 @@
+import {
+  type ExecuteCodeOptions,
+  type ExecuteCodeOutput,
+} from "@cocalc/util/types/execute-code";
+import { executeCode } from "@cocalc/backend/execute-code";
+import getLogger from "@cocalc/backend/logger";
+
+const logger = getLogger("file-server:storage:util");
+
+const DEFAULT_EXEC_TIMEOUT_MS = 60 * 1000;
+
+export async function exists(path: string) {
+  try {
+    await sudo({ command: "ls", args: [path], verbose: false });
+    return true;
+  } catch {
+    return false;
+  }
+}
+
+export async function mkdirp(paths: string[]) {
+  if (paths.length == 0) return;
+  await sudo({ command: "mkdir", args: ["-p", ...paths] });
+}
+
+export async function chmod(args: string[]) {
+  await sudo({ command: "chmod", args: args });
+}
+
+export async function sudo(
+  opts: ExecuteCodeOptions & { desc?: string },
+): Promise<ExecuteCodeOutput> {
+  if (opts.verbose !== false && opts.desc) {
+    logger.debug("exec", opts.desc);
+  }
+  let command, args;
+  if (opts.bash) {
+    command = `sudo ${opts.command}`;
+    args = undefined;
+  } else {
+    command = "sudo";
+    args = [opts.command, ...(opts.args ?? [])];
+  }
+  return await executeCode({
+    verbose: true,
+    timeout: DEFAULT_EXEC_TIMEOUT_MS / 1000,
+    ...opts,
+    command,
+    args,
+  });
+}
+
+export async function rm(paths: string[]) {
+  if (paths.length == 0) return;
+  await sudo({ command: "rm", args: paths });
+}
+
+export async function rmdir(paths: string[]) {
+  if (paths.length == 0) return;
+  await sudo({ command: "rmdir", args: paths });
+}
+
+export async function listdir(path: string) {
+  const { stdout } = await sudo({ command: "ls", args: ["-1", path] });
+  return stdout.split("\n").filter((x) => x);
+}
+
+export async function isdir(path: string) {
+  const { stdout } = await sudo({ command: "stat", args: ["-c", "%F", path] });
+  return stdout.trim() == "directory";
+}

src/packages/file-server/package.json

@@ -29,7 +29,8 @@
   "dependencies": {
     "@cocalc/backend": "workspace:*",
     "@cocalc/file-server": "workspace:*",
-    "@cocalc/util": "workspace:*"
+    "@cocalc/util": "workspace:*",
+    "awaiting": "^3.0.0"
   },
   "devDependencies": {
     "@types/jest": "^29.5.14",