btrfs: unit testing bup integration

Author: williamstein

src/packages/file-server/btrfs/subvolume.ts

@@ -5,7 +5,7 @@ A subvolume
 import { type Filesystem, DEFAULT_SUBVOLUME_SIZE } from "./filesystem";
 import refCache from "@cocalc/util/refcache";
 import { exists, listdir, mkdirp, sudo } from "./util";
-import { join } from "path";
+import { join, normalize } from "path";
 import { updateRollingSnapshots, type SnapshotCounts } from "./snapshots";
 //import { human_readable_size } from "@cocalc/util/misc";
 import getLogger from "@cocalc/backend/logger";
@@ -292,6 +292,76 @@ export class Subvolume {
       .filter((x) => x);
   };
 
+  bupRestore = async (path: string) => {
+    path = normalize(path);
+    // outdir -- path relative to subvolume
+    // path -- /branch/revision/path/to/dir
+    await sudo({
+      command: "bup",
+      args: [
+        "-d",
+        this.filesystem.bup,
+        "restore",
+        "-C",
+        this.path, //join(this.path, outdir),
+        join(`/${this.name}`, path),
+        "--quiet",
+      ],
+    });
+  };
+
+  bupLs = async (
+    path: string,
+  ): Promise<
+    {
+      path: string;
+      size: number;
+      timestamp: number;
+      isdir: boolean;
+    }[]
+  > => {
+    path = normalize(path);
+    const { stdout } = await sudo({
+      command: "bup",
+      args: [
+        "-d",
+        this.filesystem.bup,
+        "ls",
+        "--almost-all",
+        "--file-type",
+        "-l",
+        join(`/${this.name}`, path),
+      ],
+    });
+    const v: {
+      path: string;
+      size: number;
+      timestamp: number;
+      isdir: boolean;
+    }[] = [];
+    for (const x of stdout.split("\n")) {
+      // [-rw-------","6b851643360e435eb87ef9a6ab64a8b1/6b851643360e435eb87ef9a6ab64a8b1","5","2025-07-15","06:12","a.txt"]
+      const w = x.split(/\s+/);
+      if (w.length >= 6) {
+        let isdir, path;
+        if (w[5].endsWith("@") || w[5].endsWith("=") || w[5].endsWith("|")) {
+          w[5] = w[5].slice(0, -1);
+        }
+        if (w[5].endsWith("/")) {
+          isdir = true;
+          path = w[5].slice(0, -1);
+        } else {
+          path = w[5];
+          isdir = false;
+        }
+        const size = parseInt(w[2]);
+        const timestamp = new Date(w[3] + "T" + w[4]).valueOf();
+        v.push({ path, size, timestamp, isdir });
+      }
+    }
+    return v;
+  };
+
   bupPrune = async ({
     dailies = "1w",
     monthlies = "4m",

src/packages/file-server/btrfs/test/subvolume.test.ts

@@ -1,8 +1,9 @@
 import { before, after, fs, sudo } from "./setup";
-import { readFile, writeFile, unlink } from "fs/promises";
+import { mkdir, readFile, writeFile, unlink } from "fs/promises";
 import { join } from "path";
 import { wait } from "@cocalc/backend/conat/test/util";
 import { randomBytes } from "crypto";
+import { parseBupTime } from "../util";
 
 beforeAll(before);
 
@@ -89,9 +90,51 @@ describe("test snapshots", () => {
   });
 });
 
-// describe("test bup backups", ()=>{
-//   let vol;
-//   it('creates a volume')
-// })
+describe("test bup backups", () => {
+  let vol;
+  it("creates a volume", async () => {
+    vol = await fs.subvolume("bup-test");
+    await writeFile(join(vol.path, "a.txt"), "hello");
+  });
+
+  it("create a bup backup", async () => {
+    await vol.createBupBackup();
+  });
+
+  it("list bup backups of this vol -- there are 2, one for the date and 'latest'", async () => {
+    const v = await vol.bupBackups();
+    expect(v.length).toBe(2);
+    const t = parseBupTime(v[0]);
+    expect(Math.abs(t.valueOf() - Date.now())).toBeLessThan(10_000);
+  });
+
+  it("confirm a.txt is in our backup", async () => {
+    const x = await vol.bupLs("latest");
+    expect(x).toEqual([
+      { path: "a.txt", size: 5, timestamp: x[0].timestamp, isdir: false },
+    ]);
+  });
+
+  it("restore a.txt from our backup", async () => {
+    await writeFile(join(vol.path, "a.txt"), "hello2");
+    await vol.bupRestore("latest/a.txt");
+    expect(await readFile(join(vol.path, "a.txt"), "utf8")).toEqual("hello");
+  });
+
+  it("prune bup backups does nothing since we have so few", async () => {
+    await vol.bupPrune();
+    expect((await vol.bupBackups()).length).toBe(2);
+  });
+
+  it("add a directory and back up", async () => {
+    await mkdir(join(vol.path, "mydir"));
+    await vol.createBupBackup();
+    const x = await vol.bupLs("latest");
+    expect(x).toEqual([
+      { path: "a.txt", size: 5, timestamp: x[0].timestamp, isdir: false },
+      { path: "mydir", size: 0, timestamp: x[1].timestamp, isdir: true },
+    ]);
+  });
+});
 
 afterAll(after);

src/packages/file-server/btrfs/util.ts

@@ -69,3 +69,19 @@ export async function isdir(path: string) {
   const { stdout } = await sudo({ command: "stat", args: ["-c", "%F", path] });
   return stdout.trim() == "directory";
 }
+
+export function parseBupTime(s: string): Date {
+  const [year, month, day, time] = s.split("-");
+  const hours = time.slice(0, 2);
+  const minutes = time.slice(2, 4);
+  const seconds = time.slice(4, 6);
+
+  return new Date(
+    Number(year),
+    Number(month) - 1, // JS months are 0-based
+    Number(day),
+    Number(hours),
+    Number(minutes),
+    Number(seconds),
+  );
+}