Skip to content

Commit 636d589

Browse files
williamsteinwilliamstein
committed
btrfs: refactor bup backup code
1 parent 0268a40 commit 636d589

File tree

4 files changed

+209
-180
lines changed

4 files changed

+209
-180
lines changed

‎src/packages/file-server/btrfs/subvolume-bup.ts

Lines changed: 178 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,178 @@
1+
import { type DirectoryListingEntry } from "@cocalc/util/types";
2+
import { type Subvolume } from "./subvolume";
3+
import { sudo, parseBupTime } from "./util";
4+
import { join, normalize } from "path";
5+
import getLogger from "@cocalc/backend/logger";
6+
7+
const BUP_SNAPSHOT = "temp-bup-snapshot";
8+
9+
const logger = getLogger("file-server:storage-btrfs:subvolume-bup");
10+
11+
export class SubvolumeBup {
12+
constructor(private subvolume: Subvolume) {}
13+
14+
/////////////
15+
// BACKUPS
16+
// There is a single global dedup'd backup archive stored in the btrfs filesystem.
17+
// Obviously, admins should rsync this regularly to a separate location as a genuine
18+
// backup strategy.
19+
/////////////
20+
21+
// create a new bup backup
22+
save = async ({
23+
// timeout used for bup index and bup save commands
24+
timeout = 30 * 60 * 1000,
25+
}: { timeout?: number } = {}) => {
26+
if (await this.subvolume.snapshotExists(BUP_SNAPSHOT)) {
27+
logger.debug(`createBupBackup: deleting existing ${BUP_SNAPSHOT}`);
28+
await this.subvolume.deleteSnapshot(BUP_SNAPSHOT);
29+
}
30+
try {
31+
logger.debug(
32+
`createBackup: creating ${BUP_SNAPSHOT} to get a consistent backup`,
33+
);
34+
await this.subvolume.createSnapshot(BUP_SNAPSHOT);
35+
const target = this.subvolume.normalize(
36+
this.subvolume.snapshotPath(BUP_SNAPSHOT),
37+
);
38+
39+
logger.debug(`createBupBackup: indexing ${BUP_SNAPSHOT}`);
40+
await sudo({
41+
command: "bup",
42+
args: [
43+
"-d",
44+
this.subvolume.filesystem.bup,
45+
"index",
46+
"--exclude",
47+
join(target, ".snapshots"),
48+
"-x",
49+
target,
50+
],
51+
timeout,
52+
});
53+
54+
logger.debug(`createBackup: saving ${BUP_SNAPSHOT}`);
55+
await sudo({
56+
command: "bup",
57+
args: [
58+
"-d",
59+
this.subvolume.filesystem.bup,
60+
"save",
61+
"--strip",
62+
"-n",
63+
this.subvolume.name,
64+
target,
65+
],
66+
timeout,
67+
});
68+
} finally {
69+
logger.debug(`createBupBackup: deleting temporary ${BUP_SNAPSHOT}`);
70+
await this.subvolume.deleteSnapshot(BUP_SNAPSHOT);
71+
}
72+
};
73+
74+
restore = async (path: string) => {
75+
// path -- branch/revision/path/to/dir
76+
if (path.startsWith("/")) {
77+
path = path.slice(1);
78+
}
79+
path = normalize(path);
80+
// ... but to avoid potential data loss, we make a snapshot before deleting it.
81+
await this.subvolume.createSnapshot();
82+
const i = path.indexOf("/"); // remove the commit name
83+
// remove the target we're about to restore
84+
await this.subvolume.fs.rm(path.slice(i + 1), { recursive: true });
85+
await sudo({
86+
command: "bup",
87+
args: [
88+
"-d",
89+
this.subvolume.filesystem.bup,
90+
"restore",
91+
"-C",
92+
this.subvolume.path,
93+
join(`/${this.subvolume.name}`, path),
94+
"--quiet",
95+
],
96+
});
97+
};
98+
99+
ls = async (path: string = ""): Promise<DirectoryListingEntry[]> => {
100+
if (!path) {
101+
const { stdout } = await sudo({
102+
command: "bup",
103+
args: ["-d", this.subvolume.filesystem.bup, "ls", this.subvolume.name],
104+
});
105+
const v: DirectoryListingEntry[] = [];
106+
let newest = 0;
107+
for (const x of stdout.trim().split("\n")) {
108+
const name = x.split(" ").slice(-1)[0];
109+
if (name == "latest") {
110+
continue;
111+
}
112+
const mtime = parseBupTime(name).valueOf() / 1000;
113+
newest = Math.max(mtime, newest);
114+
v.push({ name, isdir: true, mtime });
115+
}
116+
if (v.length > 0) {
117+
v.push({ name: "latest", isdir: true, mtime: newest });
118+
}
119+
return v;
120+
}
121+
122+
path = normalize(path);
123+
const { stdout } = await sudo({
124+
command: "bup",
125+
args: [
126+
"-d",
127+
this.subvolume.filesystem.bup,
128+
"ls",
129+
"--almost-all",
130+
"--file-type",
131+
"-l",
132+
join(`/${this.subvolume.name}`, path),
133+
],
134+
});
135+
const v: DirectoryListingEntry[] = [];
136+
for (const x of stdout.split("\n")) {
137+
// [-rw-------","6b851643360e435eb87ef9a6ab64a8b1/6b851643360e435eb87ef9a6ab64a8b1","5","2025-07-15","06:12","a.txt"]
138+
const w = x.split(/\s+/);
139+
if (w.length >= 6) {
140+
let isdir, name;
141+
if (w[5].endsWith("@") || w[5].endsWith("=") || w[5].endsWith("|")) {
142+
w[5] = w[5].slice(0, -1);
143+
}
144+
if (w[5].endsWith("/")) {
145+
isdir = true;
146+
name = w[5].slice(0, -1);
147+
} else {
148+
name = w[5];
149+
isdir = false;
150+
}
151+
const size = parseInt(w[2]);
152+
const mtime = new Date(w[3] + "T" + w[4]).valueOf() / 1000;
153+
v.push({ name, size, mtime, isdir });
154+
}
155+
}
156+
return v;
157+
};
158+
159+
prune = async ({
160+
dailies = "1w",
161+
monthlies = "4m",
162+
all = "3d",
163+
}: { dailies?: string; monthlies?: string; all?: string } = {}) => {
164+
await sudo({
165+
command: "bup",
166+
args: [
167+
"-d",
168+
this.subvolume.filesystem.bup,
169+
"prune-older",
170+
`--keep-dailies-for=${dailies}`,
171+
`--keep-monthlies-for=${monthlies}`,
172+
`--keep-all-for=${all}`,
173+
"--unsafe",
174+
this.subvolume.name,
175+
],
176+
});
177+
};
178+
}

‎src/packages/file-server/btrfs/subvolume.ts

Lines changed: 5 additions & 160 deletions
Original file line numberDiff line numberDiff line change
@@ -7,13 +7,12 @@ import refCache from "@cocalc/util/refcache";
77
import { exists, listdir, mkdirp, sudo } from "./util";
88
import { join, normalize } from "path";
99
import { updateRollingSnapshots, type SnapshotCounts } from "./snapshots";
10-
import { type DirectoryListingEntry } from "@cocalc/util/types";
11-
import getLogger from "@cocalc/backend/logger";
1210
import { SubvolumeFilesystem } from "./subvolume-fs";
11+
import { SubvolumeBup } from "./subvolume-bup";
12+
import getLogger from "@cocalc/backend/logger";
1313

1414
export const SNAPSHOTS = ".snapshots";
1515
const SEND_SNAPSHOT_PREFIX = "send-";
16-
const BUP_SNAPSHOT = "temp-bup-snapshot";
1716
const PAD = 4;
1817

1918
const logger = getLogger("file-server:storage-btrfs:subvolume");
@@ -26,17 +25,19 @@ interface Options {
2625
export class Subvolume {
2726
public readonly name: string;
2827

29-
private filesystem: Filesystem;
28+
public readonly filesystem: Filesystem;
3029
public readonly path: string;
3130
public readonly snapshotsDir: string;
3231
public readonly fs: SubvolumeFilesystem;
32+
public readonly bup: SubvolumeBup;
3333

3434
constructor({ filesystem, name }: Options) {
3535
this.filesystem = filesystem;
3636
this.name = name;
3737
this.path = join(filesystem.opts.mount, name);
3838
this.snapshotsDir = join(this.path, SNAPSHOTS);
3939
this.fs = new SubvolumeFilesystem(this);
40+
this.bup = new SubvolumeBup(this);
4041
}
4142

4243
init = async () => {
@@ -81,10 +82,6 @@ export class Subvolume {
8182
return join(this.path, normalize(path));
8283
};
8384

84-
/////////////
85-
// Files
86-
/////////////
87-
8885
/////////////
8986
// QUOTA
9087
/////////////
@@ -259,158 +256,6 @@ export class Subvolume {
259256
return snapGen < pathGen;
260257
};
261258

262-
/////////////
263-
// BACKUPS
264-
// There is a single global dedup'd backup archive stored in the btrfs filesystem.
265-
// Obviously, admins should rsync this regularly to a separate location as a genuine
266-
// backup strategy.
267-
/////////////
268-
269-
// create a new bup backup
270-
createBupBackup = async ({
271-
// timeout used for bup index and bup save commands
272-
timeout = 30 * 60 * 1000,
273-
}: { timeout?: number } = {}) => {
274-
if (await this.snapshotExists(BUP_SNAPSHOT)) {
275-
logger.debug(`createBupBackup: deleting existing ${BUP_SNAPSHOT}`);
276-
await this.deleteSnapshot(BUP_SNAPSHOT);
277-
}
278-
try {
279-
logger.debug(
280-
`createBupBackup: creating ${BUP_SNAPSHOT} to get a consistent backup`,
281-
);
282-
await this.createSnapshot(BUP_SNAPSHOT);
283-
const target = join(this.snapshotsDir, BUP_SNAPSHOT);
284-
logger.debug(`createBupBackup: indexing ${BUP_SNAPSHOT}`);
285-
await sudo({
286-
command: "bup",
287-
args: [
288-
"-d",
289-
this.filesystem.bup,
290-
"index",
291-
"--exclude",
292-
join(target, ".snapshots"),
293-
"-x",
294-
target,
295-
],
296-
timeout,
297-
});
298-
logger.debug(`createBupBackup: saving ${BUP_SNAPSHOT}`);
299-
await sudo({
300-
command: "bup",
301-
args: [
302-
"-d",
303-
this.filesystem.bup,
304-
"save",
305-
"--strip",
306-
"-n",
307-
this.name,
308-
target,
309-
],
310-
timeout,
311-
});
312-
} finally {
313-
logger.debug(`createBupBackup: deleting temporary ${BUP_SNAPSHOT}`);
314-
await this.deleteSnapshot(BUP_SNAPSHOT);
315-
}
316-
};
317-
318-
bupBackups = async (): Promise<string[]> => {
319-
const { stdout } = await sudo({
320-
command: "bup",
321-
args: ["-d", this.filesystem.bup, "ls", this.name],
322-
});
323-
return stdout
324-
.split("\n")
325-
.map((x) => x.split(" ").slice(-1)[0])
326-
.filter((x) => x);
327-
};
328-
329-
bupRestore = async (path: string) => {
330-
// path -- branch/revision/path/to/dir
331-
if (path.startsWith("/")) {
332-
path = path.slice(1);
333-
}
334-
path = normalize(path);
335-
// ... but to avoid potential data loss, we make a snapshot before deleting it.
336-
await this.createSnapshot();
337-
const i = path.indexOf("/"); // remove the commit name
338-
await sudo({
339-
command: "rm",
340-
args: ["-rf", this.normalize(path.slice(i + 1))],
341-
});
342-
await sudo({
343-
command: "bup",
344-
args: [
345-
"-d",
346-
this.filesystem.bup,
347-
"restore",
348-
"-C",
349-
this.path,
350-
join(`/${this.name}`, path),
351-
"--quiet",
352-
],
353-
});
354-
};
355-
356-
bupLs = async (path: string): Promise<DirectoryListingEntry[]> => {
357-
path = normalize(path);
358-
const { stdout } = await sudo({
359-
command: "bup",
360-
args: [
361-
"-d",
362-
this.filesystem.bup,
363-
"ls",
364-
"--almost-all",
365-
"--file-type",
366-
"-l",
367-
join(`/${this.name}`, path),
368-
],
369-
});
370-
const v: DirectoryListingEntry[] = [];
371-
for (const x of stdout.split("\n")) {
372-
// [-rw-------","6b851643360e435eb87ef9a6ab64a8b1/6b851643360e435eb87ef9a6ab64a8b1","5","2025-07-15","06:12","a.txt"]
373-
const w = x.split(/\s+/);
374-
if (w.length >= 6) {
375-
let isdir, name;
376-
if (w[5].endsWith("@") || w[5].endsWith("=") || w[5].endsWith("|")) {
377-
w[5] = w[5].slice(0, -1);
378-
}
379-
if (w[5].endsWith("/")) {
380-
isdir = true;
381-
name = w[5].slice(0, -1);
382-
} else {
383-
name = w[5];
384-
isdir = false;
385-
}
386-
const size = parseInt(w[2]);
387-
const mtime = new Date(w[3] + "T" + w[4]).valueOf() / 1000;
388-
v.push({ name, size, mtime, isdir });
389-
}
390-
}
391-
return v;
392-
};
393-
394-
bupPrune = async ({
395-
dailies = "1w",
396-
monthlies = "4m",
397-
all = "3d",
398-
}: { dailies?: string; monthlies?: string; all?: string } = {}) => {
399-
await sudo({
400-
command: "bup",
401-
args: [
402-
"-d",
403-
this.filesystem.bup,
404-
"prune-older",
405-
`--keep-dailies-for=${dailies}`,
406-
`--keep-monthlies-for=${monthlies}`,
407-
`--keep-all-for=${all}`,
408-
"--unsafe",
409-
this.name,
410-
],
411-
});
412-
};
413-
414259
/////////////
415260
// BTRFS send/recv
416261
// Not used. Instead we will rely on bup (and snapshots of the underlying disk) for backups, since:

0 commit comments

Comments
 (0)