Skip to content

Commit 2fe725a

Browse files
williamsteinwilliamstein
committed
btrfs: refactor quota
1 parent 8458477 commit 2fe725a

File tree

5 files changed

+118
-175
lines changed

5 files changed

+118
-175
lines changed

‎src/packages/file-server/btrfs/filesystem.ts

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -211,9 +211,9 @@ export class Filesystem {
211211
}
212212
const src = await this.subvolume(source);
213213
const vol = await this.subvolume(name);
214-
const { size } = await src.usage();
214+
const { size } = await src.quota.get();
215215
if (size) {
216-
await vol.size(size);
216+
await vol.quota.set(size);
217217
}
218218
return vol;
219219
};

‎src/packages/file-server/btrfs/subvolume-bup.ts

Lines changed: 21 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,24 @@
1+
/*
2+
3+
BUP Architecture:
4+
5+
There is a single global dedup'd backup archive stored in the btrfs filesystem.
6+
Obviously, admins should rsync this regularly to a separate location as a genuine
7+
backup strategy.
8+
9+
NOTE: we use bup instead of btrfs send/recv !
10+
11+
Not used. Instead we will rely on bup (and snapshots of the underlying disk) for backups, since:
12+
- much easier to check they are valid
13+
- decoupled from any btrfs issues
14+
- not tied to any specific filesystem at all
15+
- easier to offsite via incremental rsync
16+
- much more space efficient with *global* dedup and compression
17+
- bup is really just git, which is much more proven than even btrfs
18+
19+
The drawback is speed, but that can be managed.
20+
*/
21+
122
import { type DirectoryListingEntry } from "@cocalc/util/types";
223
import { type Subvolume } from "./subvolume";
324
import { sudo, parseBupTime } from "./util";
@@ -11,13 +32,6 @@ const logger = getLogger("file-server:storage-btrfs:subvolume-bup");
1132
export class SubvolumeBup {
1233
constructor(private subvolume: Subvolume) {}
1334

14-
/////////////
15-
// BACKUPS
16-
// There is a single global dedup'd backup archive stored in the btrfs filesystem.
17-
// Obviously, admins should rsync this regularly to a separate location as a genuine
18-
// backup strategy.
19-
/////////////
20-
2135
// create a new bup backup
2236
save = async ({
2337
// timeout used for bup index and bup save commands

‎src/packages/file-server/btrfs/subvolume-quota.ts

Lines changed: 86 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,86 @@
1+
import { type Subvolume } from "./subvolume";
2+
import { btrfs } from "./util";
3+
import getLogger from "@cocalc/backend/logger";
4+
5+
const logger = getLogger("file-server:storage-btrfs:subvolume-quota");
6+
7+
export class SubvolumeQuota {
8+
constructor(public subvolume: Subvolume) {}
9+
10+
private qgroup = async () => {
11+
const { stdout } = await btrfs({
12+
verbose: false,
13+
args: ["--format=json", "qgroup", "show", "-reF", this.subvolume.path],
14+
});
15+
const x = JSON.parse(stdout);
16+
return x["qgroup-show"][0];
17+
};
18+
19+
get = async (): Promise<{
20+
size: number;
21+
used: number;
22+
}> => {
23+
let { max_referenced: size, referenced: used } = await this.qgroup();
24+
if (size == "none") {
25+
size = null;
26+
}
27+
return {
28+
used,
29+
size,
30+
};
31+
};
32+
33+
set = async (size: string | number) => {
34+
if (!size) {
35+
throw Error("size must be specified");
36+
}
37+
logger.debug("setQuota ", this.subvolume.path, size);
38+
await btrfs({
39+
args: ["qgroup", "limit", `${size}`, this.subvolume.path],
40+
});
41+
};
42+
43+
du = async () => {
44+
return await btrfs({
45+
args: ["filesystem", "du", "-s", this.subvolume.path],
46+
});
47+
};
48+
49+
usage = async (): Promise<{
50+
// used and free in bytes
51+
used: number;
52+
free: number;
53+
size: number;
54+
}> => {
55+
const { stdout } = await btrfs({
56+
args: ["filesystem", "usage", "-b", this.subvolume.path],
57+
});
58+
let used: number = -1;
59+
let free: number = -1;
60+
let size: number = -1;
61+
for (const x of stdout.split("\n")) {
62+
if (used == -1) {
63+
const i = x.indexOf("Used:");
64+
if (i != -1) {
65+
used = parseInt(x.split(":")[1].trim());
66+
continue;
67+
}
68+
}
69+
if (free == -1) {
70+
const i = x.indexOf("Free (statfs, df):");
71+
if (i != -1) {
72+
free = parseInt(x.split(":")[1].trim());
73+
continue;
74+
}
75+
}
76+
if (size == -1) {
77+
const i = x.indexOf("Device size:");
78+
if (i != -1) {
79+
size = parseInt(x.split(":")[1].trim());
80+
continue;
81+
}
82+
}
83+
}
84+
return { used, free, size };
85+
};
86+
}

‎src/packages/file-server/btrfs/subvolume.ts

Lines changed: 5 additions & 162 deletions
Original file line numberDiff line numberDiff line change
@@ -4,16 +4,14 @@ A subvolume
44

55
import { type Filesystem, DEFAULT_SUBVOLUME_SIZE } from "./filesystem";
66
import refCache from "@cocalc/util/refcache";
7-
import { exists, listdir, mkdirp, sudo } from "./util";
7+
import { exists, sudo } from "./util";
88
import { join, normalize } from "path";
99
import { SubvolumeFilesystem } from "./subvolume-fs";
1010
import { SubvolumeBup } from "./subvolume-bup";
1111
import { SubvolumeSnapshots } from "./subvolume-snapshots";
12+
import { SubvolumeQuota } from "./subvolume-quota";
1213
import getLogger from "@cocalc/backend/logger";
1314

14-
const SEND_SNAPSHOT_PREFIX = "send-";
15-
const PAD = 4;
16-
1715
const logger = getLogger("file-server:storage-btrfs:subvolume");
1816

1917
interface Options {
@@ -29,6 +27,7 @@ export class Subvolume {
2927
public readonly fs: SubvolumeFilesystem;
3028
public readonly bup: SubvolumeBup;
3129
public readonly snapshots: SubvolumeSnapshots;
30+
public readonly quota: SubvolumeQuota;
3231

3332
constructor({ filesystem, name }: Options) {
3433
this.filesystem = filesystem;
@@ -37,6 +36,7 @@ export class Subvolume {
3736
this.fs = new SubvolumeFilesystem(this);
3837
this.bup = new SubvolumeBup(this);
3938
this.snapshots = new SubvolumeSnapshots(this);
39+
this.quota = new SubvolumeQuota(this);
4040
}
4141

4242
init = async () => {
@@ -47,7 +47,7 @@ export class Subvolume {
4747
args: ["subvolume", "create", this.path],
4848
});
4949
await this.chown(this.path);
50-
await this.size(
50+
await this.quota.set(
5151
this.filesystem.opts.defaultSize ?? DEFAULT_SUBVOLUME_SIZE,
5252
);
5353
}
@@ -80,163 +80,6 @@ export class Subvolume {
8080
normalize = (path: string) => {
8181
return join(this.path, normalize(path));
8282
};
83-
84-
/////////////
85-
// QUOTA
86-
/////////////
87-
88-
private quotaInfo = async () => {
89-
const { stdout } = await sudo({
90-
verbose: false,
91-
command: "btrfs",
92-
args: ["--format=json", "qgroup", "show", "-reF", this.path],
93-
});
94-
const x = JSON.parse(stdout);
95-
return x["qgroup-show"][0];
96-
};
97-
98-
quota = async (): Promise<{
99-
size: number;
100-
used: number;
101-
}> => {
102-
let { max_referenced: size, referenced: used } = await this.quotaInfo();
103-
if (size == "none") {
104-
size = null;
105-
}
106-
return {
107-
used,
108-
size,
109-
};
110-
};
111-
112-
size = async (size: string | number) => {
113-
if (!size) {
114-
throw Error("size must be specified");
115-
}
116-
await sudo({
117-
command: "btrfs",
118-
args: ["qgroup", "limit", `${size}`, this.path],
119-
});
120-
};
121-
122-
du = async () => {
123-
return await sudo({
124-
command: "btrfs",
125-
args: ["filesystem", "du", "-s", this.path],
126-
});
127-
};
128-
129-
usage = async (): Promise<{
130-
// used and free in bytes
131-
used: number;
132-
free: number;
133-
size: number;
134-
}> => {
135-
const { stdout } = await sudo({
136-
command: "btrfs",
137-
args: ["filesystem", "usage", "-b", this.path],
138-
});
139-
let used: number = -1;
140-
let free: number = -1;
141-
let size: number = -1;
142-
for (const x of stdout.split("\n")) {
143-
if (used == -1) {
144-
const i = x.indexOf("Used:");
145-
if (i != -1) {
146-
used = parseInt(x.split(":")[1].trim());
147-
continue;
148-
}
149-
}
150-
if (free == -1) {
151-
const i = x.indexOf("Free (statfs, df):");
152-
if (i != -1) {
153-
free = parseInt(x.split(":")[1].trim());
154-
continue;
155-
}
156-
}
157-
if (size == -1) {
158-
const i = x.indexOf("Device size:");
159-
if (i != -1) {
160-
size = parseInt(x.split(":")[1].trim());
161-
continue;
162-
}
163-
}
164-
}
165-
return { used, free, size };
166-
};
167-
168-
/////////////
169-
// BTRFS send/recv
170-
// Not used. Instead we will rely on bup (and snapshots of the underlying disk) for backups, since:
171-
// - much easier to check they are valid
172-
// - decoupled from any btrfs issues
173-
// - not tied to any specific filesystem at all
174-
// - easier to offsite via incremntal rsync
175-
// - much more space efficient with *global* dedup and compression
176-
// - bup is really just git, which is very proven
177-
// The drawback is speed.
178-
/////////////
179-
180-
// this was just a quick proof of concept -- I don't like it. Should switch to using
181-
// timestamps and a lock.
182-
// To recover these, doing recv for each in order does work. Then you have to
183-
// snapshot all of the results to move them. It's awkward, but efficient
184-
// and works fine.
185-
send = async () => {
186-
await mkdirp([join(this.filesystem.streams, this.name)]);
187-
const streams = new Set(
188-
await listdir(join(this.filesystem.streams, this.name)),
189-
);
190-
const allSnapshots = (await this.snapshots.ls()).map((x) => x.name);
191-
const snapshots = allSnapshots.filter(
192-
(x) => x.startsWith(SEND_SNAPSHOT_PREFIX) && streams.has(x),
193-
);
194-
const nums = snapshots.map((x) =>
195-
parseInt(x.slice(SEND_SNAPSHOT_PREFIX.length)),
196-
);
197-
nums.sort();
198-
const last = nums.slice(-1)[0];
199-
let seq, parent;
200-
if (last) {
201-
seq = `${last + 1}`.padStart(PAD, "0");
202-
const l = `${last}`.padStart(PAD, "0");
203-
parent = `${SEND_SNAPSHOT_PREFIX}${l}`;
204-
} else {
205-
seq = "1".padStart(PAD, "0");
206-
parent = "";
207-
}
208-
const send = `${SEND_SNAPSHOT_PREFIX}${seq}`;
209-
if (allSnapshots.includes(send)) {
210-
await this.snapshots.delete(send);
211-
}
212-
await this.snapshots.create(send);
213-
await sudo({
214-
command: "btrfs",
215-
args: [
216-
"send",
217-
"--compressed-data",
218-
join(this.snapshots.path(), send),
219-
...(last ? ["-p", this.snapshots.path(parent)] : []),
220-
"-f",
221-
join(this.filesystem.streams, this.name, send),
222-
],
223-
});
224-
if (parent) {
225-
await this.snapshots.delete(parent);
226-
}
227-
};
228-
229-
// recv = async (target: string) => {
230-
// const streamsDir = join(this.filesystem.streams, this.name);
231-
// const streams = await listdir(streamsDir);
232-
// streams.sort();
233-
// for (const stream of streams) {
234-
// await sudo({
235-
// command: "btrfs",
236-
// args: ["recv", "-f", join(streamsDir, stream)],
237-
// });
238-
// }
239-
// };
24083
}
24184

24285
const cache = refCache<Options & { noCache?: boolean }, Subvolume>({

‎src/packages/file-server/btrfs/test/subvolume.test.ts

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -11,9 +11,9 @@ describe("setting and getting quota of a subvolume", () => {
1111
let vol: Subvolume;
1212
it("set the quota of a subvolume to 5 M", async () => {
1313
vol = await fs.subvolume("q");
14-
await vol.size("5M");
14+
await vol.quota.set("5M");
1515

16-
const { size, used } = await vol.quota();
16+
const { size, used } = await vol.quota.get();
1717
expect(size).toBe(5 * 1024 * 1024);
1818
expect(used).toBe(0);
1919
});
@@ -29,11 +29,11 @@ describe("setting and getting quota of a subvolume", () => {
2929
await wait({
3030
until: async () => {
3131
await sudo({ command: "sync" });
32-
const { used } = await vol.usage();
32+
const { used } = await vol.quota.usage();
3333
return used > 0;
3434
},
3535
});
36-
const { used } = await vol.usage();
36+
const { used } = await vol.quota.usage();
3737
expect(used).toBeGreaterThan(0);
3838

3939
const v = await vol.fs.ls("");

0 commit comments

Comments
 (0)