Skip to content

Commit 0268a40

Browse files
williamsteinwilliamstein
committed
btrfs: refactor fs code
1 parent 4e2563a commit 0268a40

File tree

6 files changed

+219
-210
lines changed

6 files changed

+219
-210
lines changed

‎src/packages/file-server/btrfs/subvolume-fs.ts

Lines changed: 150 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,150 @@
1+
import {
2+
appendFile,
3+
chmod,
4+
cp,
5+
copyFile,
6+
link,
7+
readFile,
8+
realpath,
9+
rename,
10+
rm,
11+
rmdir,
12+
mkdir,
13+
stat,
14+
symlink,
15+
truncate,
16+
writeFile,
17+
unlink,
18+
utimes,
19+
watch,
20+
} from "node:fs/promises";
21+
import { exists } from "@cocalc/backend/misc/async-utils-node";
22+
import { type DirectoryListingEntry } from "@cocalc/util/types";
23+
import getListing from "@cocalc/backend/get-listing";
24+
import { type Subvolume } from "./subvolume";
25+
import { isdir, sudo } from "./util";
26+
27+
export class SubvolumeFilesystem {
28+
constructor(private subvolume: Subvolume) {}
29+
30+
private normalize = this.subvolume.normalize;
31+
32+
ls = async (
33+
path: string,
34+
{ hidden, limit }: { hidden?: boolean; limit?: number } = {},
35+
): Promise<DirectoryListingEntry[]> => {
36+
return await getListing(this.normalize(path), hidden, {
37+
limit,
38+
home: "/",
39+
});
40+
};
41+
42+
readFile = async (path: string, encoding?: any): Promise<string | Buffer> => {
43+
return await readFile(this.normalize(path), encoding);
44+
};
45+
46+
writeFile = async (path: string, data: string | Buffer) => {
47+
return await writeFile(this.normalize(path), data);
48+
};
49+
50+
appendFile = async (path: string, data: string | Buffer, encoding?) => {
51+
return await appendFile(this.normalize(path), data, encoding);
52+
};
53+
54+
unlink = async (path: string) => {
55+
await unlink(this.normalize(path));
56+
};
57+
58+
stat = async (path: string) => {
59+
return await stat(this.normalize(path));
60+
};
61+
62+
exists = async (path: string) => {
63+
return await exists(this.normalize(path));
64+
};
65+
66+
// hard link
67+
link = async (existingPath: string, newPath: string) => {
68+
return await link(this.normalize(existingPath), this.normalize(newPath));
69+
};
70+
71+
symlink = async (target: string, path: string) => {
72+
return await symlink(this.normalize(target), this.normalize(path));
73+
};
74+
75+
realpath = async (path: string) => {
76+
const x = await realpath(this.normalize(path));
77+
return x.slice(this.subvolume.path.length + 1);
78+
};
79+
80+
rename = async (oldPath: string, newPath: string) => {
81+
await rename(this.normalize(oldPath), this.normalize(newPath));
82+
};
83+
84+
utimes = async (
85+
path: string,
86+
atime: number | string | Date,
87+
mtime: number | string | Date,
88+
) => {
89+
await utimes(this.normalize(path), atime, mtime);
90+
};
91+
92+
watch = (filename: string, options?) => {
93+
return watch(this.normalize(filename), options);
94+
};
95+
96+
truncate = async (path: string, len?: number) => {
97+
await truncate(this.normalize(path), len);
98+
};
99+
100+
copyFile = async (src: string, dest: string) => {
101+
await copyFile(this.normalize(src), this.normalize(dest));
102+
};
103+
104+
cp = async (src: string, dest: string, options?) => {
105+
await cp(this.normalize(src), this.normalize(dest), options);
106+
};
107+
108+
chmod = async (path: string, mode: string | number) => {
109+
await chmod(this.normalize(path), mode);
110+
};
111+
112+
mkdir = async (path: string, options?) => {
113+
await mkdir(this.normalize(path), options);
114+
};
115+
116+
rsync = async ({
117+
src,
118+
target,
119+
args = ["-axH"],
120+
timeout = 5 * 60 * 1000,
121+
}: {
122+
src: string;
123+
target: string;
124+
args?: string[];
125+
timeout?: number;
126+
}): Promise<{ stdout: string; stderr: string; exit_code: number }> => {
127+
let srcPath = this.normalize(src);
128+
let targetPath = this.normalize(target);
129+
if (!srcPath.endsWith("/") && (await isdir(srcPath))) {
130+
srcPath += "/";
131+
if (!targetPath.endsWith("/")) {
132+
targetPath += "/";
133+
}
134+
}
135+
return await sudo({
136+
command: "rsync",
137+
args: [...args, srcPath, targetPath],
138+
err_on_exit: false,
139+
timeout: timeout / 1000,
140+
});
141+
};
142+
143+
rmdir = async (path: string, options?) => {
144+
await rmdir(this.normalize(path), options);
145+
};
146+
147+
rm = async (path: string, options?) => {
148+
await rm(this.normalize(path), options);
149+
};
150+
}

‎src/packages/file-server/btrfs/subvolume.ts

Lines changed: 8 additions & 149 deletions
Original file line numberDiff line numberDiff line change
@@ -4,33 +4,12 @@ A subvolume
44

55
import { type Filesystem, DEFAULT_SUBVOLUME_SIZE } from "./filesystem";
66
import refCache from "@cocalc/util/refcache";
7-
import {
8-
appendFile,
9-
chmod,
10-
cp,
11-
copyFile,
12-
link,
13-
readFile,
14-
realpath,
15-
rename,
16-
rm,
17-
rmdir,
18-
mkdir,
19-
stat,
20-
symlink,
21-
truncate,
22-
writeFile,
23-
unlink,
24-
utimes,
25-
watch,
26-
} from "node:fs/promises";
27-
import { exists, isdir, listdir, mkdirp, sudo } from "./util";
7+
import { exists, listdir, mkdirp, sudo } from "./util";
288
import { join, normalize } from "path";
299
import { updateRollingSnapshots, type SnapshotCounts } from "./snapshots";
30-
import { DirectoryListingEntry } from "@cocalc/util/types";
31-
import getListing from "@cocalc/backend/get-listing";
10+
import { type DirectoryListingEntry } from "@cocalc/util/types";
3211
import getLogger from "@cocalc/backend/logger";
33-
import { exists as pathExists } from "@cocalc/backend/misc/async-utils-node";
12+
import { SubvolumeFilesystem } from "./subvolume-fs";
3413

3514
export const SNAPSHOTS = ".snapshots";
3615
const SEND_SNAPSHOT_PREFIX = "send-";
@@ -48,14 +27,16 @@ export class Subvolume {
4827
public readonly name: string;
4928

5029
private filesystem: Filesystem;
51-
private readonly path: string;
52-
private readonly snapshotsDir: string;
30+
public readonly path: string;
31+
public readonly snapshotsDir: string;
32+
public readonly fs: SubvolumeFilesystem;
5333

5434
constructor({ filesystem, name }: Options) {
5535
this.filesystem = filesystem;
5636
this.name = name;
5737
this.path = join(filesystem.opts.mount, name);
5838
this.snapshotsDir = join(this.path, SNAPSHOTS);
39+
this.fs = new SubvolumeFilesystem(this);
5940
}
6041

6142
init = async () => {
@@ -96,135 +77,13 @@ export class Subvolume {
9677
// this should provide a path that is guaranteed to be
9778
// inside this.path on the filesystem or throw error
9879
// [ ] TODO: not sure if the code here is sufficient!!
99-
private normalize = (path: string) => {
80+
normalize = (path: string) => {
10081
return join(this.path, normalize(path));
10182
};
10283

10384
/////////////
10485
// Files
10586
/////////////
106-
ls = async (
107-
path: string,
108-
{ hidden, limit }: { hidden?: boolean; limit?: number } = {},
109-
): Promise<DirectoryListingEntry[]> => {
110-
path = normalize(path);
111-
return await getListing(this.normalize(path), hidden, {
112-
limit,
113-
home: "/",
114-
});
115-
};
116-
117-
readFile = async (path: string, encoding?: any): Promise<string | Buffer> => {
118-
path = normalize(path);
119-
return await readFile(this.normalize(path), encoding);
120-
};
121-
122-
writeFile = async (path: string, data: string | Buffer) => {
123-
path = normalize(path);
124-
return await writeFile(this.normalize(path), data);
125-
};
126-
127-
appendFile = async (path: string, data: string | Buffer, encoding?) => {
128-
path = normalize(path);
129-
return await appendFile(this.normalize(path), data, encoding);
130-
};
131-
132-
unlink = async (path: string) => {
133-
await unlink(this.normalize(path));
134-
};
135-
136-
stat = async (path: string) => {
137-
return await stat(this.normalize(path));
138-
};
139-
140-
exists = async (path: string) => {
141-
return await pathExists(this.normalize(path));
142-
};
143-
144-
// hard link
145-
link = async (existingPath: string, newPath: string) => {
146-
return await link(this.normalize(existingPath), this.normalize(newPath));
147-
};
148-
149-
symlink = async (target: string, path: string) => {
150-
return await symlink(this.normalize(target), this.normalize(path));
151-
};
152-
153-
realpath = async (path: string) => {
154-
const x = await realpath(this.normalize(path));
155-
return x.slice(this.path.length + 1);
156-
};
157-
158-
rename = async (oldPath: string, newPath: string) => {
159-
await rename(this.normalize(oldPath), this.normalize(newPath));
160-
};
161-
162-
utimes = async (
163-
path: string,
164-
atime: number | string | Date,
165-
mtime: number | string | Date,
166-
) => {
167-
await utimes(this.normalize(path), atime, mtime);
168-
};
169-
170-
watch = (filename: string, options?) => {
171-
return watch(this.normalize(filename), options);
172-
};
173-
174-
truncate = async (path: string, len?: number) => {
175-
await truncate(this.normalize(path), len);
176-
};
177-
178-
copyFile = async (src: string, dest: string) => {
179-
await copyFile(this.normalize(src), this.normalize(dest));
180-
};
181-
182-
cp = async (src: string, dest: string, options?) => {
183-
await cp(this.normalize(src), this.normalize(dest), options);
184-
};
185-
186-
chmod = async (path: string, mode: string | number) => {
187-
await chmod(this.normalize(path), mode);
188-
};
189-
190-
mkdir = async (path: string, options?) => {
191-
await mkdir(this.normalize(path), options);
192-
};
193-
194-
rsync = async ({
195-
src,
196-
target,
197-
args = ["-axH"],
198-
timeout = 5 * 60 * 1000,
199-
}: {
200-
src: string;
201-
target: string;
202-
args?: string[];
203-
timeout?: number;
204-
}): Promise<{ stdout: string; stderr: string; exit_code: number }> => {
205-
let srcPath = this.normalize(src);
206-
let targetPath = this.normalize(target);
207-
if (!srcPath.endsWith("/") && (await isdir(srcPath))) {
208-
srcPath += "/";
209-
if (!targetPath.endsWith("/")) {
210-
targetPath += "/";
211-
}
212-
}
213-
return await sudo({
214-
command: "rsync",
215-
args: [...args, srcPath, targetPath],
216-
err_on_exit: false,
217-
timeout: timeout / 1000,
218-
});
219-
};
220-
221-
rmdir = async (path: string, options?) => {
222-
await rmdir(this.normalize(path), options);
223-
};
224-
225-
rm = async (path: string, options?) => {
226-
await rm(this.normalize(path), options);
227-
};
22887

22988
/////////////
23089
// QUOTA

‎src/packages/file-server/btrfs/test/filesystem-stress.test.ts

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -43,7 +43,7 @@ describe("stress operations with subvolumes", () => {
4343
it("write a file to each volume", async () => {
4444
for (const name of await fs.list()) {
4545
const vol = await fs.subvolume(name);
46-
await vol.writeFile("a.txt", "hi");
46+
await vol.fs.writeFile("a.txt", "hi");
4747
}
4848
});
4949

‎src/packages/file-server/btrfs/test/filesystem.test.ts

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -61,16 +61,16 @@ describe("operations with subvolumes", () => {
6161
it("rsync an actual file", async () => {
6262
const sagemath = await fs.subvolume("sagemath");
6363
const cython = await fs.subvolume("cython");
64-
await sagemath.writeFile("README.md", "hi");
64+
await sagemath.fs.writeFile("README.md", "hi");
6565
await fs.rsync({ src: "sagemath", target: "cython" });
66-
const copy = await cython.readFile("README.md", "utf8");
66+
const copy = await cython.fs.readFile("README.md", "utf8");
6767
expect(copy).toEqual("hi");
6868
});
6969

7070
it("clone a subvolume with contents", async () => {
7171
await fs.cloneSubvolume("cython", "pyrex");
7272
const pyrex = await fs.subvolume("pyrex");
73-
const clone = await pyrex.readFile("README.md", "utf8");
73+
const clone = await pyrex.fs.readFile("README.md", "utf8");
7474
expect(clone).toEqual("hi");
7575
});
7676
});

‎src/packages/file-server/btrfs/test/subvolume-stress.test.ts

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -55,7 +55,7 @@ describe(`create ${numFiles} files`, () => {
5555
log(
5656
`created ${Math.round((numFiles / (Date.now() - start)) * 1000)} files per second in serial`,
5757
);
58-
const v = await vol.ls("");
58+
const v = await vol.fs.ls("");
5959
const w = v.map(({ name }) => name);
6060
expect(w.sort()).toEqual(names.sort());
6161
});
@@ -74,7 +74,7 @@ describe(`create ${numFiles} files`, () => {
7474
`created ${Math.round((numFiles / (Date.now() - start)) * 1000)} files per second in parallel`,
7575
);
7676
const t0 = Date.now();
77-
const v = await vol.ls("p");
77+
const v = await vol.fs.ls("p");
7878
log("get listing of files took", Date.now() - t0, "ms");
7979
const w = v.map(({ name }) => name);
8080
expect(w.sort()).toEqual(names.sort());

0 commit comments

Comments
 (0)