Skip to content

Commit 3bebeef

Browse files
satyakighsatyakigh
committed
Replace in-process mutex with cross-process file locking for FileStore
1 parent cbcf884 commit 3bebeef

File tree

9 files changed

+221
-194
lines changed

9 files changed

+221
-194
lines changed

‎package-lock.json‎

Lines changed: 38 additions & 30 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎package.json‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -88,8 +88,8 @@
8888
"partial-json": "0.1.7",
8989
"pino": "9.9.0",
9090
"pino-pretty": "13.1.1",
91+
"proper-lockfile": "4.1.2",
9192
"pyodide": "0.28.2",
92-
"stack-utils": "2.0.6",
9393
"tree-sitter": "0.22.4",
9494
"tree-sitter-json": "0.24.8",
9595
"ts-essentials": "10.1.1",
@@ -106,8 +106,8 @@
106106
"@types/archiver": "7.0.0",
107107
"@types/js-yaml": "4.0.9",
108108
"@types/luxon": "3.7.1",
109+
"@types/proper-lockfile": "4.1.4",
109110
"@types/semver": "7.7.1",
110-
"@types/stack-utils": "2.0.3",
111111
"@types/yargs": "17.0.33",
112112
"@types/yauzl": "2.10.3",
113113
"@vitest/coverage-v8": "3.2.4",

‎src/datastore/FileStore.ts‎

Lines changed: 8 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,7 @@ export class FileStoreFactory implements DataStoreFactory {
2020
private readonly metricsInterval: NodeJS.Timeout;
2121
private readonly timeout: NodeJS.Timeout;
2222

23-
constructor(rootDir: string) {
23+
constructor(rootDir: string, storeNames: StoreName[] = [StoreName.public_schemas, StoreName.sam_schemas]) {
2424
this.log = LoggerFactory.getLogger('FileStore.Global');
2525

2626
this.fileDbRoot = join(rootDir, 'filedb');
@@ -30,6 +30,10 @@ export class FileStoreFactory implements DataStoreFactory {
3030
mkdirSync(this.fileDbDir, { recursive: true });
3131
}
3232

33+
for (const store of storeNames) {
34+
this.stores.set(store, new EncryptedFileStore(encryptionKey(VersionNumber), store, this.fileDbDir));
35+
}
36+
3337
this.metricsInterval = setInterval(() => {
3438
this.emitMetrics();
3539
}, 60 * 1000);
@@ -45,10 +49,9 @@ export class FileStoreFactory implements DataStoreFactory {
4549
}
4650

4751
get(store: StoreName): DataStore {
48-
let val = this.stores.get(store);
49-
if (!val) {
50-
val = new EncryptedFileStore(encryptionKey(VersionNumber), store, this.fileDbDir);
51-
this.stores.set(store, val);
52+
const val = this.stores.get(store);
53+
if (val === undefined) {
54+
throw new Error(`Store ${store} not found. Available stores: ${[...this.stores.keys()].join(', ')}`);
5255
}
5356
return val;
5457
}

‎src/datastore/file/EncryptedFileStore.ts‎

Lines changed: 66 additions & 73 deletions
Original file line numberDiff line numberDiff line change
@@ -1,122 +1,115 @@
1-
import { existsSync, readFileSync, statSync, unlinkSync } from 'fs'; // eslint-disable-line no-restricted-imports -- files being checked
1+
import { existsSync, readFileSync, statSync, writeFileSync } from 'fs'; // eslint-disable-line no-restricted-imports -- files being checked
22
import { writeFile } from 'fs/promises';
33
import { join } from 'path';
4-
import { Mutex } from 'async-mutex';
54
import { Logger } from 'pino';
5+
import { lock, lockSync } from 'proper-lockfile';
66
import { LoggerFactory } from '../../telemetry/LoggerFactory';
77
import { ScopedTelemetry } from '../../telemetry/ScopedTelemetry';
88
import { TelemetryService } from '../../telemetry/TelemetryService';
99
import { DataStore } from '../DataStore';
1010
import { decrypt, encrypt } from './Encryption';
1111

12+
const LOCK_OPTIONS = { stale: 10_000 }; // 10 seconds
13+
1214
export class EncryptedFileStore implements DataStore {
1315
private readonly log: Logger;
14-
1516
private readonly file: string;
16-
private content?: Record<string, unknown>;
17+
private content: Record<string, unknown> = {};
1718
private readonly telemetry: ScopedTelemetry;
18-
private readonly lock = new Mutex();
1919

2020
constructor(
2121
private readonly KEY: Buffer,
22-
private readonly name: string,
22+
name: string,
2323
fileDbDir: string,
2424
) {
2525
this.log = LoggerFactory.getLogger(`FileStore.${name}`);
2626
this.file = join(fileDbDir, `${name}.enc`);
27-
2827
this.telemetry = TelemetryService.instance.get(`FileStore.${name}`);
29-
}
30-
31-
get<T>(key: string): T | undefined {
32-
return this.telemetry.countExecution('get', () => {
33-
if (this.content) {
34-
return this.content[key] as T | undefined;
35-
}
3628

37-
if (!existsSync(this.file)) {
38-
return;
39-
}
40-
41-
if (this.lock.isLocked()) {
42-
return this.content?.[key];
29+
if (existsSync(this.file)) {
30+
try {
31+
this.content = this.readFile();
32+
} catch (error) {
33+
this.log.error(error, 'Failed to decrypt file store, recreating store');
34+
this.telemetry.count('filestore.recreate', 1);
35+
36+
const release = lockSync(this.file, LOCK_OPTIONS);
37+
try {
38+
this.saveSync();
39+
} finally {
40+
release();
41+
}
4342
}
43+
} else {
44+
this.saveSync();
45+
}
46+
}
4447

45-
const decrypted = decrypt(this.KEY, readFileSync(this.file));
46-
this.content = JSON.parse(decrypted) as Record<string, unknown>;
47-
return this.content[key] as T | undefined;
48-
});
48+
get<T>(key: string): T | undefined {
49+
return this.telemetry.countExecution('get', () => this.content[key] as T | undefined);
4950
}
5051

5152
put<T>(key: string, value: T): Promise<boolean> {
52-
return this.lock.runExclusive(() =>
53-
this.telemetry.measureAsync('put', async () => {
54-
if (!this.content) {
55-
this.get(key);
56-
}
57-
58-
this.content = {
59-
...this.content,
60-
[key]: value,
61-
};
62-
const encrypted = encrypt(this.KEY, JSON.stringify(this.content));
63-
await writeFile(this.file, encrypted);
64-
return true;
65-
}),
66-
);
53+
return this.withLock('put', async () => {
54+
this.content[key] = value;
55+
await this.save();
56+
return true;
57+
});
6758
}
6859

6960
remove(key: string): Promise<boolean> {
70-
return this.lock.runExclusive(() => {
71-
return this.telemetry.measureAsync('remove', async () => {
72-
if (!this.content) {
73-
this.get(key);
74-
}
75-
76-
if (!this.content || !(key in this.content)) {
77-
return false;
78-
}
61+
return this.withLock('remove', async () => {
62+
if (!(key in this.content)) {
63+
return false;
64+
}
7965

80-
delete this.content[key];
81-
const encrypted = encrypt(this.KEY, JSON.stringify(this.content));
82-
await writeFile(this.file, encrypted);
83-
return true;
84-
});
66+
delete this.content[key];
67+
await this.save();
68+
return true;
8569
});
8670
}
8771

8872
clear(): Promise<void> {
89-
return this.lock.runExclusive(() => {
90-
return this.telemetry.countExecutionAsync('clear', () => {
91-
if (existsSync(this.file)) {
92-
unlinkSync(this.file);
93-
}
94-
this.content = undefined;
95-
return Promise.resolve();
96-
});
73+
return this.withLock('clear', async () => {
74+
this.content = {};
75+
await this.save();
9776
});
9877
}
9978

10079
keys(limit: number): ReadonlyArray<string> {
101-
return this.telemetry.countExecution('keys', () => {
102-
if (!this.content) {
103-
this.get('ANY_KEY');
104-
}
105-
106-
return Object.keys(this.content ?? {}).slice(0, limit);
107-
});
80+
return this.telemetry.countExecution('keys', () => Object.keys(this.content).slice(0, limit));
10881
}
10982

11083
stats(): FileStoreStats {
111-
if (!this.content) {
112-
this.get('ANY_KEY');
113-
}
114-
11584
return {
116-
entries: Object.keys(this.content ?? {}).length,
85+
entries: Object.keys(this.content).length,
11786
totalSize: existsSync(this.file) ? statSync(this.file).size : 0,
11887
};
11988
}
89+
90+
private async withLock<T>(operation: string, fn: () => Promise<T>): Promise<T> {
91+
return await this.telemetry.measureAsync(operation, async () => {
92+
const release = await lock(this.file, LOCK_OPTIONS);
93+
try {
94+
this.content = this.readFile();
95+
return await fn();
96+
} finally {
97+
await release();
98+
}
99+
});
100+
}
101+
102+
private readFile(): Record<string, unknown> {
103+
return JSON.parse(decrypt(this.KEY, readFileSync(this.file))) as Record<string, unknown>;
104+
}
105+
106+
private saveSync() {
107+
writeFileSync(this.file, encrypt(this.KEY, JSON.stringify(this.content)));
108+
}
109+
110+
private async save() {
111+
await writeFile(this.file, encrypt(this.KEY, JSON.stringify(this.content)));
112+
}
120113
}
121114

122115
export type FileStoreStats = {

‎src/utils/ErrorStackInfo.ts‎

Lines changed: 0 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,3 @@
1-
import StackUtils from 'stack-utils';
21
import { LoggerFactory } from '../telemetry/LoggerFactory';
32

43
let errorStackInfo: string[] | undefined;
@@ -26,12 +25,3 @@ export function determineSensitiveInfo(): string[] {
2625

2726
return errorStackInfo;
2827
}
29-
30-
export function getErrorStack() {
31-
try {
32-
return new StackUtils({ cwd: process.cwd() });
33-
} catch (err) {
34-
LoggerFactory.getLogger('SensitiveInfo').warn(err, 'Cannot get process.cwd()');
35-
return new StackUtils();
36-
}
37-
}

0 commit comments

Comments
 (0)