Skip to content

Commit cdabf10

Browse files
satyakighsatyakigh
committed
Update filedb startup behaviour to create files on startup
1 parent eae1d57 commit cdabf10

File tree

3 files changed

+88
-54
lines changed

3 files changed

+88
-54
lines changed

‎src/datastore/FileStore.ts‎

Lines changed: 8 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,7 @@ export class FileStoreFactory implements DataStoreFactory {
2020
private readonly metricsInterval: NodeJS.Timeout;
2121
private readonly timeout: NodeJS.Timeout;
2222

23-
constructor(rootDir: string) {
23+
constructor(rootDir: string, storeNames: StoreName[] = [StoreName.public_schemas, StoreName.sam_schemas]) {
2424
this.log = LoggerFactory.getLogger('FileStore.Global');
2525

2626
this.fileDbRoot = join(rootDir, 'filedb');
@@ -30,6 +30,10 @@ export class FileStoreFactory implements DataStoreFactory {
3030
mkdirSync(this.fileDbDir, { recursive: true });
3131
}
3232

33+
for (const store of storeNames) {
34+
this.stores.set(store, new EncryptedFileStore(encryptionKey(VersionNumber), store, this.fileDbDir));
35+
}
36+
3337
this.metricsInterval = setInterval(() => {
3438
this.emitMetrics();
3539
}, 60 * 1000);
@@ -45,10 +49,9 @@ export class FileStoreFactory implements DataStoreFactory {
4549
}
4650

4751
get(store: StoreName): DataStore {
48-
let val = this.stores.get(store);
49-
if (!val) {
50-
val = new EncryptedFileStore(encryptionKey(VersionNumber), store, this.fileDbDir);
51-
this.stores.set(store, val);
52+
const val = this.stores.get(store);
53+
if (val === undefined) {
54+
throw new Error(`Store ${store} not found. Available stores: ${[...this.stores.keys()].join(', ')}`);
5255
}
5356
return val;
5457
}

‎src/datastore/file/EncryptedFileStore.ts‎

Lines changed: 36 additions & 48 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
import { existsSync, readFileSync, statSync, unlinkSync } from 'fs'; // eslint-disable-line no-restricted-imports -- files being checked
1+
import { existsSync, readFileSync, statSync, unlinkSync, writeFileSync } from 'fs'; // eslint-disable-line no-restricted-imports -- files being checked
22
import { writeFile } from 'fs/promises';
33
import { join } from 'path';
44
import { Mutex } from 'async-mutex';
@@ -13,7 +13,7 @@ export class EncryptedFileStore implements DataStore {
1313
private readonly log: Logger;
1414

1515
private readonly file: string;
16-
private content?: Record<string, unknown>;
16+
private content: Record<string, unknown>;
1717
private readonly telemetry: ScopedTelemetry;
1818
private readonly lock = new Mutex();
1919

@@ -24,43 +24,36 @@ export class EncryptedFileStore implements DataStore {
2424
) {
2525
this.log = LoggerFactory.getLogger(`FileStore.${name}`);
2626
this.file = join(fileDbDir, `${name}.enc`);
27-
2827
this.telemetry = TelemetryService.instance.get(`FileStore.${name}`);
28+
29+
this.content = {};
30+
if (existsSync(this.file)) {
31+
try {
32+
const decrypted = decrypt(this.KEY, readFileSync(this.file));
33+
this.content = JSON.parse(decrypted) as Record<string, unknown>;
34+
} catch (error) {
35+
this.log.error(error, 'Failed to decrypt file store, recreating store');
36+
this.telemetry.count('filestore.recreate', 1);
37+
38+
this.deleteStore();
39+
writeFileSync(this.file, encrypt(this.KEY, JSON.stringify(this.content)));
40+
}
41+
} else {
42+
writeFileSync(this.file, encrypt(this.KEY, JSON.stringify(this.content)));
43+
}
2944
}
3045

3146
get<T>(key: string): T | undefined {
3247
return this.telemetry.countExecution('get', () => {
33-
if (this.content) {
34-
return this.content[key] as T | undefined;
35-
}
36-
37-
if (!existsSync(this.file)) {
38-
return;
39-
}
40-
41-
if (this.lock.isLocked()) {
42-
return this.content?.[key];
43-
}
44-
45-
const decrypted = decrypt(this.KEY, readFileSync(this.file));
46-
this.content = JSON.parse(decrypted) as Record<string, unknown>;
4748
return this.content[key] as T | undefined;
4849
});
4950
}
5051

5152
put<T>(key: string, value: T): Promise<boolean> {
5253
return this.lock.runExclusive(() =>
5354
this.telemetry.measureAsync('put', async () => {
54-
if (!this.content) {
55-
this.get(key);
56-
}
57-
58-
this.content = {
59-
...this.content,
60-
[key]: value,
61-
};
62-
const encrypted = encrypt(this.KEY, JSON.stringify(this.content));
63-
await writeFile(this.file, encrypted);
55+
this.content[key] = value;
56+
await this.save();
6457
return true;
6558
}),
6659
);
@@ -69,17 +62,12 @@ export class EncryptedFileStore implements DataStore {
6962
remove(key: string): Promise<boolean> {
7063
return this.lock.runExclusive(() => {
7164
return this.telemetry.measureAsync('remove', async () => {
72-
if (!this.content) {
73-
this.get(key);
74-
}
75-
76-
if (!this.content || !(key in this.content)) {
65+
if (!(key in this.content)) {
7766
return false;
7867
}
7968

8069
delete this.content[key];
81-
const encrypted = encrypt(this.KEY, JSON.stringify(this.content));
82-
await writeFile(this.file, encrypted);
70+
await this.save();
8371
return true;
8472
});
8573
});
@@ -88,35 +76,35 @@ export class EncryptedFileStore implements DataStore {
8876
clear(): Promise<void> {
8977
return this.lock.runExclusive(() => {
9078
return this.telemetry.countExecutionAsync('clear', () => {
91-
if (existsSync(this.file)) {
92-
unlinkSync(this.file);
93-
}
94-
this.content = undefined;
79+
this.deleteStore();
9580
return Promise.resolve();
9681
});
9782
});
9883
}
9984

10085
keys(limit: number): ReadonlyArray<string> {
10186
return this.telemetry.countExecution('keys', () => {
102-
if (!this.content) {
103-
this.get('ANY_KEY');
104-
}
105-
106-
return Object.keys(this.content ?? {}).slice(0, limit);
87+
return Object.keys(this.content).slice(0, limit);
10788
});
10889
}
10990

11091
stats(): FileStoreStats {
111-
if (!this.content) {
112-
this.get('ANY_KEY');
113-
}
114-
11592
return {
116-
entries: Object.keys(this.content ?? {}).length,
93+
entries: Object.keys(this.content).length,
11794
totalSize: existsSync(this.file) ? statSync(this.file).size : 0,
11895
};
11996
}
97+
98+
private deleteStore() {
99+
if (existsSync(this.file)) {
100+
unlinkSync(this.file);
101+
}
102+
this.content = {};
103+
}
104+
105+
private async save() {
106+
await writeFile(this.file, encrypt(this.KEY, JSON.stringify(this.content)));
107+
}
120108
}
121109

122110
export type FileStoreStats = {

‎tst/unit/datastore/FileStore.test.ts‎

Lines changed: 44 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,10 @@
1-
import { rmSync } from 'fs';
1+
import { rmSync, mkdirSync, writeFileSync } from 'fs';
22
import { join } from 'path';
33
import { v4 } from 'uuid';
44
import { describe, it, expect, beforeEach, afterEach } from 'vitest';
55
import { DataStore, StoreName } from '../../../src/datastore/DataStore';
6+
import { EncryptedFileStore } from '../../../src/datastore/file/EncryptedFileStore';
7+
import { encryptionKey } from '../../../src/datastore/file/Encryption';
68
import { FileStoreFactory } from '../../../src/datastore/FileStore';
79

810
describe('FileStore', () => {
@@ -204,6 +206,25 @@ describe('FileStore', () => {
204206
});
205207

206208
describe('persistence', () => {
209+
it('should preserve existing data when put is called on fresh instance', async () => {
210+
// Test EncryptedFileStore directly to avoid stats() being called
211+
const encTestDir = join(testDir, 'enc-test');
212+
mkdirSync(encTestDir, { recursive: true });
213+
const key = encryptionKey(2);
214+
215+
// Session 1: write key1
216+
const store1 = new EncryptedFileStore(key, 'test', encTestDir);
217+
await store1.put('key1', 'value1');
218+
219+
// Session 2: fresh instance, put key2 WITHOUT reading first
220+
const store2 = new EncryptedFileStore(key, 'test', encTestDir);
221+
await store2.put('key2', 'value2');
222+
223+
// Verify both keys exist - key1 should NOT be lost
224+
expect(store2.get('key1')).toBe('value1');
225+
expect(store2.get('key2')).toBe('value2');
226+
});
227+
207228
it('should persist data across store instances', async () => {
208229
const key = 'persist-key';
209230
const value = { data: 'persist-value' };
@@ -294,4 +315,26 @@ describe('FileStore', () => {
294315
await expect(fileStore.clear()).resolves.not.toThrow();
295316
});
296317
});
318+
319+
describe('recovery', () => {
320+
it('should recover from corrupted file and allow new writes', async () => {
321+
const encTestDir = join(testDir, 'recovery-test');
322+
mkdirSync(encTestDir, { recursive: true });
323+
const key = encryptionKey(2);
324+
325+
// Write corrupted data to the file
326+
const corruptedFile = join(encTestDir, 'test.enc');
327+
writeFileSync(corruptedFile, 'corrupted-not-encrypted-data');
328+
329+
// Should not throw, should recover
330+
const store = new EncryptedFileStore(key, 'test', encTestDir);
331+
332+
// Should start with empty content after recovery
333+
expect(store.get('anyKey')).toBeUndefined();
334+
335+
// Should be able to write new data
336+
await store.put('newKey', 'newValue');
337+
expect(store.get('newKey')).toBe('newValue');
338+
});
339+
});
297340
});

0 commit comments

Comments
 (0)