Skip to content

Commit 87c7fa9

Browse files
nirinchevnirinchev
committed
feat: add support for recursively cleaning up old log files
1 parent e810a2d commit 87c7fa9

File tree

2 files changed

+145
-55
lines changed

2 files changed

+145
-55
lines changed

‎packages/mongodb-log-writer/src/mongo-log-manager.spec.ts

Lines changed: 101 additions & 36 deletions
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,7 @@ describe('MongoLogManager', function () {
2222
onerror = sinon.stub();
2323
directory = path.join(
2424
os.tmpdir(),
25-
`log-writer-test-${Math.random()}-${Date.now()}`
25+
`log-writer-test-${Math.random()}-${Date.now()}`,
2626
);
2727
await fs.mkdir(directory, { recursive: true });
2828
});
@@ -84,10 +84,10 @@ describe('MongoLogManager', function () {
8484

8585
const writer = await manager.createLogWriter();
8686
expect(
87-
path.relative(directory, writer.logFilePath as string)[0]
87+
path.relative(directory, writer.logFilePath as string)[0],
8888
).to.not.equal('.');
8989
expect((writer.logFilePath as string).includes(writer.logId)).to.equal(
90-
true
90+
true,
9191
);
9292

9393
writer.info('component', mongoLogId(12345), 'context', 'message', {
@@ -147,15 +147,54 @@ describe('MongoLogManager', function () {
147147
}
148148
});
149149

150+
it('can recursively clean up old files', async function () {
151+
const childDirectory = path.join(directory, 'child', '1');
152+
await fs.mkdir(childDirectory, { recursive: true });
153+
// The child manager writes in a subdirectory of the log directory
154+
// The expectation is that when the parent manager recursively cleans up the
155+
// old log files, it will be able to delete the files of the child manager.
156+
const childManager = new MongoLogManager({
157+
directory: childDirectory,
158+
retentionDays: 60,
159+
onwarn,
160+
onerror,
161+
});
162+
163+
const childWriter = await childManager.createLogWriter();
164+
childWriter.info('child', mongoLogId(12345), 'context', 'message');
165+
166+
childWriter.end();
167+
await once(childWriter, 'finish');
168+
await fs.stat(childWriter.logFilePath as string);
169+
await new Promise((resolve) => setTimeout(resolve, 100));
170+
171+
const parentManager = new MongoLogManager({
172+
directory,
173+
retentionDays: 0.000001, // 86.4 ms
174+
onwarn,
175+
onerror,
176+
});
177+
178+
await parentManager.cleanupOldLogFiles({ recursive: true });
179+
180+
try {
181+
await fs.stat(childWriter.logFilePath as string);
182+
183+
expect.fail('missed exception');
184+
} catch (err: any) {
185+
expect(err.code).to.equal('ENOENT');
186+
}
187+
});
188+
150189
const getFilesState = async (paths: string[]) => {
151190
return (
152191
await Promise.all(
153192
paths.map((path) =>
154193
fs.stat(path).then(
155194
() => 1,
156-
() => 0
157-
)
158-
)
195+
() => 0,
196+
),
197+
),
159198
)
160199
).join('');
161200
};
@@ -174,7 +213,7 @@ describe('MongoLogManager', function () {
174213
for (let i = 0; i < 10; i++) {
175214
const filename = path.join(
176215
directory,
177-
ObjectId.createFromTime(offset - i).toHexString() + '_log'
216+
ObjectId.createFromTime(offset - i).toHexString() + '_log',
178217
);
179218
await fs.writeFile(filename, '');
180219
paths.unshift(filename);
@@ -198,7 +237,7 @@ describe('MongoLogManager', function () {
198237

199238
const faultyFile = path.join(
200239
directory,
201-
ObjectId.createFromTime(offset - 10).toHexString() + '_log'
240+
ObjectId.createFromTime(offset - 10).toHexString() + '_log',
202241
);
203242
await fs.writeFile(faultyFile, '');
204243

@@ -209,7 +248,7 @@ describe('MongoLogManager', function () {
209248
for (let i = 5; i >= 0; i--) {
210249
const filename = path.join(
211250
directory,
212-
ObjectId.createFromTime(offset - i).toHexString() + '_log'
251+
ObjectId.createFromTime(offset - i).toHexString() + '_log',
213252
);
214253
await fs.writeFile(filename, '');
215254
validFiles.push(filename);
@@ -254,7 +293,7 @@ describe('MongoLogManager', function () {
254293
for (let i = 1; i >= 0; i--) {
255294
const withoutPrefix = path.join(
256295
directory,
257-
ObjectId.createFromTime(offset - i).toHexString() + '_log'
296+
ObjectId.createFromTime(offset - i).toHexString() + '_log',
258297
);
259298
await fs.writeFile(withoutPrefix, '');
260299
paths.push(withoutPrefix);
@@ -263,7 +302,7 @@ describe('MongoLogManager', function () {
263302
directory,
264303
'different_' +
265304
ObjectId.createFromTime(offset - i).toHexString() +
266-
'_log'
305+
'_log',
267306
);
268307
await fs.writeFile(withDifferentPrefix, '');
269308
paths.push(withDifferentPrefix);
@@ -273,7 +312,7 @@ describe('MongoLogManager', function () {
273312
for (let i = 9; i >= 0; i--) {
274313
const filename = path.join(
275314
directory,
276-
`custom_${ObjectId.createFromTime(offset - i).toHexString()}_log`
315+
`custom_${ObjectId.createFromTime(offset - i).toHexString()}_log`,
277316
);
278317
await fs.writeFile(filename, '');
279318
paths.push(filename);
@@ -305,7 +344,7 @@ describe('MongoLogManager', function () {
305344
for (let i = 0; i < 10; i++) {
306345
const filename = path.join(
307346
directory,
308-
ObjectId.createFromTime(offset - i).toHexString() + '_log'
347+
ObjectId.createFromTime(offset - i).toHexString() + '_log',
309348
);
310349
await fs.writeFile(filename, '0'.repeat(1024));
311350
paths.unshift(filename);
@@ -332,7 +371,7 @@ describe('MongoLogManager', function () {
332371
for (let i = 0; i < 10; i++) {
333372
const filename = path.join(
334373
directory,
335-
ObjectId.createFromTime(offset - i).toHexString() + '_log'
374+
ObjectId.createFromTime(offset - i).toHexString() + '_log',
336375
);
337376
await fs.writeFile(filename, '');
338377
}
@@ -345,11 +384,12 @@ describe('MongoLogManager', function () {
345384
describe('with a random file order', function () {
346385
let paths: string[] = [];
347386
const times = [92, 90, 1, 2, 3, 91];
387+
let offset: number;
348388

349389
beforeEach(async function () {
350390
const fileNames: string[] = [];
351391
paths = [];
352-
const offset = Math.floor(Date.now() / 1000);
392+
offset = Math.floor(Date.now() / 1000);
353393

354394
for (const time of times) {
355395
const fileName =
@@ -359,19 +399,6 @@ describe('MongoLogManager', function () {
359399
fileNames.push(fileName);
360400
paths.push(fullPath);
361401
}
362-
363-
sinon.replace(fs, 'opendir', async () =>
364-
Promise.resolve({
365-
[Symbol.asyncIterator]: function* () {
366-
for (const fileName of fileNames) {
367-
yield {
368-
name: fileName,
369-
isFile: () => true,
370-
};
371-
}
372-
},
373-
} as unknown as Dir)
374-
);
375402
});
376403

377404
it('cleans up in the expected order with maxLogFileCount', async function () {
@@ -405,6 +432,44 @@ describe('MongoLogManager', function () {
405432

406433
expect(await getFilesState(paths)).to.equal('001110');
407434
});
435+
436+
describe('with subdirectories', function () {
437+
it('cleans up in the expected order with maxLogFileCount', async function () {
438+
// This should exist since the file was created recently
439+
const childPath1 = path.join(
440+
directory,
441+
'subdir1',
442+
ObjectId.createFromTime(offset - 2).toHexString() + '_log',
443+
);
444+
await fs.mkdir(path.join(directory, 'subdir1'), { recursive: true });
445+
await fs.writeFile(childPath1, '0'.repeat(1024));
446+
paths.push(childPath1);
447+
448+
// This should not exist since it was created a long time ago
449+
const childPath2 = path.join(
450+
directory,
451+
'subdir2',
452+
ObjectId.createFromTime(offset - 20).toHexString() + '_log',
453+
);
454+
await fs.mkdir(path.join(directory, 'subdir2'), { recursive: true });
455+
await fs.writeFile(childPath2, '0'.repeat(1024));
456+
paths.push(childPath2);
457+
458+
const manager = new MongoLogManager({
459+
directory,
460+
retentionDays,
461+
maxLogFileCount: 3,
462+
onwarn,
463+
onerror,
464+
});
465+
466+
expect(await getFilesState(paths)).to.equal('11111111');
467+
468+
await manager.cleanupOldLogFiles({ recursive: true });
469+
470+
expect(await getFilesState(paths)).to.equal('00110010');
471+
});
472+
});
408473
});
409474

410475
describe('with multiple log retention settings', function () {
@@ -426,13 +491,13 @@ describe('MongoLogManager', function () {
426491
const yesterday = today - 25 * 60 * 60;
427492
const todayFile = path.join(
428493
directory,
429-
ObjectId.createFromTime(today - i).toHexString() + '_log'
494+
ObjectId.createFromTime(today - i).toHexString() + '_log',
430495
);
431496
await fs.writeFile(todayFile, '0'.repeat(1024));
432497

433498
const yesterdayFile = path.join(
434499
directory,
435-
ObjectId.createFromTime(yesterday - i).toHexString() + '_log'
500+
ObjectId.createFromTime(yesterday - i).toHexString() + '_log',
436501
);
437502
await fs.writeFile(yesterdayFile, '0'.repeat(1024));
438503

@@ -467,7 +532,7 @@ describe('MongoLogManager', function () {
467532
for (let i = 0; i < 10; i++) {
468533
const filename = path.join(
469534
directory,
470-
ObjectId.createFromTime(offset - i).toHexString() + '_log'
535+
ObjectId.createFromTime(offset - i).toHexString() + '_log',
471536
);
472537
await fs.writeFile(filename, '0'.repeat(1024));
473538
paths.unshift(filename);
@@ -496,7 +561,7 @@ describe('MongoLogManager', function () {
496561
for (let i = 0; i < 10; i++) {
497562
const filename = path.join(
498563
directory,
499-
ObjectId.createFromTime(offset - i).toHexString() + '_log'
564+
ObjectId.createFromTime(offset - i).toHexString() + '_log',
500565
);
501566
await fs.writeFile(filename, '0'.repeat(1024));
502567
paths.unshift(filename);
@@ -528,7 +593,7 @@ describe('MongoLogManager', function () {
528593
});
529594

530595
const writer = await manager.createLogWriter();
531-
expect(onwarn).to.have.been.calledOnce; // eslint-disable-line
596+
expect(onwarn).to.have.been.calledOnce;
532597
expect(writer.logFilePath).to.equal(null);
533598

534599
writer.info('component', mongoLogId(12345), 'context', 'message', {
@@ -538,7 +603,7 @@ describe('MongoLogManager', function () {
538603
await once(writer, 'finish');
539604
});
540605

541-
it('optionally allow gzip’ed log files', async function () {
606+
it("optionally allow gzip'ed log files", async function () {
542607
const manager = new MongoLogManager({
543608
directory,
544609
retentionDays,
@@ -566,7 +631,7 @@ describe('MongoLogManager', function () {
566631
expect(log[0].t.$date).to.be.a('string');
567632
});
568633

569-
it('optionally can read truncated gzip’ed log files', async function () {
634+
it("optionally can read truncated gzip'ed log files", async function () {
570635
const manager = new MongoLogManager({
571636
directory,
572637
retentionDays,
@@ -608,7 +673,7 @@ describe('MongoLogManager', function () {
608673
};
609674
const opendirStub = sinon
610675
.stub(fs, 'opendir')
611-
.resolves(fakeDirHandle as any);
676+
.resolves(fakeDirHandle as unknown as Dir);
612677

613678
retentionDays = 0.000001; // 86.4 ms
614679
const manager = new MongoLogManager({

0 commit comments

Comments
 (0)