Skip to content

feat(mongodb-log-writer): add support for recursively cleaning up old log files MCP-103 #565

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wants to merge 2 commits into from
Closed

feat(mongodb-log-writer): add support for recursively cleaning up old log files MCP-103 #565

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
87c7fa9
feat: add support for recursively cleaning up old log files
nirinchev Aug 5, 2025
8d3e8b1
Clean up empty child directories
nirinchev Aug 5, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
176 changes: 139 additions & 37 deletions packages/mongodb-log-writer/src/mongo-log-manager.spec.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,13 +22,13 @@ describe('MongoLogManager', function () {
onerror = sinon.stub();
directory = path.join(
os.tmpdir(),
`log-writer-test-${Math.random()}-${Date.now()}`
`log-writer-test-${Math.random()}-${Date.now()}`,
);
await fs.mkdir(directory, { recursive: true });
});

afterEach(async function () {
await fs.rmdir(directory, { recursive: true });
await fs.rm(directory, { recursive: true });
sinon.restore();
});

Expand Down Expand Up @@ -84,10 +84,10 @@ describe('MongoLogManager', function () {

const writer = await manager.createLogWriter();
expect(
path.relative(directory, writer.logFilePath as string)[0]
path.relative(directory, writer.logFilePath as string)[0],
).to.not.equal('.');
expect((writer.logFilePath as string).includes(writer.logId)).to.equal(
true
true,
);

writer.info('component', mongoLogId(12345), 'context', 'message', {
Expand Down Expand Up @@ -147,15 +147,54 @@ describe('MongoLogManager', function () {
}
});

it('can recursively clean up old files', async function () {
const childDirectory = path.join(directory, 'child', '1');
await fs.mkdir(childDirectory, { recursive: true });
// The child manager writes in a subdirectory of the log directory
// The expectation is that when the parent manager recursively cleans up the
// old log files, it will be able to delete the files of the child manager.
const childManager = new MongoLogManager({
directory: childDirectory,
retentionDays: 60,
onwarn,
onerror,
});

const childWriter = await childManager.createLogWriter();
childWriter.info('child', mongoLogId(12345), 'context', 'message');

childWriter.end();
await once(childWriter, 'finish');
await fs.stat(childWriter.logFilePath as string);
await new Promise((resolve) => setTimeout(resolve, 100));

const parentManager = new MongoLogManager({
directory,
retentionDays: 0.000001, // 86.4 ms
onwarn,
onerror,
});

await parentManager.cleanupOldLogFiles({ recursive: true });

try {
await fs.stat(childWriter.logFilePath as string);

expect.fail('missed exception');
} catch (err: any) {
expect(err.code).to.equal('ENOENT');
}
});

const getFilesState = async (paths: string[]) => {
return (
await Promise.all(
paths.map((path) =>
fs.stat(path).then(
() => 1,
() => 0
)
)
() => 0,
),
),
)
).join('');
};
Expand All @@ -174,7 +213,7 @@ describe('MongoLogManager', function () {
for (let i = 0; i < 10; i++) {
const filename = path.join(
directory,
ObjectId.createFromTime(offset - i).toHexString() + '_log'
ObjectId.createFromTime(offset - i).toHexString() + '_log',
);
await fs.writeFile(filename, '');
paths.unshift(filename);
Expand All @@ -198,7 +237,7 @@ describe('MongoLogManager', function () {

const faultyFile = path.join(
directory,
ObjectId.createFromTime(offset - 10).toHexString() + '_log'
ObjectId.createFromTime(offset - 10).toHexString() + '_log',
);
await fs.writeFile(faultyFile, '');

Expand All @@ -209,7 +248,7 @@ describe('MongoLogManager', function () {
for (let i = 5; i >= 0; i--) {
const filename = path.join(
directory,
ObjectId.createFromTime(offset - i).toHexString() + '_log'
ObjectId.createFromTime(offset - i).toHexString() + '_log',
);
await fs.writeFile(filename, '');
validFiles.push(filename);
Expand Down Expand Up @@ -254,7 +293,7 @@ describe('MongoLogManager', function () {
for (let i = 1; i >= 0; i--) {
const withoutPrefix = path.join(
directory,
ObjectId.createFromTime(offset - i).toHexString() + '_log'
ObjectId.createFromTime(offset - i).toHexString() + '_log',
);
await fs.writeFile(withoutPrefix, '');
paths.push(withoutPrefix);
Expand All @@ -263,7 +302,7 @@ describe('MongoLogManager', function () {
directory,
'different_' +
ObjectId.createFromTime(offset - i).toHexString() +
'_log'
'_log',
);
await fs.writeFile(withDifferentPrefix, '');
paths.push(withDifferentPrefix);
Expand All @@ -273,7 +312,7 @@ describe('MongoLogManager', function () {
for (let i = 9; i >= 0; i--) {
const filename = path.join(
directory,
`custom_${ObjectId.createFromTime(offset - i).toHexString()}_log`
`custom_${ObjectId.createFromTime(offset - i).toHexString()}_log`,
);
await fs.writeFile(filename, '');
paths.push(filename);
Expand Down Expand Up @@ -305,7 +344,7 @@ describe('MongoLogManager', function () {
for (let i = 0; i < 10; i++) {
const filename = path.join(
directory,
ObjectId.createFromTime(offset - i).toHexString() + '_log'
ObjectId.createFromTime(offset - i).toHexString() + '_log',
);
await fs.writeFile(filename, '0'.repeat(1024));
paths.unshift(filename);
Expand All @@ -332,7 +371,7 @@ describe('MongoLogManager', function () {
for (let i = 0; i < 10; i++) {
const filename = path.join(
directory,
ObjectId.createFromTime(offset - i).toHexString() + '_log'
ObjectId.createFromTime(offset - i).toHexString() + '_log',
);
await fs.writeFile(filename, '');
}
Expand All @@ -345,11 +384,12 @@ describe('MongoLogManager', function () {
describe('with a random file order', function () {
let paths: string[] = [];
const times = [92, 90, 1, 2, 3, 91];
let offset: number;

beforeEach(async function () {
const fileNames: string[] = [];
paths = [];
const offset = Math.floor(Date.now() / 1000);
offset = Math.floor(Date.now() / 1000);

for (const time of times) {
const fileName =
Expand All @@ -359,19 +399,6 @@ describe('MongoLogManager', function () {
fileNames.push(fileName);
paths.push(fullPath);
}

sinon.replace(fs, 'opendir', async () =>
Promise.resolve({
[Symbol.asyncIterator]: function* () {
for (const fileName of fileNames) {
yield {
name: fileName,
isFile: () => true,
};
}
},
} as unknown as Dir)
);
});

it('cleans up in the expected order with maxLogFileCount', async function () {
Expand Down Expand Up @@ -405,6 +432,81 @@ describe('MongoLogManager', function () {

expect(await getFilesState(paths)).to.equal('001110');
});

describe('with subdirectories', function () {
beforeEach(async function () {
// Add a recent child file
const childPath1 = path.join(
directory,
'subdir1',
ObjectId.createFromTime(offset - 2).toHexString() + '_log',
);
await fs.mkdir(path.join(directory, 'subdir1'), { recursive: true });
await fs.writeFile(childPath1, '0'.repeat(1024));
paths.push(childPath1);

// Add an older child file
const childPath2 = path.join(
directory,
'subdir2',
ObjectId.createFromTime(offset - 20).toHexString() + '_log',
);
await fs.mkdir(path.join(directory, 'subdir2'), { recursive: true });
await fs.writeFile(childPath2, '0'.repeat(1024));
paths.push(childPath2);
});

it('cleans up in the expected order with maxLogFileCount', async function () {
const manager = new MongoLogManager({
directory,
retentionDays,
maxLogFileCount: 3,
onwarn,
onerror,
});

expect(await getFilesState(paths)).to.equal('11111111');

await manager.cleanupOldLogFiles({ recursive: true });

expect(await getFilesState(paths)).to.equal('00110010');
});

it('deletes empty directories after cleanup', async function () {
// Add an old file in subdir1 - it should be deleted, but subdir1 should remain
const oldSubdir1Child = path.join(
directory,
'subdir1',
ObjectId.createFromTime(offset - 50).toHexString() + '_log',
);
await fs.writeFile(oldSubdir1Child, '0'.repeat(1024));
paths.push(oldSubdir1Child);

const manager = new MongoLogManager({
directory,
retentionDays,
maxLogFileCount: 3,
onwarn,
onerror,
});

expect(await getFilesState(paths)).to.equal('111111111');

await manager.cleanupOldLogFiles({ recursive: true });

expect(await getFilesState(paths)).to.equal('001100100');

// subdir1 should have been left alone because of the newer child
// subdir2 should have been deleted because it was empty
await fs.stat(path.join(directory, 'subdir1'));
try {
await fs.stat(path.join(directory, 'subdir2'));
expect.fail('subdir2 should have been deleted');
} catch (err: any) {
expect(err.code).to.equal('ENOENT');
}
});
});
});

describe('with multiple log retention settings', function () {
Expand All @@ -426,13 +528,13 @@ describe('MongoLogManager', function () {
const yesterday = today - 25 * 60 * 60;
const todayFile = path.join(
directory,
ObjectId.createFromTime(today - i).toHexString() + '_log'
ObjectId.createFromTime(today - i).toHexString() + '_log',
);
await fs.writeFile(todayFile, '0'.repeat(1024));

const yesterdayFile = path.join(
directory,
ObjectId.createFromTime(yesterday - i).toHexString() + '_log'
ObjectId.createFromTime(yesterday - i).toHexString() + '_log',
);
await fs.writeFile(yesterdayFile, '0'.repeat(1024));

Expand Down Expand Up @@ -467,7 +569,7 @@ describe('MongoLogManager', function () {
for (let i = 0; i < 10; i++) {
const filename = path.join(
directory,
ObjectId.createFromTime(offset - i).toHexString() + '_log'
ObjectId.createFromTime(offset - i).toHexString() + '_log',
);
await fs.writeFile(filename, '0'.repeat(1024));
paths.unshift(filename);
Expand Down Expand Up @@ -496,7 +598,7 @@ describe('MongoLogManager', function () {
for (let i = 0; i < 10; i++) {
const filename = path.join(
directory,
ObjectId.createFromTime(offset - i).toHexString() + '_log'
ObjectId.createFromTime(offset - i).toHexString() + '_log',
);
await fs.writeFile(filename, '0'.repeat(1024));
paths.unshift(filename);
Expand Down Expand Up @@ -528,7 +630,7 @@ describe('MongoLogManager', function () {
});

const writer = await manager.createLogWriter();
expect(onwarn).to.have.been.calledOnce; // eslint-disable-line
expect(onwarn).to.have.been.calledOnce;
expect(writer.logFilePath).to.equal(null);

writer.info('component', mongoLogId(12345), 'context', 'message', {
Expand All @@ -538,7 +640,7 @@ describe('MongoLogManager', function () {
await once(writer, 'finish');
});

it('optionally allow gzip’ed log files', async function () {
it("optionally allow gzip'ed log files", async function () {
const manager = new MongoLogManager({
directory,
retentionDays,
Expand Down Expand Up @@ -566,7 +668,7 @@ describe('MongoLogManager', function () {
expect(log[0].t.$date).to.be.a('string');
});

it('optionally can read truncated gzip’ed log files', async function () {
it("optionally can read truncated gzip'ed log files", async function () {
const manager = new MongoLogManager({
directory,
retentionDays,
Expand Down Expand Up @@ -608,7 +710,7 @@ describe('MongoLogManager', function () {
};
const opendirStub = sinon
.stub(fs, 'opendir')
.resolves(fakeDirHandle as any);
.resolves(fakeDirHandle as unknown as Dir);

retentionDays = 0.000001; // 86.4 ms
const manager = new MongoLogManager({
Expand Down
Loading
Loading