Skip to content

Commit bd2b763

Browse files
andrewshellandrewshell
authored
Merge pull request #20 from rsscloud/feature/cleanup-logs-and-subscriptions
Implement log cleanup with 2-hour retention and fix subscription cleanup
2 parents 43aff40 + 5c3f2db commit bd2b763

File tree

5 files changed

+158
-35
lines changed

5 files changed

+158
-35
lines changed

‎app.js‎

Lines changed: 25 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -6,16 +6,28 @@ const config = require('./config'),
66
exphbs = require('express-handlebars'),
77
getDayjs = require('./services/dayjs-wrapper'),
88
mongodb = require('./services/mongodb'),
9-
morgan = require('morgan');
10-
// removeExpiredSubscriptions = require('./services/remove-expired-subscriptions');
9+
morgan = require('morgan'),
10+
{ setupLogRetention } = require('./services/log-cleanup'),
11+
removeExpiredSubscriptions = require('./services/remove-expired-subscriptions');
1112

1213
let app, hbs, server, dayjs;
1314

1415
require('console-stamp')(console, 'HH:MM:ss.l');
1516

1617
console.log(`${config.appName} ${config.appVersion}`);
1718

18-
// TODO: Every 24 hours run removeExpiredSubscriptions(data);
19+
// Schedule cleanup tasks
20+
function scheduleCleanupTasks() {
21+
// Run subscription cleanup every 24 hours
22+
setInterval(async() => {
23+
try {
24+
console.log('Running scheduled subscription cleanup...');
25+
await removeExpiredSubscriptions();
26+
} catch (error) {
27+
console.error('Error in scheduled subscription cleanup:', error);
28+
}
29+
}, 24 * 60 * 60 * 1000); // 24 hours in milliseconds
30+
}
1931

2032
morgan.format('mydate', () => {
2133
return new Date().toLocaleTimeString('en-US', { hour12: false, fractionalSecondDigits: 3 }).replace(/:/g, ':');
@@ -60,6 +72,16 @@ async function startServer() {
6072
await initializeDayjs();
6173
await mongodb.connect('rsscloud', config.mongodbUri);
6274

75+
// Setup log retention TTL index
76+
try {
77+
await setupLogRetention();
78+
} catch (error) {
79+
console.error('Failed to setup log retention, continuing without it:', error);
80+
}
81+
82+
// Start cleanup scheduling
83+
scheduleCleanupTasks();
84+
6385
server = app.listen(config.port, () => {
6486
app.locals.host = config.domain;
6587
app.locals.port = server.address().port;

‎config.js‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,8 @@ nconf
2121
'MAX_RESOURCE_SIZE': 256000,
2222
'CT_SECS_RESOURCE_EXPIRE': 90000,
2323
'MIN_SECS_BETWEEN_PINGS': 0,
24-
'REQUEST_TIMEOUT': 4000
24+
'REQUEST_TIMEOUT': 4000,
25+
'LOG_RETENTION_HOURS': 2
2526
});
2627

2728
module.exports = {
@@ -34,5 +35,6 @@ module.exports = {
3435
maxResourceSize: nconf.get('MAX_RESOURCE_SIZE'),
3536
ctSecsResourceExpire: nconf.get('CT_SECS_RESOURCE_EXPIRE'),
3637
minSecsBetweenPings: nconf.get('MIN_SECS_BETWEEN_PINGS'),
37-
requestTimeout: nconf.get('REQUEST_TIMEOUT')
38+
requestTimeout: nconf.get('REQUEST_TIMEOUT'),
39+
logRetentionHours: nconf.get('LOG_RETENTION_HOURS')
3840
};

‎eslint.config.js‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,8 @@ module.exports = [
2121
AbortController: 'readonly',
2222
setTimeout: 'readonly',
2323
clearTimeout: 'readonly',
24+
setInterval: 'readonly',
25+
clearInterval: 'readonly',
2426
URLSearchParams: 'readonly',
2527
// Mocha globals
2628
describe: 'readonly',

‎services/log-cleanup.js‎

Lines changed: 59 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,59 @@
1+
const getDatabase = require('./mongodb');
2+
const config = require('../config');
3+
4+
/**
5+
* Sets up TTL (Time To Live) index on the events collection
6+
* This will automatically expire log entries after the configured retention period
7+
*/
8+
async function setupLogRetention() {
9+
try {
10+
const db = await getDatabase();
11+
const collection = db.collection('events');
12+
13+
const retentionSeconds = config.logRetentionHours * 3600;
14+
15+
// Create TTL index on the when field (log timestamp)
16+
// MongoDB will automatically delete documents when 'when' is older than retentionSeconds
17+
await collection.createIndex(
18+
{ when: 1 },
19+
{
20+
expireAfterSeconds: retentionSeconds,
21+
name: 'log_retention_ttl'
22+
}
23+
);
24+
25+
console.log(`Log retention TTL index created: ${config.logRetentionHours} hours (${retentionSeconds} seconds)`);
26+
} catch (error) {
27+
console.error('Error setting up log retention TTL index:', error);
28+
throw error;
29+
}
30+
}
31+
32+
/**
33+
* Manually removes expired log entries (alternative to TTL)
34+
* This is a fallback method if TTL index setup fails
35+
*/
36+
async function removeExpiredLogs() {
37+
try {
38+
const db = await getDatabase();
39+
const collection = db.collection('events');
40+
41+
const cutoffDate = new Date();
42+
cutoffDate.setHours(cutoffDate.getHours() - config.logRetentionHours);
43+
44+
const result = await collection.deleteMany({
45+
when: { $lt: cutoffDate }
46+
});
47+
48+
console.log(`Removed ${result.deletedCount} expired log entries older than ${cutoffDate.toISOString()}`);
49+
return result.deletedCount;
50+
} catch (error) {
51+
console.error('Error removing expired logs:', error);
52+
throw error;
53+
}
54+
}
55+
56+
module.exports = {
57+
setupLogRetention,
58+
removeExpiredLogs
59+
};

‎services/remove-expired-subscriptions.js‎

Lines changed: 68 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -1,39 +1,77 @@
1-
// TODO: Rewrite for mongodb
2-
1+
const getDatabase = require('./mongodb');
32
const getDayjs = require('./dayjs-wrapper');
3+
const config = require('../config');
44

5-
async function checkSubscription(data, resourceUrl, apiurl) {
6-
const dayjs = await getDayjs();
7-
const subscription = data.subscriptions[resourceUrl][apiurl];
8-
if (dayjs(subscription.whenExpires).isBefore(dayjs())) {
9-
delete data.subscriptions[resourceUrl][apiurl];
10-
} else if (subscription.ctConsecutiveErrors > data.prefs.maxConsecutiveErrors) {
11-
delete data.subscriptions[resourceUrl][apiurl];
12-
}
13-
}
5+
/**
6+
* Removes expired and errored subscriptions from MongoDB
7+
* Works with the MongoDB schema: { _id: resourceUrl, pleaseNotify: [...] }
8+
*/
9+
async function removeExpiredSubscriptions() {
10+
try {
11+
const db = await getDatabase();
12+
const dayjs = await getDayjs();
13+
const collection = db.collection('subscriptions');
1414

15-
async function scanApiUrls(data, resourceUrl) {
16-
const subscriptions = data.subscriptions[resourceUrl];
17-
for (const apiurl in subscriptions) {
18-
if (Object.prototype.hasOwnProperty.call(subscriptions, apiurl)) {
19-
await checkSubscription(data, resourceUrl, apiurl);
20-
}
21-
}
22-
if (0 === subscriptions.length) {
23-
delete data.subscriptions[resourceUrl];
24-
}
25-
}
15+
let totalRemoved = 0;
16+
let documentsProcessed = 0;
17+
let documentsDeleted = 0;
18+
19+
// Find all subscription documents
20+
const cursor = collection.find({});
21+
22+
while (await cursor.hasNext()) {
23+
const doc = await cursor.next();
24+
documentsProcessed++;
2625

27-
async function scanResources(data) {
28-
for (const resourceUrl in data.subscriptions) {
29-
if (Object.prototype.hasOwnProperty.call(data.subscriptions, resourceUrl)) {
30-
await scanApiUrls(data, resourceUrl);
26+
if (!doc.pleaseNotify || !Array.isArray(doc.pleaseNotify)) {
27+
continue;
28+
}
29+
30+
// Filter out expired and errored subscriptions
31+
const validSubscriptions = doc.pleaseNotify.filter(subscription => {
32+
// Remove if expired
33+
if (dayjs(subscription.whenExpires).isBefore(dayjs())) {
34+
totalRemoved++;
35+
return false;
36+
}
37+
38+
// Remove if too many consecutive errors
39+
if (subscription.ctConsecutiveErrors > config.maxConsecutiveErrors) {
40+
totalRemoved++;
41+
return false;
42+
}
43+
44+
return true;
45+
});
46+
47+
// Update document if subscriptions were removed
48+
if (validSubscriptions.length !== doc.pleaseNotify.length) {
49+
if (validSubscriptions.length === 0) {
50+
// Remove entire document if no valid subscriptions remain
51+
await collection.deleteOne({ _id: doc._id });
52+
documentsDeleted++;
53+
} else {
54+
// Update document with filtered subscriptions
55+
await collection.updateOne(
56+
{ _id: doc._id },
57+
{ $set: { pleaseNotify: validSubscriptions } }
58+
);
59+
}
60+
}
3161
}
32-
}
33-
}
3462

35-
async function removeExpiredSubscriptions(data) {
36-
await scanResources(data);
63+
console.log(`Subscription cleanup completed: ${totalRemoved} expired/errored subscriptions removed, ${documentsProcessed} documents processed, ${documentsDeleted} empty documents deleted`);
64+
65+
return {
66+
subscriptionsRemoved: totalRemoved,
67+
documentsProcessed,
68+
documentsDeleted
69+
};
70+
71+
} catch (error) {
72+
console.error('Error removing expired subscriptions:', error);
73+
throw error;
74+
}
3775
}
3876

3977
module.exports = removeExpiredSubscriptions;

0 commit comments

Comments
 (0)