Skip to content

Commit fe45c1b

Browse files
barisusaklibarisusakli
committed
refactor: counters, add estimates, only show progress bar
when working
1 parent 21bc7bc commit fe45c1b

File tree

7 files changed

+102
-55
lines changed

7 files changed

+102
-55
lines changed

‎lib/dbsearch.js‎

Lines changed: 34 additions & 36 deletions
Original file line numberDiff line numberDiff line change
@@ -247,9 +247,9 @@ search.filterMessagingSearchMessages = async function (data) {
247247

248248
search.reindex = async function () {
249249
await db.setObject('nodebb-plugin-dbsearch', {
250-
topicsIndexed: 0,
251-
postsIndexed: 0,
252-
messagesIndexed: 0,
250+
topicsProgress: 0,
251+
postsProgress: 0,
252+
messagesProgress: 0,
253253
working: 1,
254254
});
255255
await Promise.all([
@@ -264,8 +264,11 @@ search.reindex = async function () {
264264

265265
async function reIndexTopics() {
266266
await batch.processSortedSet('topics:tid', async (tids) => {
267-
const topicData = await topics.getTopicsFields(tids, ['tid', 'title', 'uid', 'cid', 'deleted', 'timestamp']);
267+
const topicData = await topics.getTopicsFields(tids, [
268+
'tid', 'title', 'uid', 'cid', 'deleted', 'timestamp',
269+
]);
268270
await topicsSave(topicData);
271+
await db.incrObjectFieldBy('nodebb-plugin-dbsearch', 'topicsProgress', tids.length);
269272
}, {
270273
batch: batchSize,
271274
});
@@ -306,7 +309,6 @@ async function topicsSave(topics) {
306309

307310
const result = await plugins.hooks.fire('filter:search.indexTopics', { data: data, tids: tids, topics: topics });
308311
await db.searchIndex('topic', result.data, result.tids);
309-
await db.incrObjectFieldBy('nodebb-plugin-dbsearch', 'topicsIndexed', result.tids.length);
310312
}
311313

312314
async function reIndexPosts() {
@@ -323,6 +325,7 @@ async function reIndexPosts() {
323325
});
324326
postData = postData.filter(post => tidToTopic[post.tid].deleted !== 1);
325327
await postsSave(postData);
328+
await db.incrObjectFieldBy('nodebb-plugin-dbsearch', 'postsProgress', pids.length);
326329
}, {
327330
batch: batchSize,
328331
});
@@ -363,21 +366,23 @@ async function postsSave(posts) {
363366

364367
const result = await plugins.hooks.fire('filter:search.indexPosts', { data: data, pids: pids, posts: posts });
365368
await db.searchIndex('post', result.data, result.pids);
366-
await db.incrObjectFieldBy('nodebb-plugin-dbsearch', 'postsIndexed', result.pids.length);
367369
}
368370

369371
async function reIndexMessages() {
370372
await batch.processSortedSet(`messages:mid`, async (mids) => {
371373
let messageData = await messaging.getMessagesFields(mids, ['mid', 'content', 'roomId', 'fromuid', 'deleted', 'system']);
372374
messageData = messageData.filter(p => p && p.deleted !== 1 && p.system !== 1);
373375
await messagesSave(messageData);
376+
await db.incrObjectFieldBy('nodebb-plugin-dbsearch', 'messagesProgress', mids.length);
374377
}, {
375378
batch: batchSize,
376379
});
377380
}
378381

379382
async function messagesSave(msgs) {
380-
msgs = msgs.filter(m => m && utils.isNumber(m.mid) && parseInt(m.deleted, 10) !== 1 && parseInt(m.system, 10) !== 1);
383+
msgs = msgs.filter(
384+
m => m && utils.isNumber(m.mid) && parseInt(m.deleted, 10) !== 1 && parseInt(m.system, 10) !== 1
385+
);
381386

382387
let data = msgs.map((msgData) => {
383388
const indexData = {};
@@ -404,19 +409,11 @@ async function messagesSave(msgs) {
404409

405410
const result = await plugins.hooks.fire('filter:search.indexMessages', { data: data, mids: mids, messages: msgs });
406411
await searchModule.chat.index(result.data, result.mids);
407-
await db.incrObjectFieldBy('nodebb-plugin-dbsearch', 'messagesIndexed', result.mids.length);
408412
}
409413

410414
async function searchRemove(key, ids) {
411415
ids = ids.filter(id => id && utils.isNumber(id));
412416
await db.searchRemove(key, ids);
413-
if (key === 'topic') {
414-
await db.incrObjectFieldBy('nodebb-plugin-dbsearch', 'topicsIndexed', -ids.length);
415-
} else if (key === 'post') {
416-
await db.incrObjectFieldBy('nodebb-plugin-dbsearch', 'postsIndexed', -ids.length);
417-
} else if (key === 'chat') {
418-
await db.incrObjectFieldBy('nodebb-plugin-dbsearch', 'messagesIndexed', -ids.length);
419-
}
420417
}
421418

422419
async function reIndexTids(tids) {
@@ -468,6 +465,14 @@ async function reIndexPids(pids, topic) {
468465

469466
async function renderAdmin(req, res) {
470467
const results = await getGlobalAndPluginData();
468+
const [topicsIndexed, postsIndexed, messagesIndexed] = await Promise.all([
469+
searchModule.getIndexedTopicCount(),
470+
searchModule.getIndexedPostCount(),
471+
searchModule.getIndexedChatMessageCount(),
472+
]);
473+
results.plugin.topicsIndexed = parseInt(topicsIndexed, 10) || 0;
474+
results.plugin.postsIndexed = parseInt(postsIndexed, 10) || 0;
475+
results.plugin.messagesIndexed = parseInt(messagesIndexed, 10) || 0;
471476
results.plugin.progressData = await getProgress();
472477
results.plugin.title = 'DB Search';
473478
res.render('admin/plugins/dbsearch', results.plugin);
@@ -498,14 +503,12 @@ socketAdmin.plugins.dbsearch.checkProgress = async function () {
498503

499504
async function getPluginData() {
500505
const data = await db.getObject('nodebb-plugin-dbsearch') || {};
501-
data.topicsIndexed = parseInt(data.topicsIndexed, 10) || 0;
502-
data.postsIndexed = parseInt(data.postsIndexed, 10) || 0;
503-
data.messagesIndexed = parseInt(data.messagesIndexed, 10) || 0;
506+
504507
data.excludeCategories = data.excludeCategories || '[]';
505508
data.postLimit = data.postLimit || defaultPostLimit;
506509
data.topicLimit = data.topicLimit || defaultTopicLimit;
507510
data.indexLanguage = data.indexLanguage || 'en';
508-
data.working = data.working || 0;
511+
data.working = parseInt(data.working, 10) || 0;
509512

510513
try {
511514
data.excludeCategories = JSON.parse(data.excludeCategories);
@@ -537,9 +540,6 @@ async function getGlobalAndPluginData() {
537540
plugin.messageCount = parseInt(global.messageCount, 10);
538541
plugin.topicLimit = plugin.topicLimit || defaultTopicLimit;
539542
plugin.postLimit = plugin.postLimit || defaultPostLimit;
540-
plugin.topicsIndexed = plugin.topicsIndexed > plugin.topicCount ? plugin.topicCount : plugin.topicsIndexed;
541-
plugin.postsIndexed = plugin.postsIndexed > plugin.postCount ? plugin.postCount : plugin.postsIndexed;
542-
plugin.messagesIndexed = plugin.messagesIndexed > plugin.messageCount ? plugin.messageCount : plugin.messagesIndexed;
543543
plugin.languageSupported = languageSupported;
544544
plugin.languages = languages;
545545
plugin.indexLanguage = plugin.indexLanguage || 'en';
@@ -557,21 +557,18 @@ async function getGlobalAndPluginData() {
557557
async function getProgress() {
558558
const [global, pluginData] = await Promise.all([
559559
db.getObjectFields('global', ['topicCount', 'postCount', 'messageCount']),
560-
getPluginData(),
560+
db.getObjectFields('nodebb-plugin-dbsearch', ['topicsProgress', 'postsProgress', 'messagesProgress', 'working']),
561561
]);
562562
const topicCount = parseInt(global.topicCount, 10);
563563
const postCount = parseInt(global.postCount, 10);
564564
const messageCount = parseInt(global.messageCount, 10);
565-
const topicsPercent = topicCount ? (pluginData.topicsIndexed / topicCount) * 100 : 0;
566-
const postsPercent = postCount ? (pluginData.postsIndexed / postCount) * 100 : 0;
567-
const messagesPercent = messageCount ? (pluginData.messagesIndexed / messageCount) * 100 : 0;
565+
const topicsPercent = topicCount ? (pluginData.topicsProgress / topicCount) * 100 : 0;
566+
const postsPercent = postCount ? (pluginData.postsProgress / postCount) * 100 : 0;
567+
const messagesPercent = messageCount ? (pluginData.messagesProgress / messageCount) * 100 : 0;
568568
return {
569569
topicsPercent: Math.max(0, Math.min(100, topicsPercent.toFixed(2))),
570570
postsPercent: Math.max(0, Math.min(100, postsPercent.toFixed(2))),
571571
messagesPercent: Math.max(0, Math.min(100, messagesPercent.toFixed(2))),
572-
topicsIndexed: topicsPercent >= 100 ? topicCount : Math.max(0, pluginData.topicsIndexed),
573-
postsIndexed: postsPercent >= 100 ? postCount : Math.max(0, pluginData.postsIndexed),
574-
messagesIndexed: messagesPercent >= 100 ? messageCount : Math.max(0, pluginData.messagesIndexed),
575572
working: pluginData.working,
576573
};
577574
}
@@ -599,26 +596,27 @@ socketAdmin.plugins.dbsearch.clearIndex = async function () {
599596

600597
async function clearIndex() {
601598
await db.setObject('nodebb-plugin-dbsearch', {
599+
postsProgress: 0,
600+
topicsProgress: 0,
601+
messagesProgress: 0,
602602
working: 1,
603603
});
604604

605605
await Promise.all([
606-
clearSet('topics:tid', 'topic'),
607-
clearSet('posts:pid', 'post'),
608-
clearSet('messages:mid', 'chat'),
606+
clearSet('topics:tid', 'topic', 'topicsProgress'),
607+
clearSet('posts:pid', 'post', 'postsProgress'),
608+
clearSet('messages:mid', 'chat', 'messagesProgress'),
609609
]);
610610

611611
await db.setObject('nodebb-plugin-dbsearch', {
612-
postsIndexed: 0,
613-
topicsIndexed: 0,
614-
messagesIndexed: 0,
615612
working: 0,
616613
});
617614
}
618615

619-
async function clearSet(set, key) {
616+
async function clearSet(set, key, progressKey) {
620617
await batch.processSortedSet(set, async (ids) => {
621618
await searchRemove(key, ids);
619+
await db.incrObjectFieldBy('nodebb-plugin-dbsearch', progressKey, ids.length);
622620
}, {
623621
batch: batchSize,
624622
});

‎lib/mongo.js‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -226,3 +226,14 @@ function buildTextQuery(content, matchWords) {
226226
return { $search: words.join(' ') };
227227
}
228228

229+
exports.getIndexedTopicCount = async () => {
230+
return await db.client.collection('searchtopic').estimatedDocumentCount();
231+
};
232+
233+
exports.getIndexedPostCount = async () => {
234+
return await db.client.collection('searchpost').estimatedDocumentCount();
235+
};
236+
237+
exports.getIndexedChatMessageCount = async () => {
238+
return await db.client.collection('searchchat').estimatedDocumentCount();
239+
};

‎lib/postgres.js‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -164,3 +164,19 @@ exports.chat.search = async (data, limit) => {
164164
return [];
165165
}
166166
};
167+
168+
169+
exports.getIndexedTopicCount = async () => {
170+
const res = await db.pool.query(`SELECT reltuples::bigint AS estimate FROM pg_class WHERE relname = 'searchtopic'`);
171+
return res.rows && res.rows[0] ? parseInt(res.rows[0].estimate, 10) : 0;
172+
};
173+
174+
exports.getIndexedPostCount = async () => {
175+
const res = await db.pool.query(`SELECT reltuples::bigint AS estimate FROM pg_class WHERE relname = 'searchpost'`);
176+
return res.rows && res.rows[0] ? parseInt(res.rows[0].estimate, 10) : 0;
177+
};
178+
179+
exports.getIndexedChatMessageCount = async () => {
180+
const res = await db.pool.query(`SELECT reltuples::bigint AS estimate FROM pg_class WHERE relname = 'searchchat'`);
181+
return res.rows && res.rows[0] ? parseInt(res.rows[0].estimate, 10) : 0;
182+
};

‎lib/redis.js‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -86,3 +86,15 @@ exports.chat.search = async (data, limit) => {
8686

8787
return await db.chatSearch.query(query, 0, limit - 1);
8888
};
89+
90+
exports.getIndexedTopicCount = async () => {
91+
return await db.topicSearch.count('nodebbtopicsearch');
92+
};
93+
94+
exports.getIndexedPostCount = async () => {
95+
return await db.postSearch.count('nodebbpostsearch');
96+
};
97+
98+
exports.getIndexedChatMessageCount = async () => {
99+
return await db.chatSearch.count('nodebbchatsearch');
100+
};

‎package.json‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,7 @@
2020
"license": "BSD-2-Clause",
2121
"dependencies": {
2222
"lodash": "4.17.21",
23-
"redisearch": "^2.0.0"
23+
"redisearch": "^2.0.1"
2424
},
2525
"devDependencies": {
2626
"eslint": "^9.25.1",

‎public/admin.js‎

Lines changed: 22 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -78,13 +78,19 @@ define('admin/plugins/dbsearch', [
7878
};
7979

8080
function startProgress() {
81+
$('#topics-indexed').text(0);
82+
$('#posts-indexed').text(0);
83+
$('#messages-indexed').text(0);
84+
$('.progress-bar').css('width', '0%').text('0%');
85+
$('.progress').removeClass('invisible');
8186
clearProgress();
8287
intervalId = setInterval(checkProgress, 1000);
8388
}
8489

8590
function clearProgress() {
8691
if (intervalId) {
8792
clearInterval(intervalId);
93+
$('.progress').addClass('invisible');
8894
intervalId = 0;
8995
}
9096
}
@@ -96,27 +102,31 @@ define('admin/plugins/dbsearch', [
96102
return alerts.error(err);
97103
}
98104

105+
$('.topic-progress').css('width', progress.topicsPercent + '%').text(progress.topicsPercent + '%');
106+
$('.post-progress').css('width', progress.postsPercent + '%').text(progress.postsPercent + '%');
107+
$('.message-progress').css('width', progress.messagesPercent + '%').text(progress.messagesPercent + '%');
108+
99109
var working = parseInt(progress.working, 10);
100110
if (!working) {
101111
clearInterval(intervalId);
102112
$('#reindex').removeAttr('disabled');
113+
$('#clear-index').removeAttr('disabled');
114+
setTimeout(() => {
115+
$('.progress').addClass('invisible');
116+
}, 2000);
117+
118+
$.getJSON(config.relative_path + '/api/admin/plugins/dbsearch', function (data) {
119+
$('#topics-indexed').text(data.topicsIndexed);
120+
$('#posts-indexed').text(data.postsIndexed);
121+
$('#messages-indexed').text(data.messagesIndexed);
122+
});
103123
} else {
104124
$('#reindex').attr('disabled', true);
125+
$('#clear-index').attr('disabled', true);
126+
$('.progress').removeClass('invisible');
105127
}
106128

107129
$('#work-in-progress').toggleClass('hidden', !working);
108-
109-
if (progress.topicsPercent >= 100 && progress.postsPercent >= 100) {
110-
progress.topicsPercent = 100;
111-
progress.postsPercent = 100;
112-
}
113-
114-
$('#topics-indexed').text(progress.topicsIndexed);
115-
$('#posts-indexed').text(progress.postsIndexed);
116-
$('#messages-indexed').text(progress.messagesIndexed);
117-
$('.topic-progress').css('width', progress.topicsPercent + '%').text(progress.topicsPercent + '%');
118-
$('.post-progress').css('width', progress.postsPercent + '%').text(progress.postsPercent + '%');
119-
$('.message-progress').css('width', progress.messagesPercent + '%').text(progress.messagesPercent + '%');
120130
});
121131
}
122132

‎templates/admin/plugins/dbsearch.tpl‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -9,27 +9,27 @@
99
<div class="col-6">
1010
<div class="mb-3">
1111
<div class="alert alert-info">
12-
Topics Indexed: <strong id="topics-indexed">{topicsIndexed}</strong> / <strong>{topicCount}</strong>
12+
Topics Indexed: <strong id="topics-indexed">{topicsIndexed}</strong> <i class="fa fa-circle-question" title="Deleted topics are not indexed" data-bs-toggle="tooltip"></i>
1313
</div>
1414

15-
<div class="progress" style="height:24px;">
15+
<div class="progress {{{ if !working }}}invisible{{{ end }}}" style="height:24px;">
1616
<div class="topic-progress progress-bar" role="progressbar" aria-valuenow="0" aria-valuemin="0" aria-valuemax="100" style="width:{progressData.topicsPercent}%;min-width: 2em;">{progressData.topicsPercent}%</div>
1717
</div>
1818
</div>
1919
<div class="mb-3">
2020
<div class="alert alert-info">
21-
Posts Indexed: <strong id="posts-indexed">{postsIndexed}</strong> / <strong>{postCount}</strong>
21+
Posts Indexed: <strong id="posts-indexed">{postsIndexed}</strong> <i class="fa fa-circle-question" title="Deleted posts are not indexed" data-bs-toggle="tooltip"></i>
2222
</div>
23-
<div class="progress" style="height:24px;">
23+
<div class="progress {{{ if !working }}}invisible{{{ end }}}" style="height:24px;">
2424
<div class="post-progress progress-bar" role="progressbar" aria-valuenow="0" aria-valuemin="0" aria-valuemax="100" style="width:{progressData.postsPercent}%;min-width: 2em;">{progressData.postsPercent}%</div>
2525
</div>
2626
</div>
2727

2828
<div class="mb-3">
2929
<div class="alert alert-info">
30-
Messages Indexed: <strong id="messages-indexed">{messagesIndexed}</strong> / <strong>{messageCount}</strong>
30+
Messages Indexed: <strong id="messages-indexed">{messagesIndexed}</strong> <i class="fa fa-circle-question" title="Deleted messages are not indexed" data-bs-toggle="tooltip"></i>
3131
</div>
32-
<div class="progress" style="height:24px;">
32+
<div class="progress {{{ if !working }}}invisible{{{ end }}}" style="height:24px;">
3333
<div class="message-progress progress-bar" role="progressbar" aria-valuenow="0" aria-valuemin="0" aria-valuemax="100" style="width:{progressData.messagesPercent}%;min-width: 2em;">{progressData.messagesPercent}%</div>
3434
</div>
3535
</div>

0 commit comments

Comments
 (0)