Skip to content

Commit 552b03f

Browse files
committed
feat(scraper): vacuum & analyze tables after scrape
1 parent 8467f8e commit 552b03f

File tree

1 file changed

+22
-0
lines changed

1 file changed

+22
-0
lines changed

backend/commands/scraper.ts

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -512,6 +512,24 @@ export default class Scraper extends BaseCommand {
512512
);
513513
}
514514

515+
async vacuumTablesTask(task: TaskHandle) {
516+
const tables = [
517+
"departments",
518+
"registrations",
519+
"courses",
520+
"groups_archive",
521+
"group_archive_lecturers",
522+
"lecturers",
523+
"groups",
524+
"group_lecturers",
525+
];
526+
527+
for (const table of tables) {
528+
task.update(`Vacuuming '${table}'`);
529+
await db.rawQuery("VACUUM ANALYZE ??", [table]);
530+
}
531+
}
532+
515533
async run() {
516534
this.scrapingSemaphore = new Semaphore(this.maxUsosRequests);
517535
this.dbSemaphore = new Semaphore(this.maxDbRequests);
@@ -542,6 +560,10 @@ export default class Scraper extends BaseCommand {
542560
await this.scrapeGroupsTask(task);
543561
return "Done";
544562
})
563+
.add("Vacuum & analyze tables", async (task) => {
564+
await this.vacuumTablesTask(task);
565+
return "Done";
566+
})
545567
.run();
546568
}
547569
}

0 commit comments

Comments
 (0)