fix: add pagination to loadSpaces (#1042)

* fix: add pagination to loadSpaces

* run map at the end

* fix empty line

* remove try catch like before

Author: ChaituVR

src/helpers/spaces.ts

@@ -6,6 +6,7 @@ import log from './log';
 import db from './mysql';
 
 const RUN_INTERVAL = 120e3;
+const BATCH_SIZE = 70000;
 const TEST_STRATEGIES = [
   'ticket',
   'api',
@@ -160,17 +161,37 @@ function mapSpaces(spaces: Record<string, any>) {
 
 async function loadSpaces() {
   const startTime = +Date.now();
+  let offset = 0;
+  let allResults: any[] = [];
+  let hasMore = true;
+  let batchCount = 0;
+
+  while (hasMore) {
+    const query = `
+        SELECT id, settings, flagged, verified, turbo, turbo_expiration, hibernated, follower_count, proposal_count, vote_count
+        FROM spaces
+        WHERE deleted = 0
+        ORDER BY id ASC
+        LIMIT ?, ?
+      `;
+
+    const results = await db.queryAsync(query, [offset, BATCH_SIZE]);
+    batchCount++;
+
+    allResults = allResults.concat(results);
+    offset += BATCH_SIZE;
+
+    log.info(
+      `[spaces] loaded batch ${batchCount}: ${results.length} spaces (total: ${allResults.length})`
+    );
 
-  const query = `
-      SELECT id, settings, flagged, verified, turbo, turbo_expiration, hibernated, follower_count, proposal_count, vote_count
-      FROM spaces
-      WHERE deleted = 0
-      ORDER BY id ASC
-    `;
-  const results = await db.queryAsync(query);
+    if (results.length < BATCH_SIZE) {
+      hasMore = false;
+    }
+  }
 
   const spaces = Object.fromEntries(
-    results.map(space => [
+    allResults.map(space => [
       space.id,
       {
         ...JSON.parse(space.settings),
@@ -188,7 +209,9 @@ async function loadSpaces() {
   );
 
   log.info(
-    `[spaces] total spaces ${Object.keys(spaces).length}, in (${(
+    `[spaces] total spaces ${
+      Object.keys(spaces).length
+    }, in ${batchCount} batches (${(
       (+Date.now() - startTime) /
       1000
     ).toFixed()}s)`