diff --git a/src/helpers/spaces.ts b/src/helpers/spaces.ts index 4d9d1542..c26fb9b1 100644 --- a/src/helpers/spaces.ts +++ b/src/helpers/spaces.ts @@ -6,6 +6,7 @@ import log from './log'; import db from './mysql'; const RUN_INTERVAL = 120e3; +const BATCH_SIZE = 70000; const TEST_STRATEGIES = [ 'ticket', 'api', @@ -160,17 +161,37 @@ function mapSpaces(spaces: Record) { async function loadSpaces() { const startTime = +Date.now(); + let offset = 0; + let allResults: any[] = []; + let hasMore = true; + let batchCount = 0; + + while (hasMore) { + const query = ` + SELECT id, settings, flagged, verified, turbo, turbo_expiration, hibernated, follower_count, proposal_count, vote_count + FROM spaces + WHERE deleted = 0 + ORDER BY id ASC + LIMIT ?, ? + `; + + const results = await db.queryAsync(query, [offset, BATCH_SIZE]); + batchCount++; + + allResults = allResults.concat(results); + offset += BATCH_SIZE; + + log.info( + `[spaces] loaded batch ${batchCount}: ${results.length} spaces (total: ${allResults.length})` + ); - const query = ` - SELECT id, settings, flagged, verified, turbo, turbo_expiration, hibernated, follower_count, proposal_count, vote_count - FROM spaces - WHERE deleted = 0 - ORDER BY id ASC - `; - const results = await db.queryAsync(query); + if (results.length < BATCH_SIZE) { + hasMore = false; + } + } const spaces = Object.fromEntries( - results.map(space => [ + allResults.map(space => [ space.id, { ...JSON.parse(space.settings), @@ -188,7 +209,9 @@ async function loadSpaces() { ); log.info( - `[spaces] total spaces ${Object.keys(spaces).length}, in (${( + `[spaces] total spaces ${ + Object.keys(spaces).length + }, in ${batchCount} batches (${( (+Date.now() - startTime) / 1000 ).toFixed()}s)`