feat: validate space strategies

src/helpers/strategies.ts

@@ -0,0 +1,68 @@
+import { URL } from 'url';
+import { capture } from '@snapshot-labs/snapshot-sentry';
+import snapshot from '@snapshot-labs/snapshot.js';
+import log from './log';
+
+type Strategy = {
+  id: string;
+  override: boolean;
+  disabled: boolean;
+};
+
+const RUN_INTERVAL = 60e3;
+const MAX_CONSECUTIVE_FAILS = 3;
+const URI = new URL(
+  '/api/strategies',
+  process.env.SCORE_API_URL ?? 'https://score.snapshot.org'
+).toString();
+
+let consecutiveFailsCount = 0;
+let shouldStop = false;
+export let strategies: Record<Strategy['id'], Strategy> = {};
+
+async function loadStrategies() {
+  const res = await snapshot.utils.getJSON(URI);
+
+  if (res.hasOwnProperty('error')) {
+    capture(new Error('Failed to load strategies'), {
+      contexts: { input: { uri: URI }, res }
+    });
+    return true;
+  }
+
+  const strat = Object.values(res).map((strategy: any) => {
+    strategy.id = strategy.key;
+    strategy.override = strategy.dependOnOtherAddress || false;
+    strategy.disabled = strategy.disabled || false;
+    return strategy;
+  });
+
+  strategies = Object.fromEntries(strat.map(strategy => [strategy.id, strategy]));
+}
+
+export async function run() {
+  while (!shouldStop) {
+    try {
+      log.info('[strategies] Start strategies refresh');
+      await loadStrategies();
+      consecutiveFailsCount = 0;
+      log.info('[strategies] End strategies refresh');
+    } catch (e: any) {
+      consecutiveFailsCount++;
+
+      if (consecutiveFailsCount >= MAX_CONSECUTIVE_FAILS) {
+        capture(e);
+      }
+      log.error(`[strategies] failed to load ${JSON.stringify(e)}`);
+    }
+
+    // if stop() has been called after sleep started,
+    // the loop will exit only after the sleep has completed
+    await snapshot.utils.sleep(RUN_INTERVAL);
+  }
+}
+
+export function stop() {
+  log.info('[strategies] Stopping strategies refresh');
+  shouldStop = true;
+}

src/helpers/validation.ts

@@ -0,0 +1,80 @@
+import snapshot from '@snapshot-labs/snapshot.js';
+import log from './log';
+import { strategies } from './strategies';
+
+const DEFAULT_SNAPSHOT_ENV: string = 'testnet';
+
+export async function validateSpaceSettings(
+  originalSpace: any,
+  snapshotEnv = DEFAULT_SNAPSHOT_ENV
+): Promise<void> {
+  const spaceType = originalSpace.turbo ? 'turbo' : 'default';
+  const space = snapshot.utils.clone(originalSpace);
+
+  if (space?.deleted) return Promise.reject('space deleted, contact admin');
+
+  delete space.deleted;
+  delete space.flagged;
+  delete space.verified;
+  delete space.turbo;
+  delete space.hibernated;
+  delete space.id;
+
+  if (space.parent && space.parent === originalSpace.id) {
+    return Promise.reject('space cannot be its own parent');
+  }
+
+  if (
+    space.children &&
+    Array.isArray(space.children) &&
+    space.children.includes(originalSpace.id)
+  ) {
+    return Promise.reject('space cannot be its own child');
+  }
+
+  const schemaIsValid: any = snapshot.utils.validateSchema(snapshot.schemas.space, space, {
+    spaceType,
+    snapshotEnv
+  });
+
+  if (schemaIsValid !== true) {
+    log.warn('[writer] Wrong space format', schemaIsValid);
+    const firstErrorObject: any = Object.values(schemaIsValid)[0];
+    if (firstErrorObject.message === 'network not allowed') {
+      return Promise.reject(firstErrorObject.message);
+    }
+    return Promise.reject('wrong space format');
+  }
+
+  const strategiesIds: string[] = space.strategies.map((strategy: any) => strategy.name);
+  if (snapshotEnv !== 'testnet') {
+    const hasTicket = strategiesIds.includes('ticket');
+    const hasVotingValidation =
+      space.voteValidation?.name && !['any'].includes(space.voteValidation.name);
+
+    if (hasTicket && !hasVotingValidation) {
+      return Promise.reject('space with ticket requires voting validation');
+    }
+
+    const hasProposalValidation =
+      (space.validation?.name && space.validation.name !== 'any') ||
+      space.filters?.minScore ||
+      space.filters?.onlyMembers;
+
+    if (!hasProposalValidation) {
+      return Promise.reject('space missing proposal validation');
+    }
+  }
+
+  for (const id of strategiesIds) {
+    const strategy = strategies[id];
+
+    if (!strategy) {
+      return Promise.reject(`strategy "${id}" is not a valid strategy`);
+    }
+
+    if (strategy.disabled) {
+      return Promise.reject(`strategy "${id}" is not available anymore`);
+    }
+  }
+}

src/index.ts

@@ -8,12 +8,14 @@ import initMetrics from './helpers/metrics';
 import refreshModeration from './helpers/moderation';
 import rateLimit from './helpers/rateLimit';
 import shutter from './helpers/shutter';
+import { run as refreshStrategies, stop as stopStrategies } from './helpers/strategies';
 import { trackTurboStatuses } from './helpers/turbo';
 
 const app = express();
 
 initLogger(app);
 refreshModeration();
+refreshStrategies();
 initMetrics(app);
 trackTurboStatuses();
 
@@ -29,4 +31,23 @@ app.use('/shutter', shutter);
 fallbackLogger(app);
 
 const PORT = process.env.PORT || 3001;
-app.listen(PORT, () => log.info(`Started on: http://localhost:${PORT}`));
+const server = app.listen(PORT, () => log.info(`Started on: http://localhost:${PORT}`));
+
+const gracefulShutdown = (signal: string) => {
+  log.info(`Received ${signal}, shutting down gracefully...`);
+
+  stopStrategies();
+
+  server.close(() => {
+    log.info('Server closed');
+    process.exit(0);
+  });
+
+  setTimeout(() => {
+    log.error('Could not close connections in time, forcefully shutting down');
+    process.exit(1);
+  }, 10000);
+};
+
+process.on('SIGTERM', () => gracefulShutdown('SIGTERM'));
+process.on('SIGINT', () => gracefulShutdown('SIGINT'));

src/writer/proposal.ts

@@ -2,14 +2,14 @@ import { capture } from '@snapshot-labs/snapshot-sentry';
 import snapshot from '@snapshot-labs/snapshot.js';
 import networks from '@snapshot-labs/snapshot.js/src/networks.json';
 import { uniq } from 'lodash';
-import { validateSpaceSettings } from './settings';
 import { getPremiumNetworkIds, getSpace } from '../helpers/actions';
 import log from '../helpers/log';
 import { containsFlaggedLinks, flaggedAddresses } from '../helpers/moderation';
 import { isMalicious } from '../helpers/monitoring';
 import db from '../helpers/mysql';
 import { getLimits, getSpaceType } from '../helpers/options';
 import { captureError, getQuorum, jsonParse, validateChoices } from '../helpers/utils';
+import { validateSpaceSettings } from '../helpers/validation';
 
 const scoreAPIUrl = process.env.SCORE_API_URL || 'https://score.snapshot.org';
 const broviderUrl = process.env.BROVIDER_URL || 'https://rpc.snapshot.org';
@@ -66,7 +66,7 @@ async function validateSpace(space: any) {
   }
 
   try {
-    await validateSpaceSettings(space);
+    await validateSpaceSettings(space, process.env.NETWORK);
   } catch (e) {
     return Promise.reject(e);
   }

src/writer/settings.ts

@@ -1,5 +1,4 @@
 import { capture } from '@snapshot-labs/snapshot-sentry';
-import snapshot from '@snapshot-labs/snapshot.js';
 import isEqual from 'lodash/isEqual';
 import { addOrUpdateSpace, getSpace } from '../helpers/actions';
 import log from '../helpers/log';
@@ -12,68 +11,10 @@ import {
   jsonParse,
   removeFromWalletConnectWhitelist
 } from '../helpers/utils';
+import { validateSpaceSettings } from '../helpers/validation';
 
 const SNAPSHOT_ENV = process.env.NETWORK || 'testnet';
 
-export async function validateSpaceSettings(originalSpace: any) {
-  const spaceType = originalSpace.turbo ? 'turbo' : 'default';
-  const space = snapshot.utils.clone(originalSpace);
-
-  if (space?.deleted) return Promise.reject('space deleted, contact admin');
-
-  delete space.deleted;
-  delete space.flagged;
-  delete space.verified;
-  delete space.turbo;
-  delete space.hibernated;
-  delete space.id;
-
-  if (space.parent && space.parent === originalSpace.id) {
-    return Promise.reject('space cannot be its own parent');
-  }
-
-  if (
-    space.children &&
-    Array.isArray(space.children) &&
-    space.children.includes(originalSpace.id)
-  ) {
-    return Promise.reject('space cannot be its own child');
-  }
-
-  const schemaIsValid: any = snapshot.utils.validateSchema(snapshot.schemas.space, space, {
-    spaceType,
-    snapshotEnv: SNAPSHOT_ENV
-  });
-
-  if (schemaIsValid !== true) {
-    log.warn('[writer] Wrong space format', schemaIsValid);
-    const firstErrorObject: any = Object.values(schemaIsValid)[0];
-    if (firstErrorObject.message === 'network not allowed') {
-      return Promise.reject(firstErrorObject.message);
-    }
-    return Promise.reject('wrong space format');
-  }
-
-  if (SNAPSHOT_ENV !== 'testnet') {
-    const hasTicket = space.strategies.some(strategy => strategy.name === 'ticket');
-    const hasVotingValidation =
-      space.voteValidation?.name && !['any'].includes(space.voteValidation.name);
-
-    if (hasTicket && !hasVotingValidation) {
-      return Promise.reject('space with ticket requires voting validation');
-    }
-
-    const hasProposalValidation =
-      (space.validation?.name && space.validation.name !== 'any') ||
-      space.filters?.minScore ||
-      space.filters?.onlyMembers;
-
-    if (!hasProposalValidation) {
-      return Promise.reject('space missing proposal validation');
-    }
-  }
-}
-
 export async function verify(body): Promise<any> {
   const msg = jsonParse(body.msg);
   if (msg.space.length > 64) {
@@ -82,12 +23,15 @@ export async function verify(body): Promise<any> {
   const space = await getSpace(msg.space, true);
 
   try {
-    await validateSpaceSettings({
-      ...msg.payload,
-      id: msg.space,
-      deleted: space?.deleted,
-      turbo: space?.turbo
-    });
+    await validateSpaceSettings(
+      {
+        ...msg.payload,
+        id: msg.space,
+        deleted: space?.deleted,
+        turbo: space?.turbo
+      },
+      process.env.NETWORK
+    );
   } catch (e) {
     return Promise.reject(e);
   }

test/integration/ingestor.test.ts

@@ -2,6 +2,7 @@ import cloneDeep from 'lodash/cloneDeep';
 import omit from 'lodash/omit';
 import db, { sequencerDB } from '../../src/helpers/mysql';
 import relayer from '../../src/helpers/relayer';
+import { run, stop } from '../../src/helpers/strategies';
 import ingestor from '../../src/ingestor';
 import proposalInput from '../fixtures/ingestor-payload/proposal.json';
 import voteInput from '../fixtures/ingestor-payload/vote.json';
@@ -37,6 +38,7 @@ const LIMITS = {
   'user.default.follow.limit': 25
 };
 const ECOSYSTEM_LIST = ['test.eth', 'snapshot.eth'];
+
 jest.mock('../../src/helpers/options', () => {
   const originalModule = jest.requireActual('../../src/helpers/options');
 
@@ -129,7 +131,13 @@ function cloneWithNewMessage(data: Record<string, any>) {
 }
 
 describe('ingestor', () => {
-  beforeAll(() => {
+  beforeAll(async () => {
+    // Start the strategies loader (runs in background)
+    run();
+
+    // Wait a bit for the first load to complete
+    await new Promise(resolve => setTimeout(resolve, 2000));
+
     proposalInput.data.message.timestamp = Math.floor(Date.now() / 1e3) - 60;
     proposalInput.data.message.end = Math.floor(Date.now() / 1e3) + 60;
     voteInput.data.message.timestamp = Math.floor(Date.now() / 1e3) - 60;
@@ -142,6 +150,8 @@ describe('ingestor', () => {
   });
 
   afterAll(async () => {
+    // Stop any running strategies loader
+    stop();
     await db.queryAsync('DELETE FROM snapshot_sequencer_test.proposals;');
     await db.queryAsync('DELETE FROM snapshot_sequencer_test.messages;');
     await db.endAsync();