feat: get turbo limits from database (#494)

* feat: get turbo limits from database

* refactor: fetch options from DB, without cache

* fix: fix missing await

* fix: fix wrong options name

* feat: migrate turbo limits checks from SDK (#495)

* feat: validate number of strategies depending on space type

* feat: check proposal limits

* refactor: avoid duplicate method calls

* chore(deps): bump @snapshot-labs/snapshot.js from 0.12.45 to 0.12.46 (#499)

Bumps [@snapshot-labs/snapshot.js](https://github.com/snapshot-labs/snapshot.js) from 0.12.45 to 0.12.46.
- [Release notes](https://github.com/snapshot-labs/snapshot.js/releases)
- [Commits](snapshot-labs/snapshot.js@v0.12.45...v0.12.46)

---
updated-dependencies:
- dependency-name: "@snapshot-labs/snapshot.js"
  dependency-type: direct:production
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <[email protected]>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
Co-authored-by: Chaitanya <[email protected]>

* fix tests

* Update src/writer/proposal.ts

* fix lint and remove extra brace

* Update src/writer/proposal.ts

* perf: get limits with a single sql query

* move getSpaceType after validateSpace

---------

Signed-off-by: dependabot[bot] <[email protected]>
Co-authored-by: Chaitanya <[email protected]>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>

Author: 3 people

package.json

@@ -32,7 +32,7 @@
     "@snapshot-labs/pineapple": "^1.1.0",
     "@snapshot-labs/snapshot-metrics": "^1.4.1",
     "@snapshot-labs/snapshot-sentry": "^1.5.5",
-    "@snapshot-labs/snapshot.js": "^0.12.45",
+    "@snapshot-labs/snapshot.js": "^0.12.46",
     "bluebird": "^3.7.2",
     "connection-string": "^1.0.1",
     "cors": "^2.8.5",

src/helpers/limits.ts

@@ -0,0 +1,57 @@
+import db from './mysql';
+
+async function getOptions<T>(
+  keys: string[],
+  defaultValue: T,
+  formattingFn: (val: string) => T
+): Promise<Record<string, T>> {
+  const results = keys.reduce((acc, key) => {
+    acc[key] = defaultValue;
+    return acc;
+  }, {});
+
+  const options = await db.queryAsync('select name, value from options where name in (?)', [keys]);
+
+  options.forEach(result => {
+    results[result.name] = formattingFn(result.value);
+  });
+
+  return results;
+}
+
+export async function getLimit(key: string): Promise<number> {
+  return (await getLimits([key]))[key];
+}
+
+export async function getList(key: string): Promise<string[]> {
+  return (await getLists([key]))[key];
+}
+
+export async function getLimits(keys: string[]): Promise<Record<string, number>> {
+  return await getOptions<number>(keys, 0, val => Number(val));
+}
+
+export async function getLists(keys: string[]): Promise<Record<string, string[]>> {
+  return await getOptions<string[]>(keys, [], val => val.split(','));
+}
+
+export async function getSpaceType(
+  space: {
+    verified: boolean;
+    turbo: boolean;
+    flagged: boolean;
+    id: string;
+  },
+  withEcosystem = false
+) {
+  let type = 'default';
+
+  if (withEcosystem && (await getList('space.ecosystem.list')).includes(space.id)) {
+    type = 'ecosystem';
+  }
+  if (space.flagged) type = 'flagged';
+  if (space.verified) type = 'verified';
+  if (space.turbo) type = 'turbo';
+
+  return type;
+}

src/helpers/options.ts

@@ -1,5 +1,5 @@
-import { FOLLOWS_LIMIT_PER_USER } from '../helpers/limits';
 import db from '../helpers/mysql';
+import { getLimit } from '../helpers/options';
 import { DEFAULT_NETWORK_ID, NETWORK_IDS } from '../helpers/utils';
 
 export const getFollowsCount = async (follower: string): Promise<number> => {
@@ -22,9 +22,10 @@ export async function verify(message): Promise<any> {
   if (follows.length !== 0) return Promise.reject('you are already following this space');
 
   const count = await getFollowsCount(message.from);
+  const limit = await getLimit('user.default.follow_limit');
 
-  if (count >= FOLLOWS_LIMIT_PER_USER) {
-    return Promise.reject(`you can join max ${FOLLOWS_LIMIT_PER_USER} spaces`);
+  if (count >= limit) {
+    return Promise.reject(`you can join max ${limit} spaces`);
   }
 
   if (message.network && !NETWORK_IDS.includes(message.network)) {

src/writer/follow.ts

@@ -3,11 +3,11 @@ import snapshot from '@snapshot-labs/snapshot.js';
 import networks from '@snapshot-labs/snapshot.js/src/networks.json';
 import { validateSpaceSettings } from './settings';
 import { getSpace } from '../helpers/actions';
-import { ACTIVE_PROPOSAL_BY_AUTHOR_LIMIT, getSpaceLimits } from '../helpers/limits';
 import log from '../helpers/log';
 import { containsFlaggedLinks, flaggedAddresses } from '../helpers/moderation';
 import { isMalicious } from '../helpers/monitoring';
 import db from '../helpers/mysql';
+import { getLimits, getSpaceType } from '../helpers/options';
 import { captureError, getQuorum, jsonParse, validateChoices } from '../helpers/utils';
 
 const scoreAPIUrl = process.env.SCORE_API_URL || 'https://score.snapshot.org';
@@ -57,7 +57,6 @@ export async function verify(body): Promise<any> {
   const created = parseInt(msg.timestamp);
   const addressLC = body.address.toLowerCase();
   const space = await getSpace(msg.space);
-
   try {
     await validateSpace(space);
   } catch (e) {
@@ -66,6 +65,17 @@ export async function verify(body): Promise<any> {
 
   space.id = msg.space;
 
+  const spaceType = await getSpaceType(space);
+  const spaceTypeWithEcosystem = await getSpaceType(space, true);
+
+  const limits = await getLimits([
+    `space.${spaceType}.body_limit`,
+    `space.${spaceType}.choices_limit`,
+    'space.active_proposal_limit_per_author',
+    `space.${spaceTypeWithEcosystem}.proposal_limit_per_day`,
+    `space.${spaceTypeWithEcosystem}.proposal_limit_per_month`
+  ]);
+
   const schemaIsValid = snapshot.utils.validateSchema(snapshot.schemas.proposal, msg.payload, {
     spaceType: space.turbo ? 'turbo' : 'default'
   });
@@ -186,16 +196,29 @@ export async function verify(body): Promise<any> {
       space.id,
       body.address
     );
-    const [dayLimit, monthLimit] = getSpaceLimits(space);
+
+    const dayLimit = limits[`space.${spaceTypeWithEcosystem}.proposal_limit_per_day`];
+    const monthLimit = limits[`space.${spaceTypeWithEcosystem}.proposal_limit_per_month`];
 
     if (dayCount >= dayLimit || monthCount >= monthLimit)
       return Promise.reject('proposal limit reached');
-    if (!isAuthorized && activeProposalsByAuthor >= ACTIVE_PROPOSAL_BY_AUTHOR_LIMIT)
+    const activeProposalLimitPerAuthor = limits['space.active_proposal_limit_per_author'];
+    if (!isAuthorized && activeProposalsByAuthor >= activeProposalLimitPerAuthor)
       return Promise.reject('active proposal limit reached for author');
   } catch (e) {
     capture(e);
     return Promise.reject('failed to check proposals limit');
   }
+
+  const bodyLengthLimit = limits[`space.${spaceType}.body_limit`];
+  if (msg.payload.body.length > bodyLengthLimit) {
+    return Promise.reject(`proposal body length can not exceed ${bodyLengthLimit} characters`);
+  }
+
+  const choicesLimit = limits[`space.${spaceType}.choices_limit`];
+  if (msg.payload.choices.length > choicesLimit) {
+    return Promise.reject(`number of choices can not exceed ${choicesLimit}`);
+  }
 }
 
 export async function action(body, ipfs, receipt, id): Promise<void> {

src/writer/proposal.ts

@@ -4,6 +4,7 @@ import isEqual from 'lodash/isEqual';
 import { addOrUpdateSpace, getSpace } from '../helpers/actions';
 import log from '../helpers/log';
 import db from '../helpers/mysql';
+import { getLimit, getSpaceType } from '../helpers/options';
 import { clearStampCache, DEFAULT_NETWORK, jsonParse } from '../helpers/utils';
 
 const SNAPSHOT_ENV = process.env.NETWORK || 'testnet';
@@ -73,6 +74,12 @@ export async function verify(body): Promise<any> {
     return Promise.reject(e);
   }
 
+  const strategiesLimit = await getLimit(`space.${await getSpaceType(space)}.strategies_limit`);
+
+  if (msg.payload.strategies.length > strategiesLimit) {
+    return Promise.reject(`max number of strategies is ${strategiesLimit}`);
+  }
+
   const controller = await snapshot.utils.getSpaceController(msg.space, DEFAULT_NETWORK, {
     broviderUrl
   });

src/writer/settings.ts

@@ -1,11 +1,62 @@
-import ingestor from '../../src/ingestor';
-import proposalInput from '../fixtures/ingestor-payload/proposal.json';
-import { spacesGetSpaceFixtures } from '../fixtures/space';
-import voteInput from '../fixtures/ingestor-payload/vote.json';
 import cloneDeep from 'lodash/cloneDeep';
 import omit from 'lodash/omit';
 import db, { sequencerDB } from '../../src/helpers/mysql';
 import relayer from '../../src/helpers/relayer';
+import ingestor from '../../src/ingestor';
+import proposalInput from '../fixtures/ingestor-payload/proposal.json';
+import voteInput from '../fixtures/ingestor-payload/vote.json';
+import { spacesGetSpaceFixtures } from '../fixtures/space';
+
+const LIMITS = {
+  'space.active_proposal_limit_per_author': 20,
+  'space.ecosystem.proposal_limit_per_day': 150,
+  'space.ecosystem.proposal_limit_per_month': 750,
+  'space.ecosystem.choices_limit': 20,
+  'space.ecosystem.body_length': 10000,
+  'space.ecosystem.strategies_limit': 8,
+  'space.flagged.proposal_limit_per_day': 5,
+  'space.flagged.proposal_limit_per_month': 7,
+  'space.flagged.choices_limit': 20,
+  'space.flagged.body_length': 10000,
+  'space.flagged.strategies_limit': 8,
+  'space.default.proposal_limit_per_day': 10,
+  'space.default.proposal_limit_per_month': 150,
+  'space.default.choices_limit': 20,
+  'space.default.body_length': 10000,
+  'space.default.strategies_limit': 8,
+  'space.turbo.proposal_limit_per_day': 40,
+  'space.turbo.proposal_limit_per_month': 200,
+  'space.turbo.choices_limit': 1000,
+  'space.turbo.body_length': 40000,
+  'space.turbo.strategies_limit': 10,
+  'space.verified.proposal_limit_per_day': 20,
+  'space.verified.proposal_limit_per_month': 100,
+  'space.verified.choices_limit': 20,
+  'space.verified.body_length': 10000,
+  'space.verified.strategies_limit': 6,
+  'user.default.follow.limit': 25
+};
+const ECOSYSTEM_LIST = ['test.eth', 'snapshot.eth'];
+jest.mock('../../src/helpers/options', () => {
+  const originalModule = jest.requireActual('../../src/helpers/options');
+
+  return {
+    __esModule: true,
+    ...originalModule,
+    getList: () => {
+      return ECOSYSTEM_LIST;
+    },
+    getLimit: async (key: string) => {
+      return LIMITS[key];
+    },
+    getLimits: () => {
+      return LIMITS;
+    },
+    getSpaceType: () => {
+      return 'default';
+    }
+  };
+});
 
 jest.mock('../../src/helpers/moderation', () => {
   const originalModule = jest.requireActual('../../src/helpers/moderation');

test/integration/ingestor.test.ts

@@ -1,8 +1,19 @@
-import { FOLLOWS_LIMIT_PER_USER } from '../../../src/helpers/limits';
 import db, { sequencerDB } from '../../../src/helpers/mysql';
 import { action, verify } from '../../../src/writer/follow';
 import { spacesSqlFixtures } from '../../fixtures/space';
 
+const LIMIT = 10;
+
+jest.mock('../../../src/helpers/options', () => {
+  const originalModule = jest.requireActual('../../../src/helpers/options');
+
+  return {
+    __esModule: true,
+    ...originalModule,
+    getLimit: () => LIMIT
+  };
+});
+
 describe('writer/follow', () => {
   const TEST_PREFIX = 'test-follow-';
   const space = spacesSqlFixtures[1];
@@ -21,7 +32,7 @@ describe('writer/follow', () => {
       let i = 1;
       const promises: Promise<any>[] = [];
 
-      while (i <= FOLLOWS_LIMIT_PER_USER) {
+      while (i <= LIMIT) {
         promises.push(
           db.queryAsync('INSERT INTO snapshot_sequencer_test.spaces SET ?', {
             ...space,
@@ -45,7 +56,7 @@ describe('writer/follow', () => {
 
     it('rejects when the user has followed too much spaces', () => {
       return expect(verify({ from: followerId })).rejects.toEqual(
-        `you can join max ${FOLLOWS_LIMIT_PER_USER} spaces`
+        `you can join max ${LIMIT} spaces`
       );
     });