diff --git a/.env.example b/.env.example index 3ff9dd04..7a9ab964 100644 --- a/.env.example +++ b/.env.example @@ -10,7 +10,7 @@ SIDEKICK_URL=https://sh5.co PINEAPPLE_URL=https://.../upload # optional SCORE_API_URL=https://score.snapshot.org SCHNAPS_API_URL=https://schnaps.snapshot.box -# If you need unlimted access to score-api, use `https://score.snapshot.org?apiKey=...` +# If you need unlimited access to score-api, use `https://score.snapshot.org?apiKey=...` RATE_LIMIT_DATABASE_URL= # optional RATE_LIMIT_KEYS_PREFIX=snapshot-sequencer: # optional BROVIDER_URL=https://rpc.snapshot.org # optional @@ -19,3 +19,5 @@ STARKNET_RPC_URL= # optional AUTH_SECRET=1dfd84a695705665668c260222ded178d1f1d62d251d7bee8148428dac6d0487 # optional WALLETCONNECT_PROJECT_ID=e6454bd61aba40b786e866a69bd4c5c6 REOWN_SECRET= # optional +OVERLORD_URL=https://overlord.snapshot.org +LAST_CB=1 diff --git a/package.json b/package.json index ecf6cd20..451eb213 100644 --- a/package.json +++ b/package.json @@ -53,7 +53,7 @@ "@snapshot-labs/eslint-config": "^0.1.0-beta.18", "@snapshot-labs/prettier-config": "^0.1.0-beta.7", "@types/jest": "^29.5.2", - "@types/node": "^14.0.13", + "@types/node": "^16.0.0", "dotenv-cli": "^7.2.1", "eslint": "^8.28.0", "jest": "^29.6.1", diff --git a/src/helpers/entityValue.ts b/src/helpers/entityValue.ts new file mode 100644 index 00000000..66478445 --- /dev/null +++ b/src/helpers/entityValue.ts @@ -0,0 +1,26 @@ +import { jsonRpcRequest } from './utils'; + +type Proposal = { + network: string; + strategies: any[]; + start: number; +}; + +const OVERLORD_URL = process.env.OVERLORD_URL ?? 'https://overlord.snapshot.org'; +// Round strategy values to 9 decimal places +const STRATEGIES_VALUE_PRECISION = 9; + +export async function getStrategiesValue(proposal: Proposal): Promise { + const result: number[] = await jsonRpcRequest(OVERLORD_URL, 'get_vp_value_by_strategy', { + network: proposal.network, + strategies: proposal.strategies, + snapshot: proposal.start + }); + + // Handle unlikely case where strategies value array length does not match strategies length + if (result.length !== proposal.strategies.length) { + throw new Error('Strategies value length mismatch'); + } + + return result.map(value => parseFloat(value.toFixed(STRATEGIES_VALUE_PRECISION))); +} diff --git a/src/helpers/shutter.ts b/src/helpers/shutter.ts index 4b062402..85c7ea98 100644 --- a/src/helpers/shutter.ts +++ b/src/helpers/shutter.ts @@ -1,4 +1,3 @@ -import { randomBytes } from 'crypto'; import { arrayify } from '@ethersproject/bytes'; import { toUtf8String } from '@ethersproject/strings'; import { decrypt, init } from '@shutter-network/shutter-crypto'; @@ -6,7 +5,7 @@ import { capture } from '@snapshot-labs/snapshot-sentry'; import express from 'express'; import log from './log'; import db from './mysql'; -import { fetchWithKeepAlive, getIp, jsonParse, rpcError, rpcSuccess } from './utils'; +import { getIp, jsonParse, jsonRpcRequest, rpcError, rpcSuccess } from './utils'; import { updateProposalAndVotes } from '../scores'; init().then(() => log.info('[shutter] init')); @@ -36,28 +35,9 @@ function idToProposal(id: string): string { return `0x${id}`; } -export async function rpcRequest(method, params, url: string = SHUTTER_URL) { - const init = { - method: 'POST', - headers: { - Accept: 'application/json', - 'Content-Type': 'application/json' - }, - body: JSON.stringify({ - jsonrpc: '2.0', - method, - params, - id: randomBytes(6).toString('hex') - }) - }; - const res = await fetchWithKeepAlive(url, init); - const { result } = await res.json(); - return result; -} - export async function getDecryptionKey(proposal: string, url: string = SHUTTER_URL) { const id = proposalToId(proposal); - const result = await rpcRequest('get_decryption_key', ['1', id], url); + const result = await jsonRpcRequest(url, 'get_decryption_key', ['1', id]); log.info(`[shutter] get_decryption_key ${proposal} ${JSON.stringify(result)}`); return result; diff --git a/src/helpers/utils.ts b/src/helpers/utils.ts index ea9af0f0..bf246eeb 100644 --- a/src/helpers/utils.ts +++ b/src/helpers/utils.ts @@ -1,4 +1,4 @@ -import { createHash } from 'crypto'; +import { createHash, randomUUID } from 'crypto'; import http from 'http'; import https from 'https'; import { URL } from 'url'; @@ -62,6 +62,46 @@ export function rpcError(res, code, e, id) { }); } +export async function jsonRpcRequest(url: string, method: string, params: any): Promise { + const init = { + method: 'POST', + headers: { + Accept: 'application/json', + 'Content-Type': 'application/json' + }, + body: JSON.stringify({ + jsonrpc: '2.0', + method, + params, + id: randomUUID() + }) + }; + + try { + const res = await fetchWithKeepAlive(url, init); + + if (!res.ok) { + throw new Error(`HTTP error: ${res.status} ${res.statusText}`); + } + + const response = await res.json(); + + if (response.error) { + throw new Error( + `JSON-RPC error: ${response.error.message || response.error.code || 'Unknown error'}` + ); + } + + return response.result; + } catch (error) { + capture(error, { + url, + request: { method, params } + }); + throw error; + } +} + export function hasStrategyOverride(strategies: any[]) { const keywords = [ '"aura-vlaura-vebal-with-overrides"', diff --git a/src/writer/proposal.ts b/src/writer/proposal.ts index 9cec1e74..2a730a09 100644 --- a/src/writer/proposal.ts +++ b/src/writer/proposal.ts @@ -4,6 +4,7 @@ import networks from '@snapshot-labs/snapshot.js/src/networks.json'; import { uniq } from 'lodash'; import { validateSpaceSettings } from './settings'; import { getPremiumNetworkIds, getSpace } from '../helpers/actions'; +import { getStrategiesValue } from '../helpers/entityValue'; import log from '../helpers/log'; import { containsFlaggedLinks, flaggedAddresses } from '../helpers/moderation'; import { isMalicious } from '../helpers/monitoring'; @@ -13,6 +14,7 @@ import { captureError, getQuorum, jsonParse, validateChoices } from '../helpers/ const scoreAPIUrl = process.env.SCORE_API_URL || 'https://score.snapshot.org'; const broviderUrl = process.env.BROVIDER_URL || 'https://rpc.snapshot.org'; +const LAST_CB = parseInt(process.env.LAST_CB ?? '1'); export const getProposalsCount = async (space, author) => { const query = ` @@ -241,9 +243,24 @@ export async function verify(body): Promise { if (msg.payload.choices.length > choicesLimit) { return Promise.reject(`number of choices can not exceed ${choicesLimit}`); } + + let strategiesValue: number[] = []; + + try { + strategiesValue = await getStrategiesValue({ + network: space.network, + start: msg.payload.start, + strategies: space.strategies + }); + } catch (e: any) { + log.warn('unable to get strategies value', e.message); + return Promise.reject('failed to get strategies value'); + } + + return { strategiesValue }; } -export async function action(body, ipfs, receipt, id): Promise { +export async function action(body, ipfs, receipt, id, context): Promise { const msg = jsonParse(body.msg); const space = msg.space; @@ -268,7 +285,7 @@ export async function action(body, ipfs, receipt, id): Promise { try { quorum = await getQuorum(spaceSettings.plugins.quorum, spaceNetwork, proposalSnapshot); } catch (e: any) { - console.log('unable to get quorum', e.message); + log.warn('unable to get quorum', e.message); return Promise.reject('unable to get quorum'); } } @@ -301,10 +318,12 @@ export async function action(body, ipfs, receipt, id): Promise { scores_state: 'pending', scores_total: 0, scores_updated: 0, - vp_value_by_strategy: JSON.stringify([]), + scores_total_value: 0, + vp_value_by_strategy: JSON.stringify(context.strategiesValue), votes: 0, validation, - flagged: +containsFlaggedLinks(msg.payload.body) + flagged: +containsFlaggedLinks(msg.payload.body), + cb: LAST_CB }; const query = ` diff --git a/test/integration/ingestor.test.ts b/test/integration/ingestor.test.ts index bc2483b0..f9edb56a 100644 --- a/test/integration/ingestor.test.ts +++ b/test/integration/ingestor.test.ts @@ -111,6 +111,11 @@ jest.mock('@snapshot-labs/pineapple', () => { }; }); +jest.mock('../../src/helpers/entityValue', () => ({ + __esModule: true, + getStrategiesValue: jest.fn(() => Promise.resolve([])) +})); + const proposalRequest = { headers: { 'x-real-ip': '1.1.1.1' }, body: proposalInput diff --git a/test/integration/writer/proposal.test.ts b/test/integration/writer/proposal.test.ts index 28565d67..171d322f 100644 --- a/test/integration/writer/proposal.test.ts +++ b/test/integration/writer/proposal.test.ts @@ -19,6 +19,11 @@ jest.mock('../../../src/helpers/moderation', () => { }; }); +jest.mock('../../../src/helpers/entityValue', () => ({ + __esModule: true, + getStrategiesValue: jest.fn(() => Promise.resolve([])) +})); + const getSpaceMock = jest.spyOn(actionHelper, 'getSpace'); getSpaceMock.mockResolvedValue(spacesGetSpaceFixtures); @@ -45,7 +50,7 @@ describe('writer/proposal', () => { expect.hasAssertions(); mockContainsFlaggedLinks.mockReturnValueOnce(true); const id = '0x01-flagged'; - expect(await action(input, 'ipfs', 'receipt', id)).toBeUndefined(); + expect(await action(input, 'ipfs', 'receipt', id, { strategiesValue: [] })).toBeUndefined(); expect(mockContainsFlaggedLinks).toBeCalledTimes(1); const [proposal] = await db.queryAsync('SELECT * FROM proposals WHERE id = ?', [id]); @@ -57,7 +62,7 @@ describe('writer/proposal', () => { it('creates and does not flag proposal', async () => { expect.hasAssertions(); const id = '0x02-non-flagged'; - expect(await action(input, 'ipfs', 'receipt', id)).toBeUndefined(); + expect(await action(input, 'ipfs', 'receipt', id, { strategiesValue: [] })).toBeUndefined(); expect(mockContainsFlaggedLinks).toBeCalledTimes(1); const [proposal] = await db.queryAsync('SELECT * FROM proposals WHERE id = ?', [id]); diff --git a/test/unit/writer/proposal.test.ts b/test/unit/writer/proposal.test.ts index 90a87af5..3e99e46d 100644 --- a/test/unit/writer/proposal.test.ts +++ b/test/unit/writer/proposal.test.ts @@ -93,6 +93,11 @@ jest.mock('../../../src/helpers/moderation', () => { }; }); +jest.mock('../../../src/helpers/entityValue', () => ({ + __esModule: true, + getStrategiesValue: jest.fn(() => Promise.resolve([])) +})); + const mockGetProposalsCount = jest.spyOn(writer, 'getProposalsCount'); mockGetProposalsCount.mockResolvedValue([ { @@ -155,7 +160,7 @@ describe('writer/proposal', () => { msg.payload.type = 'basic'; msg.payload.choices = ['For', 'Against', 'Abstain']; - await expect(writer.verify({ ...input, msg: JSON.stringify(msg) })).resolves.toBeUndefined(); + await expect(writer.verify({ ...input, msg: JSON.stringify(msg) })).resolves.toBeDefined(); expect(mockGetSpace).toHaveBeenCalledTimes(1); expect(mockGetProposalsCount).toHaveBeenCalledTimes(1); }); @@ -193,7 +198,7 @@ describe('writer/proposal', () => { voteValidation: { name: 'gitcoin' } }); - await expect(writer.verify(input)).resolves.toBeUndefined(); + await expect(writer.verify(input)).resolves.toBeDefined(); expect(mockGetSpace).toHaveBeenCalledTimes(1); expect(mockGetProposalsCount).toHaveBeenCalledTimes(1); }); @@ -223,7 +228,7 @@ describe('writer/proposal', () => { voting: { ...spacesGetSpaceFixtures.voting, period: VOTING_PERIOD } }); - await expect(writer.verify(inputWithVotingPeriod)).resolves.toBeUndefined(); + await expect(writer.verify(inputWithVotingPeriod)).resolves.toBeDefined(); expect(mockGetSpace).toHaveBeenCalledTimes(1); expect(mockGetProposalsCount).toHaveBeenCalledTimes(1); }); @@ -255,7 +260,7 @@ describe('writer/proposal', () => { await expect( writer.verify({ ...input, msg: JSON.stringify(msg) }) - ).resolves.toBeUndefined(); + ).resolves.toBeDefined(); expect(mockGetSpace).toHaveBeenCalledTimes(1); expect(mockGetProposalsCount).toHaveBeenCalledTimes(1); }); @@ -320,7 +325,7 @@ describe('writer/proposal', () => { it('does not validate the space validation', async () => { expect.assertions(2); - await expect(writer.verify(input)).resolves.toBeUndefined(); + await expect(writer.verify(input)).resolves.toBeDefined(); expect(mockSnapshotUtilsValidate).toHaveBeenCalledTimes(0); }); }); @@ -329,7 +334,7 @@ describe('writer/proposal', () => { it('does not validate the space validation', async () => { expect.assertions(2); - await expect(writer.verify(input)).resolves.toBeUndefined(); + await expect(writer.verify(input)).resolves.toBeDefined(); expect(mockSnapshotUtilsValidate).toHaveBeenCalledTimes(0); }); }); @@ -345,19 +350,19 @@ describe('writer/proposal', () => { it('accepts a proposal with shutter privacy', () => { return expect( writer.verify(updateInputPayload(input, { privacy: 'shutter' })) - ).resolves.toBeUndefined(); + ).resolves.toBeDefined(); }); it('accepts a proposal with undefined privacy', () => { return expect( writer.verify(updateInputPayload(input, { privacy: undefined })) - ).resolves.toBeUndefined(); + ).resolves.toBeDefined(); }); it('accepts a proposal with no privacy', () => { return expect( writer.verify(updateInputPayload(input, { privacy: '' })) - ).resolves.toBeUndefined(); + ).resolves.toBeDefined(); }); }); @@ -373,19 +378,19 @@ describe('writer/proposal', () => { it('accepts a proposal with shutter privacy', () => { return expect( writer.verify(updateInputPayload(input, { privacy: 'shutter' })) - ).resolves.toBeUndefined(); + ).resolves.toBeDefined(); }); it('accepts a proposal with undefined privacy', () => { return expect( writer.verify(updateInputPayload(input, { privacy: undefined })) - ).resolves.toBeUndefined(); + ).resolves.toBeDefined(); }); it('accepts a proposal with no privacy', () => { return expect( writer.verify(updateInputPayload(input, { privacy: '' })) - ).resolves.toBeUndefined(); + ).resolves.toBeDefined(); }); }); @@ -406,13 +411,13 @@ describe('writer/proposal', () => { it('accepts a proposal with undefined privacy', () => { return expect( writer.verify(updateInputPayload(input, { privacy: undefined })) - ).resolves.toBeUndefined(); + ).resolves.toBeDefined(); }); it('accepts a proposal with no privacy', () => { return expect( writer.verify(updateInputPayload(input, { privacy: '' })) - ).resolves.toBeUndefined(); + ).resolves.toBeDefined(); }); }); @@ -427,13 +432,13 @@ describe('writer/proposal', () => { it('accepts a proposal with shutter privacy', () => { return expect( writer.verify(updateInputPayload(input, { privacy: 'shutter' })) - ).resolves.toBeUndefined(); + ).resolves.toBeDefined(); }); it('accepts a proposal with undefined privacy', () => { return expect( writer.verify(updateInputPayload(input, { privacy: undefined })) - ).resolves.toBeUndefined(); + ).resolves.toBeDefined(); }); it('rejects a proposal with privacy empty string', async () => { @@ -605,7 +610,7 @@ describe('writer/proposal', () => { turbo: '1' }); - await expect(writer.verify(input)).resolves.toBeUndefined(); + await expect(writer.verify(input)).resolves.toBeDefined(); expect(mockGetSpace).toHaveBeenCalledTimes(1); expect(mockGetProposalsCount).toHaveBeenCalledTimes(1); }); @@ -678,7 +683,7 @@ describe('writer/proposal', () => { it('verifies a valid input', async () => { expect.assertions(1); - await expect(writer.verify(input)).resolves.toBeUndefined(); + await expect(writer.verify(input)).resolves.toBeDefined(); }); }); }); diff --git a/yarn.lock b/yarn.lock index 40804aa9..25e06b6b 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1450,10 +1450,10 @@ resolved "https://registry.yarnpkg.com/@types/node/-/node-20.4.0.tgz#01d637d1891e419bc85763b46f42809cd2d5addb" integrity sha512-jfT7iTf/4kOQ9S7CHV9BIyRaQqHu67mOjsIQBC3BKZvzvUB6zLxEwJ6sBE3ozcvP8kF6Uk5PXN0Q+c0dfhGX0g== -"@types/node@^14.0.13": - version "14.14.44" - resolved "https://registry.yarnpkg.com/@types/node/-/node-14.14.44.tgz#df7503e6002847b834371c004b372529f3f85215" - integrity sha512-+gaugz6Oce6ZInfI/tK4Pq5wIIkJMEJUu92RB3Eu93mtj4wjjjz9EB5mLp5s1pSsLXdC/CPut/xF20ZzAQJbTA== +"@types/node@^16.0.0": + version "16.18.126" + resolved "https://registry.yarnpkg.com/@types/node/-/node-16.18.126.tgz#27875faa2926c0f475b39a8bb1e546c0176f8d4b" + integrity sha512-OTcgaiwfGFBKacvfwuHzzn1KLxH/er8mluiy8/uM3sGXHaRe73RrSIj01jow9t4kJEW633Ov+cOexXeiApTyAw== "@types/prettier@^2.1.5": version "2.7.3"