Skip to content

Commit 49162ef

Browse files
wa0x6ewa0x6e
committed
refactor: better strategies handling
refactor: refactor:
1 parent ec91679 commit 49162ef

File tree

9 files changed

+425
-311
lines changed

9 files changed

+425
-311
lines changed

‎src/helpers/strategies.ts

Lines changed: 63 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,63 @@
1+
import { URL } from 'url';
2+
import { capture } from '@snapshot-labs/snapshot-sentry';
3+
import snapshot from '@snapshot-labs/snapshot.js';
4+
import log from './log';
5+
6+
type Strategy = {
7+
id: string;
8+
override: boolean;
9+
disabled: boolean;
10+
};
11+
12+
const RUN_INTERVAL = 60e3;
13+
const URI = new URL(
14+
'/api/strategies',
15+
process.env.SCORE_API_URL ?? 'https://score.snapshot.org'
16+
).toString();
17+
18+
let consecutiveFailsCount = 0;
19+
let shouldStop = false;
20+
export let strategies: Record<Strategy['id'], Strategy> = {};
21+
22+
async function loadStrategies() {
23+
const res = await snapshot.utils.getJSON(URI);
24+
25+
if (res.hasOwnProperty('error')) {
26+
capture(new Error('Failed to load strategies'), {
27+
contexts: { input: { uri: URI }, res }
28+
});
29+
return true;
30+
}
31+
32+
const strat = Object.values(res).map((strategy: any) => {
33+
strategy.id = strategy.key;
34+
strategy.override = strategy.dependOnOtherAddress || false;
35+
strategy.disabled = strategy.disabled || false;
36+
return strategy;
37+
});
38+
39+
strategies = Object.fromEntries(strat.map(strategy => [strategy.id, strategy]));
40+
}
41+
42+
export async function run() {
43+
while (!shouldStop) {
44+
try {
45+
log.info('[strategies] Start strategies refresh');
46+
await loadStrategies();
47+
consecutiveFailsCount = 0;
48+
log.info('[strategies] End strategies refresh');
49+
} catch (e: any) {
50+
consecutiveFailsCount++;
51+
52+
if (consecutiveFailsCount >= 3) {
53+
capture(e);
54+
}
55+
log.error(`[strategies] failed to load ${JSON.stringify(e)}`);
56+
}
57+
await snapshot.utils.sleep(RUN_INTERVAL);
58+
}
59+
}
60+
61+
export function stop() {
62+
shouldStop = true;
63+
}

‎src/helpers/validation.ts

Lines changed: 80 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,80 @@
1+
import snapshot from '@snapshot-labs/snapshot.js';
2+
import log from './log';
3+
import { strategies } from './strategies';
4+
5+
const DEFAULT_SNAPSHOT_ENV: string = 'testnet';
6+
7+
export async function validateSpaceSettings(
8+
originalSpace: any,
9+
snapshotEnv = DEFAULT_SNAPSHOT_ENV
10+
): Promise<void> {
11+
const spaceType = originalSpace.turbo ? 'turbo' : 'default';
12+
const space = snapshot.utils.clone(originalSpace);
13+
14+
if (space?.deleted) return Promise.reject('space deleted, contact admin');
15+
16+
delete space.deleted;
17+
delete space.flagged;
18+
delete space.verified;
19+
delete space.turbo;
20+
delete space.hibernated;
21+
delete space.id;
22+
23+
if (space.parent && space.parent === originalSpace.id) {
24+
return Promise.reject('space cannot be its own parent');
25+
}
26+
27+
if (
28+
space.children &&
29+
Array.isArray(space.children) &&
30+
space.children.includes(originalSpace.id)
31+
) {
32+
return Promise.reject('space cannot be its own child');
33+
}
34+
35+
const schemaIsValid: any = snapshot.utils.validateSchema(snapshot.schemas.space, space, {
36+
spaceType,
37+
snapshotEnv
38+
});
39+
40+
if (schemaIsValid !== true) {
41+
log.warn('[writer] Wrong space format', schemaIsValid);
42+
const firstErrorObject: any = Object.values(schemaIsValid)[0];
43+
if (firstErrorObject.message === 'network not allowed') {
44+
return Promise.reject(firstErrorObject.message);
45+
}
46+
return Promise.reject('wrong space format');
47+
}
48+
49+
const strategiesIds: string[] = space.strategies.map((strategy: any) => strategy.name);
50+
if (snapshotEnv !== 'testnet') {
51+
const hasTicket = strategiesIds.includes('ticket');
52+
const hasVotingValidation =
53+
space.voteValidation?.name && !['any'].includes(space.voteValidation.name);
54+
55+
if (hasTicket && !hasVotingValidation) {
56+
return Promise.reject('space with ticket requires voting validation');
57+
}
58+
59+
const hasProposalValidation =
60+
(space.validation?.name && space.validation.name !== 'any') ||
61+
space.filters?.minScore ||
62+
space.filters?.onlyMembers;
63+
64+
if (!hasProposalValidation) {
65+
return Promise.reject('space missing proposal validation');
66+
}
67+
}
68+
69+
for (const id of strategiesIds) {
70+
const strategy = strategies[id];
71+
72+
if (!strategy) {
73+
return Promise.reject(`strategy "${id}" is not a valid strategy`);
74+
}
75+
76+
if (strategy.disabled) {
77+
return Promise.reject(`strategy "${id}" has been deprecated`);
78+
}
79+
}
80+
}

‎src/index.ts

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,12 +8,14 @@ import initMetrics from './helpers/metrics';
88
import refreshModeration from './helpers/moderation';
99
import rateLimit from './helpers/rateLimit';
1010
import shutter from './helpers/shutter';
11+
import { run as refreshStrategies } from './helpers/strategies';
1112
import { trackTurboStatuses } from './helpers/turbo';
1213

1314
const app = express();
1415

1516
initLogger(app);
1617
refreshModeration();
18+
refreshStrategies();
1719
initMetrics(app);
1820
trackTurboStatuses();
1921

‎src/writer/proposal.ts

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2,14 +2,14 @@ import { capture } from '@snapshot-labs/snapshot-sentry';
22
import snapshot from '@snapshot-labs/snapshot.js';
33
import networks from '@snapshot-labs/snapshot.js/src/networks.json';
44
import { uniq } from 'lodash';
5-
import { validateSpaceSettings } from './settings';
65
import { getPremiumNetworkIds, getSpace } from '../helpers/actions';
76
import log from '../helpers/log';
87
import { containsFlaggedLinks, flaggedAddresses } from '../helpers/moderation';
98
import { isMalicious } from '../helpers/monitoring';
109
import db from '../helpers/mysql';
1110
import { getLimits, getSpaceType } from '../helpers/options';
1211
import { captureError, getQuorum, jsonParse, validateChoices } from '../helpers/utils';
12+
import { validateSpaceSettings } from '../helpers/validation';
1313

1414
const scoreAPIUrl = process.env.SCORE_API_URL || 'https://score.snapshot.org';
1515
const broviderUrl = process.env.BROVIDER_URL || 'https://rpc.snapshot.org';
@@ -66,7 +66,7 @@ async function validateSpace(space: any) {
6666
}
6767

6868
try {
69-
await validateSpaceSettings(space);
69+
await validateSpaceSettings(space, process.env.NETWORK);
7070
} catch (e) {
7171
return Promise.reject(e);
7272
}

‎src/writer/settings.ts

Lines changed: 10 additions & 88 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,4 @@
11
import { capture } from '@snapshot-labs/snapshot-sentry';
2-
import snapshot from '@snapshot-labs/snapshot.js';
32
import isEqual from 'lodash/isEqual';
43
import { addOrUpdateSpace, getSpace } from '../helpers/actions';
54
import log from '../helpers/log';
@@ -8,73 +7,13 @@ import { getLimit, getSpaceType } from '../helpers/options';
87
import {
98
addToWalletConnectWhitelist,
109
clearStampCache,
11-
fetchWithKeepAlive,
1210
getSpaceController,
1311
jsonParse,
1412
removeFromWalletConnectWhitelist
1513
} from '../helpers/utils';
14+
import { validateSpaceSettings } from '../helpers/validation';
1615

1716
const SNAPSHOT_ENV = process.env.NETWORK || 'testnet';
18-
const scoreAPIUrl = process.env.SCORE_API_URL || 'https://score.snapshot.org';
19-
20-
export async function validateSpaceSettings(originalSpace: any) {
21-
const spaceType = originalSpace.turbo ? 'turbo' : 'default';
22-
const space = snapshot.utils.clone(originalSpace);
23-
24-
if (space?.deleted) return Promise.reject('space deleted, contact admin');
25-
26-
delete space.deleted;
27-
delete space.flagged;
28-
delete space.verified;
29-
delete space.turbo;
30-
delete space.hibernated;
31-
delete space.id;
32-
33-
if (space.parent && space.parent === originalSpace.id) {
34-
return Promise.reject('space cannot be its own parent');
35-
}
36-
37-
if (
38-
space.children &&
39-
Array.isArray(space.children) &&
40-
space.children.includes(originalSpace.id)
41-
) {
42-
return Promise.reject('space cannot be its own child');
43-
}
44-
45-
const schemaIsValid: any = snapshot.utils.validateSchema(snapshot.schemas.space, space, {
46-
spaceType,
47-
snapshotEnv: SNAPSHOT_ENV
48-
});
49-
50-
if (schemaIsValid !== true) {
51-
log.warn('[writer] Wrong space format', schemaIsValid);
52-
const firstErrorObject: any = Object.values(schemaIsValid)[0];
53-
if (firstErrorObject.message === 'network not allowed') {
54-
return Promise.reject(firstErrorObject.message);
55-
}
56-
return Promise.reject('wrong space format');
57-
}
58-
59-
if (SNAPSHOT_ENV !== 'testnet') {
60-
const hasTicket = space.strategies.some(strategy => strategy.name === 'ticket');
61-
const hasVotingValidation =
62-
space.voteValidation?.name && !['any'].includes(space.voteValidation.name);
63-
64-
if (hasTicket && !hasVotingValidation) {
65-
return Promise.reject('space with ticket requires voting validation');
66-
}
67-
68-
const hasProposalValidation =
69-
(space.validation?.name && space.validation.name !== 'any') ||
70-
space.filters?.minScore ||
71-
space.filters?.onlyMembers;
72-
73-
if (!hasProposalValidation) {
74-
return Promise.reject('space missing proposal validation');
75-
}
76-
}
77-
}
7817

7918
export async function verify(body): Promise<any> {
8019
const msg = jsonParse(body.msg);
@@ -84,12 +23,15 @@ export async function verify(body): Promise<any> {
8423
const space = await getSpace(msg.space, true);
8524

8625
try {
87-
await validateSpaceSettings({
88-
...msg.payload,
89-
id: msg.space,
90-
deleted: space?.deleted,
91-
turbo: space?.turbo
92-
});
26+
await validateSpaceSettings(
27+
{
28+
...msg.payload,
29+
id: msg.space,
30+
deleted: space?.deleted,
31+
turbo: space?.turbo
32+
},
33+
process.env.NETWORK
34+
);
9335
} catch (e) {
9436
return Promise.reject(e);
9537
}
@@ -100,26 +42,6 @@ export async function verify(body): Promise<any> {
10042
return Promise.reject(`max number of strategies is ${strategiesLimit}`);
10143
}
10244

103-
try {
104-
const strategiesList = await (await fetchWithKeepAlive(`${scoreAPIUrl}/api/strategies`)).json();
105-
106-
msg.payload.strategies
107-
.map(strategy => strategy.name)
108-
.forEach(strategyName => {
109-
const strategy = strategiesList[strategyName];
110-
111-
if (!strategy) {
112-
return Promise.reject(`strategy "${strategyName}" is not a valid strategy`);
113-
}
114-
115-
if (strategy.disabled) {
116-
return Promise.reject(`strategy "${strategyName}" has been deprecated`);
117-
}
118-
});
119-
} catch (e) {
120-
return Promise.reject('failed to validate strategies');
121-
}
122-
12345
const controller = await getSpaceController(msg.space, SNAPSHOT_ENV);
12446
const isController = controller === body.address;
12547

‎test/integration/ingestor.test.ts

Lines changed: 11 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@ import cloneDeep from 'lodash/cloneDeep';
22
import omit from 'lodash/omit';
33
import db, { sequencerDB } from '../../src/helpers/mysql';
44
import relayer from '../../src/helpers/relayer';
5+
import { run, stop } from '../../src/helpers/strategies';
56
import ingestor from '../../src/ingestor';
67
import proposalInput from '../fixtures/ingestor-payload/proposal.json';
78
import voteInput from '../fixtures/ingestor-payload/vote.json';
@@ -37,6 +38,7 @@ const LIMITS = {
3738
'user.default.follow.limit': 25
3839
};
3940
const ECOSYSTEM_LIST = ['test.eth', 'snapshot.eth'];
41+
4042
jest.mock('../../src/helpers/options', () => {
4143
const originalModule = jest.requireActual('../../src/helpers/options');
4244

@@ -129,7 +131,13 @@ function cloneWithNewMessage(data: Record<string, any>) {
129131
}
130132

131133
describe('ingestor', () => {
132-
beforeAll(() => {
134+
beforeAll(async () => {
135+
// Start the strategies loader (runs in background)
136+
run();
137+
138+
// Wait a bit for the first load to complete
139+
await new Promise(resolve => setTimeout(resolve, 2000));
140+
133141
proposalInput.data.message.timestamp = Math.floor(Date.now() / 1e3) - 60;
134142
proposalInput.data.message.end = Math.floor(Date.now() / 1e3) + 60;
135143
voteInput.data.message.timestamp = Math.floor(Date.now() / 1e3) - 60;
@@ -142,6 +150,8 @@ describe('ingestor', () => {
142150
});
143151

144152
afterAll(async () => {
153+
// Stop any running strategies loader
154+
stop();
145155
await db.queryAsync('DELETE FROM snapshot_sequencer_test.proposals;');
146156
await db.queryAsync('DELETE FROM snapshot_sequencer_test.messages;');
147157
await db.endAsync();

0 commit comments

Comments
 (0)