feat: fix fetch delegates error when skip value is greater than 5000 (#956)

* feat: fix fetch delegates error when `skip` value is greater than 5000

* refactor: improve code readability and testing

* fix: handle case where there are more than 1000 duplicates

* fix: raise error on 1000 duplicates instead

* refactor: remove params

* Update src/utils/delegation.ts

Co-authored-by: Chaitanya <[email protected]>

* Update src/utils/delegation.ts

Co-authored-by: Chaitanya <[email protected]>

* refactor: use map for unique delegation handling

* chore: add more test

* Update src/utils/delegation.ts

* Update src/utils/delegation.ts

* v0.11.0

---------

Co-authored-by: Chaitanya <[email protected]>

Author: wa0x6e

package.json

@@ -1,6 +1,6 @@
 {
   "name": "@snapshot-labs/snapshot.js",
-  "version": "0.10.1",
+  "version": "0.11.0",
   "repository": "snapshot-labs/snapshot.js",
   "license": "MIT",
   "main": "dist/snapshot.cjs.js",

src/utils.ts

@@ -14,8 +14,8 @@ import { signMessage, getBlockNumber } from './utils/web3';
 import { getHash, verify } from './sign/utils';
 import gateways from './gateways.json';
 import networks from './networks.json';
-import delegationSubgraphs from './delegationSubgraphs.json';
 import voting from './voting';
+import getDelegatesBySpace, { SNAPSHOT_SUBGRAPH_URL } from './utils/delegation';
 
 interface Options {
   url?: string;
@@ -27,7 +27,6 @@ interface Strategy {
   params: any;
 }
 
-export const SNAPSHOT_SUBGRAPH_URL = delegationSubgraphs;
 const ENS_RESOLVER_ABI = [
   'function text(bytes32 node, string calldata key) external view returns (string memory)'
 ];
@@ -572,55 +571,6 @@ export async function getSpaceController(
   return await getEnsOwner(id, network, options);
 }
 
-export async function getDelegatesBySpace(
-  network: string,
-  space: string,
-  snapshot = 'latest',
-  options: any = {}
-) {
-  const subgraphUrl = options.subgraphUrl || SNAPSHOT_SUBGRAPH_URL[network];
-  if (!subgraphUrl) {
-    return Promise.reject(
-      `Delegation subgraph not available for network ${network}`
-    );
-  }
-  const spaceIn = ['', space];
-  if (space.includes('.eth')) spaceIn.push(space.replace('.eth', ''));
-
-  const PAGE_SIZE = 1000;
-  let result = [];
-  let page = 0;
-  const params: any = {
-    delegations: {
-      __args: {
-        where: {
-          space_in: spaceIn
-        },
-        first: PAGE_SIZE,
-        skip: 0
-      },
-      delegator: true,
-      space: true,
-      delegate: true
-    }
-  };
-  if (snapshot !== 'latest') {
-    params.delegations.__args.block = { number: snapshot };
-  }
-
-  while (true) {
-    params.delegations.__args.skip = page * PAGE_SIZE;
-
-    const pageResult = await subgraphRequest(subgraphUrl, params);
-    const pageDelegations = pageResult.delegations || [];
-    result = result.concat(pageDelegations);
-    page++;
-    if (pageDelegations.length < PAGE_SIZE) break;
-  }
-
-  return result;
-}
-
 export function clone(item) {
   return JSON.parse(JSON.stringify(item));
 }
@@ -656,6 +606,8 @@ function inputError(message: string) {
   return Promise.reject(new Error(message));
 }
 
+export { getDelegatesBySpace, SNAPSHOT_SUBGRAPH_URL };
+
 export default {
   call,
   multicall,

src/utils/delegation.ts

@@ -0,0 +1,117 @@
+import { subgraphRequest } from '../utils';
+import delegationSubgraphs from '../delegationSubgraphs.json';
+
+export const SNAPSHOT_SUBGRAPH_URL = delegationSubgraphs;
+const PAGE_SIZE = 1000;
+
+type Delegation = {
+  delegator: string;
+  delegate: string;
+  space: string;
+  timestamp: number;
+};
+
+export default async function getDelegatesBySpace(
+  network: string,
+  space: string,
+  snapshot: string | number = 'latest',
+  options: any = {}
+) {
+  const subgraphUrl = options.subgraphUrl || SNAPSHOT_SUBGRAPH_URL[network];
+  if (!subgraphUrl) {
+    return Promise.reject(
+      `Delegation subgraph not available for network ${network}`
+    );
+  }
+
+  let pivot = 0;
+  const result = new Map<string, Delegation>();
+  const spaceIn = buildSpaceIn(space);
+
+  while (true) {
+    const newResults = await fetchData({
+      url: subgraphUrl,
+      spaces: spaceIn,
+      pivot,
+      snapshot
+    });
+
+    if (checkAllDuplicates(newResults)) {
+      throw new Error('Unable to paginate delegation');
+    }
+
+    newResults.forEach((delegation) => {
+      concatUniqueDelegation(result, delegation);
+      pivot = delegation.timestamp;
+    });
+
+    if (newResults.length < PAGE_SIZE) break;
+  }
+
+  return [...result.values()];
+}
+
+function checkAllDuplicates(delegations: Delegation[]) {
+  return (
+    delegations.length === PAGE_SIZE &&
+    delegations[0].timestamp === delegations[delegations.length - 1].timestamp
+  );
+}
+
+function delegationKey(delegation: Delegation) {
+  return `${delegation.delegator}-${delegation.delegate}-${delegation.space}`;
+}
+
+function concatUniqueDelegation(
+  result: Map<string, Delegation>,
+  delegation: Delegation
+): void {
+  const key = delegationKey(delegation);
+  if (!result.has(key)) {
+    result.set(key, delegation);
+  }
+}
+
+function buildSpaceIn(space: string) {
+  const spaces = ['', space];
+  if (space.includes('.eth')) spaces.push(space.replace('.eth', ''));
+
+  return spaces;
+}
+
+async function fetchData({
+  url,
+  spaces,
+  pivot,
+  snapshot
+}: {
+  url: string;
+  spaces: string[];
+  pivot: number;
+  snapshot: string | number;
+}): Promise<Delegation[]> {
+  const params: any = {
+    delegations: {
+      __args: {
+        where: {
+          space_in: spaces,
+          timestamp_gte: pivot
+        },
+        first: PAGE_SIZE,
+        skip: 0,
+        orderBy: 'timestamp',
+        orderDirection: 'asc'
+      },
+      delegator: true,
+      space: true,
+      delegate: true,
+      timestamp: true
+    }
+  };
+
+  if (snapshot !== 'latest') {
+    params.delegations.__args.block = { number: snapshot };
+  }
+
+  return (await subgraphRequest(url, params)).delegations || [];
+}

test/e2e/utils/delegation.spec.ts

@@ -0,0 +1,54 @@
+import { test, expect, describe } from 'vitest';
+import { getDelegatesBySpace } from '../../../src/utils';
+import fixtures from '../../examples/delegates.json';
+
+const SPACE = 'stgdao.eth';
+const NETWORK = '42161';
+const BLOCK_NUMBER = 73900000;
+
+describe('test delegation', () => {
+  describe('getDelegatesBySpace()', () => {
+    test('should return a list of delegations at specified block', async () => {
+      expect.assertions(2);
+      const results: any = await getDelegatesBySpace(
+        NETWORK,
+        SPACE,
+        BLOCK_NUMBER
+      );
+
+      expect(results.length).toEqual(fixtures.length);
+      expect(
+        results.every((a: any) => {
+          return fixtures.some((b) => {
+            return (
+              a.delegator === b.delegator &&
+              a.space === b.space &&
+              a.delegate === b.delegate
+            );
+          });
+        })
+      ).toEqual(true);
+    });
+
+    test('should return an empty array when no results', async () => {
+      expect.assertions(1);
+      const results: any = await getDelegatesBySpace(NETWORK, SPACE, 22531439);
+
+      expect(results.length).toEqual(0);
+    });
+
+    test('should reject with a message on unsupported network', async () => {
+      expect.assertions(1);
+      await expect(
+        getDelegatesBySpace('99999', SPACE, BLOCK_NUMBER)
+      ).to.rejects.toMatch(/not available/);
+    });
+
+    test('should raise error on subgraph graphql errors', async () => {
+      expect.assertions(1);
+      await expect(getDelegatesBySpace('1', SPACE, 0)).to.rejects.toMatch(
+        /startBlock/
+      );
+    });
+  });
+});