Skip to content

Commit da627e1

Browse files
LukasDecoLukasDeco
authored
feat: batch accounts for less connects grpc drift client subscriber (#1922)
* feat: batch accounts for less connects grpc drift client subscriber * fix: lint issues and package upgrades * feat: group oracle accounts under 1 grpc connection * feat: separate multi grpc subscriber into v2 and test script * fix: broken test script * fix: unhandled resub logic * fix: lint and prettify * fix: more formatting
1 parent d56a23a commit da627e1

File tree

11 files changed

+794
-37
lines changed

11 files changed

+794
-37
lines changed

‎sdk/bun.lock‎

Lines changed: 25 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@
77
"@coral-xyz/anchor": "0.29.0",
88
"@coral-xyz/anchor-30": "npm:@coral-xyz/[email protected]",
99
"@ellipsis-labs/phoenix-sdk": "1.4.5",
10-
"@grpc/grpc-js": "1.12.6",
10+
"@grpc/grpc-js": "1.14.0",
1111
"@openbook-dex/openbook-v2": "0.2.10",
1212
"@project-serum/serum": "0.13.65",
1313
"@pythnetwork/client": "2.5.3",
@@ -17,7 +17,7 @@
1717
"@solana/web3.js": "1.98.0",
1818
"@switchboard-xyz/common": "3.0.14",
1919
"@switchboard-xyz/on-demand": "2.4.1",
20-
"@triton-one/yellowstone-grpc": "1.3.0",
20+
"@triton-one/yellowstone-grpc": "1.4.1",
2121
"anchor-bankrun": "0.3.0",
2222
"gill": "^0.10.2",
2323
"helius-laserstream": "0.1.8",
@@ -59,15 +59,24 @@
5959
},
6060
},
6161
"overrides": {
62-
"debug": "<4.4.2",
63-
"supports-color": "7.2.0",
6462
"ansi-regex": "5.0.1",
65-
"color-convert": "<3.1.1",
6663
"ansi-styles": "4.3.0",
67-
"wrap-ansi": "7.0.0",
64+
"backslash": "<0.2.1",
6865
"chalk": "4.1.2",
69-
"strip-ansi": "6.0.1",
66+
"chalk-template": "<1.1.1",
67+
"color-convert": "<3.1.1",
7068
"color-name": "<2.0.1",
69+
"color-string": "<2.1.1",
70+
"debug": "<4.4.2",
71+
"error-ex": "<1.3.3",
72+
"has-ansi": "<6.0.1",
73+
"is-arrayish": "<0.3.3",
74+
"simple-swizzle": "<0.2.3",
75+
"slice-ansi": "3.0.0",
76+
"strip-ansi": "6.0.1",
77+
"supports-color": "7.2.0",
78+
"supports-hyperlinks": "<4.1.1",
79+
"wrap-ansi": "7.0.0",
7180
},
7281
"packages": {
7382
"@babel/code-frame": ["@babel/[email protected]", "", { "dependencies": { "@babel/helper-validator-identifier": "^7.25.9", "js-tokens": "^4.0.0", "picocolors": "^1.0.0" } }, "sha512-RJlIHRueQgwWitWgF8OdFYGZX328Ax5BCemNGlqHfplnRT9ESi8JkFlvaVYbS+UubVY6dpv87Fs2u5M29iNFVQ=="],
@@ -104,9 +113,9 @@
104113

105114
"@eslint/js": ["@eslint/[email protected]", "", {}, "sha512-Ys+3g2TaW7gADOJzPt83SJtCDhMjndcDMFVQ/Tj9iA1BfJzFKD9mAUXT3OenpuPHbI6P/myECxRJrofUsDx/5g=="],
106115

107-
"@grpc/grpc-js": ["@grpc/grpc-js@1.12.6", "", { "dependencies": { "@grpc/proto-loader": "^0.7.13", "@js-sdsl/ordered-map": "^4.4.2" } }, "sha512-JXUj6PI0oqqzTGvKtzOkxtpsyPRNsrmhh41TtIz/zEB6J+AUiZZ0dxWzcMwO9Ns5rmSPuMdghlTbUuqIM48d3Q=="],
116+
"@grpc/grpc-js": ["@grpc/grpc-js@1.14.0", "", { "dependencies": { "@grpc/proto-loader": "^0.8.0", "@js-sdsl/ordered-map": "^4.4.2" } }, "sha512-N8Jx6PaYzcTRNzirReJCtADVoq4z7+1KQ4E70jTg/koQiMoUSN1kbNjPOqpPbhMFhfU1/l7ixspPl8dNY+FoUg=="],
108117

109-
"@grpc/proto-loader": ["@grpc/proto-loader@0.7.13", "", { "dependencies": { "lodash.camelcase": "^4.3.0", "long": "^5.0.0", "protobufjs": "^7.2.5", "yargs": "^17.7.2" }, "bin": { "proto-loader-gen-types": "build/bin/proto-loader-gen-types.js" } }, "sha512-AiXO/bfe9bmxBjxxtYxFAXGZvMaN5s8kO+jBHAJCON8rJoB5YS/D6X7ZNc6XQkuHNmyl4CYaMI1fJ/Gn27RGGw=="],
118+
"@grpc/proto-loader": ["@grpc/proto-loader@0.8.0", "", { "dependencies": { "lodash.camelcase": "^4.3.0", "long": "^5.0.0", "protobufjs": "^7.5.3", "yargs": "^17.7.2" }, "bin": { "proto-loader-gen-types": "build/bin/proto-loader-gen-types.js" } }, "sha512-rc1hOQtjIWGxcxpb9aHAfLpIctjEnsDehj0DAiVfBlmT84uvR0uUtN2hEi/ecvWVjXUGf5qPF4qEgiLOx1YIMQ=="],
110119

111120
"@humanwhocodes/config-array": ["@humanwhocodes/[email protected]", "", { "dependencies": { "@humanwhocodes/object-schema": "^2.0.2", "debug": "^4.3.1", "minimatch": "^3.0.5" } }, "sha512-3T8LkOmg45BV5FICb15QQMsyUSWrQ8AygVfC7ZG32zOalnqrilm018ZVCw0eapXux8FtA33q8PSRSstjee3jSg=="],
112121

@@ -294,7 +303,7 @@
294303

295304
"@switchboard-xyz/on-demand": ["@switchboard-xyz/[email protected]", "", { "dependencies": { "@coral-xyz/anchor-30": "npm:@coral-xyz/[email protected]", "@isaacs/ttlcache": "^1.4.1", "@switchboard-xyz/common": ">=3.0.0", "axios": "^1.8.3", "bs58": "^6.0.0", "buffer": "^6.0.3", "js-yaml": "^4.1.0" } }, "sha512-eSlBp+c8lxpcSgh0/2xK8OaLHPziTSZlcs8V96gZGdiCJz1KgWJRNE1qnIJDOwaGdFecZdwcmajfQRtLRLED3w=="],
296305

297-
"@triton-one/yellowstone-grpc": ["@triton-one/yellowstone-grpc@1.3.0", "", { "dependencies": { "@grpc/grpc-js": "^1.8.0" } }, "sha512-tuwHtoYzvqnahsMrecfNNkQceCYwgiY0qKS8RwqtaxvDEgjm0E+0bXwKz2eUD3ZFYifomJmRKDmSBx9yQzAeMQ=="],
306+
"@triton-one/yellowstone-grpc": ["@triton-one/yellowstone-grpc@1.4.1", "", { "dependencies": { "@grpc/grpc-js": "^1.8.0" } }, "sha512-ZN49vooxFbOqWttll8u7AOsIVnX+srqX9ddhZ9ttE+OcehUo8c2p2suK8Gr2puab49cgsV0VGjiTn9Gua/ntIw=="],
298307

299308
"@ts-graphviz/adapter": ["@ts-graphviz/[email protected]", "", { "dependencies": { "@ts-graphviz/common": "^2.1.5" } }, "sha512-kJ10lIMSWMJkLkkCG5gt927SnGZcBuG0s0HHswGzcHTgvtUe7yk5/3zTEr0bafzsodsOq5Gi6FhQeV775nC35Q=="],
300309

@@ -1176,6 +1185,8 @@
11761185

11771186
"@ellipsis-labs/phoenix-sdk/bs58": ["[email protected]", "", { "dependencies": { "base-x": "^4.0.0" } }, "sha512-r+ihvQJvahgYT50JD05dyJNKlmmSlMoOGwn1lCcEzanPglg7TxYjioQUYehQ9mAR/+hOSd2jRc/Z2y5UxBymvQ=="],
11781187

1188+
"@grpc/proto-loader/protobufjs": ["[email protected]", "", { "dependencies": { "@protobufjs/aspromise": "^1.1.2", "@protobufjs/base64": "^1.1.2", "@protobufjs/codegen": "^2.0.4", "@protobufjs/eventemitter": "^1.1.0", "@protobufjs/fetch": "^1.1.0", "@protobufjs/float": "^1.0.2", "@protobufjs/inquire": "^1.1.0", "@protobufjs/path": "^1.1.2", "@protobufjs/pool": "^1.1.0", "@protobufjs/utf8": "^1.1.0", "@types/node": ">=13.7.0", "long": "^5.0.0" } }, "sha512-CvexbZtbov6jW2eXAvLukXjXUW1TzFaivC46BpWc/3BpcCysb5Vffu+B3XHMm8lVEuy2Mm4XGex8hBSg1yapPg=="],
1189+
11791190
"@metaplex-foundation/beet-solana/bs58": ["[email protected]", "", { "dependencies": { "base-x": "^4.0.0" } }, "sha512-r+ihvQJvahgYT50JD05dyJNKlmmSlMoOGwn1lCcEzanPglg7TxYjioQUYehQ9mAR/+hOSd2jRc/Z2y5UxBymvQ=="],
11801191

11811192
"@metaplex-foundation/solita/@metaplex-foundation/beet": ["@metaplex-foundation/[email protected]", "", { "dependencies": { "ansicolors": "^0.3.2", "bn.js": "^5.2.0", "debug": "^4.3.3" } }, "sha512-2OAKJnLatCc3mBXNL0QmWVQKAWK2C7XDfepgL0p/9+8oSx4bmRAFHFqptl1A/C0U5O3dxGwKfmKluW161OVGcA=="],
@@ -1248,6 +1259,8 @@
12481259

12491260
"jayson/ws": ["[email protected]", "", { "peerDependencies": { "bufferutil": "^4.0.1", "utf-8-validate": "^5.0.2" }, "optionalPeers": ["bufferutil", "utf-8-validate"] }, "sha512-+dbF1tHwZpXcbOJdVOkzLDxZP1ailvSxM6ZweXTegylPny803bFhA+vqBYw4s31NSAk4S2Qz+AKXK9a4wkdjcQ=="],
12501261

1262+
"jito-ts/@grpc/grpc-js": ["@grpc/[email protected]", "", { "dependencies": { "@grpc/proto-loader": "^0.7.13", "@js-sdsl/ordered-map": "^4.4.2" } }, "sha512-JXUj6PI0oqqzTGvKtzOkxtpsyPRNsrmhh41TtIz/zEB6J+AUiZZ0dxWzcMwO9Ns5rmSPuMdghlTbUuqIM48d3Q=="],
1263+
12511264
"jito-ts/@solana/web3.js": ["@solana/[email protected]", "", { "dependencies": { "@babel/runtime": "^7.12.5", "@noble/curves": "^1.0.0", "@noble/hashes": "^1.3.0", "@solana/buffer-layout": "^4.0.0", "agentkeepalive": "^4.2.1", "bigint-buffer": "^1.1.5", "bn.js": "^5.0.0", "borsh": "^0.7.0", "bs58": "^4.0.1", "buffer": "6.0.3", "fast-stable-stringify": "^1.0.0", "jayson": "^4.1.0", "node-fetch": "^2.6.7", "rpc-websockets": "^7.5.1", "superstruct": "^0.14.2" } }, "sha512-XdN0Lh4jdY7J8FYMyucxCwzn6Ga2Sr1DHDWRbqVzk7ZPmmpSPOVWHzO67X1cVT+jNi1D6gZi2tgjHgDPuj6e9Q=="],
12521265

12531266
"jito-ts/dotenv": ["[email protected]", "", {}, "sha512-47qPchRCykZC03FhkYAhrvwU4xDBFIj1QPqaarj6mdM/hgUzfPHcpkHJOn3mJAufFeeAxAzeGsr5X0M4k6fLZQ=="],
@@ -1332,6 +1345,8 @@
13321345

13331346
"glob/minimatch/brace-expansion": ["[email protected]", "", { "dependencies": { "balanced-match": "^1.0.0" } }, "sha512-XnAIvQ8eM+kC6aULx6wuQiwVsnzsi9d3WxzV3FpWTGA19F621kwdbsAcFKXgKUHZWsy+mY6iL1sHTxWEFCytDA=="],
13341347

1348+
"jito-ts/@grpc/grpc-js/@grpc/proto-loader": ["@grpc/[email protected]", "", { "dependencies": { "lodash.camelcase": "^4.3.0", "long": "^5.0.0", "protobufjs": "^7.2.5", "yargs": "^17.7.2" }, "bin": { "proto-loader-gen-types": "build/bin/proto-loader-gen-types.js" } }, "sha512-AiXO/bfe9bmxBjxxtYxFAXGZvMaN5s8kO+jBHAJCON8rJoB5YS/D6X7ZNc6XQkuHNmyl4CYaMI1fJ/Gn27RGGw=="],
1349+
13351350
"jito-ts/@solana/web3.js/superstruct": ["[email protected]", "", {}, "sha512-nPewA6m9mR3d6k7WkZ8N8zpTWfenFH3q9pA2PkuiZxINr9DKB2+40wEQf0ixn8VaGuJ78AB6iWOtStI+/4FKZQ=="],
13361351

13371352
"mocha/minimatch/brace-expansion": ["[email protected]", "", { "dependencies": { "balanced-match": "^1.0.0" } }, "sha512-XnAIvQ8eM+kC6aULx6wuQiwVsnzsi9d3WxzV3FpWTGA19F621kwdbsAcFKXgKUHZWsy+mY6iL1sHTxWEFCytDA=="],

‎sdk/package.json‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -42,7 +42,7 @@
4242
"@coral-xyz/anchor": "0.29.0",
4343
"@coral-xyz/anchor-30": "npm:@coral-xyz/[email protected]",
4444
"@ellipsis-labs/phoenix-sdk": "1.4.5",
45-
"@grpc/grpc-js": "1.12.6",
45+
"@grpc/grpc-js": "1.14.0",
4646
"@openbook-dex/openbook-v2": "0.2.10",
4747
"@project-serum/serum": "0.13.65",
4848
"@pythnetwork/client": "2.5.3",
@@ -52,7 +52,7 @@
5252
"@solana/web3.js": "1.98.0",
5353
"@switchboard-xyz/common": "3.0.14",
5454
"@switchboard-xyz/on-demand": "2.4.1",
55-
"@triton-one/yellowstone-grpc": "1.3.0",
55+
"@triton-one/yellowstone-grpc": "1.4.1",
5656
"anchor-bankrun": "0.3.0",
5757
"gill": "^0.10.2",
5858
"helius-laserstream": "0.1.8",

‎sdk/scripts/client-test.ts‎

Lines changed: 87 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,87 @@
1+
import { DriftClient } from '../src/driftClient';
2+
import { grpcDriftClientAccountSubscriberV2 } from '../src/accounts/grpcDriftClientAccountSubscriberV2';
3+
import { Connection, Keypair, PublicKey } from '@solana/web3.js';
4+
import { DriftClientConfig } from '../src/driftClientConfig';
5+
import { decodeName, DRIFT_PROGRAM_ID, Wallet } from '../src';
6+
import { CommitmentLevel } from '@triton-one/yellowstone-grpc';
7+
import dotenv from 'dotenv';
8+
9+
const GRPC_ENDPOINT = process.env.GRPC_ENDPOINT;
10+
const TOKEN = process.env.TOKEN;
11+
12+
async function initializeGrpcDriftClientV2() {
13+
const connection = new Connection('https://api.mainnet-beta.solana.com');
14+
const wallet = new Wallet(new Keypair());
15+
dotenv.config({ path: '../' });
16+
const config: DriftClientConfig = {
17+
connection,
18+
wallet,
19+
programID: new PublicKey(DRIFT_PROGRAM_ID),
20+
accountSubscription: {
21+
type: 'grpc',
22+
grpcConfigs: {
23+
endpoint: GRPC_ENDPOINT,
24+
token: TOKEN,
25+
commitmentLevel: 'confirmed' as unknown as CommitmentLevel,
26+
channelOptions: {
27+
'grpc.keepalive_time_ms': 10_000,
28+
'grpc.keepalive_timeout_ms': 1_000,
29+
'grpc.keepalive_permit_without_calls': 1,
30+
},
31+
},
32+
driftClientAccountSubscriber: grpcDriftClientAccountSubscriberV2,
33+
},
34+
perpMarketIndexes: [0, 1, 2], // Example market indexes
35+
spotMarketIndexes: [0, 1, 2], // Example market indexes
36+
oracleInfos: [], // Add oracle information if needed
37+
};
38+
39+
const driftClient = new DriftClient(config);
40+
41+
let perpMarketUpdateCount = 0;
42+
let spotMarketUpdateCount = 0;
43+
let oraclePriceUpdateCount = 0;
44+
let userAccountUpdateCount = 0;
45+
46+
const updatePromise = new Promise<void>((resolve) => {
47+
driftClient.accountSubscriber.eventEmitter.on('perpMarketAccountUpdate', (data) => {
48+
console.log('Perp market account update:', decodeName(data.name));
49+
perpMarketUpdateCount++;
50+
if (perpMarketUpdateCount >= 10 && spotMarketUpdateCount >= 10 && oraclePriceUpdateCount >= 10 && userAccountUpdateCount >= 2) {
51+
resolve();
52+
}
53+
});
54+
55+
driftClient.accountSubscriber.eventEmitter.on('spotMarketAccountUpdate', (data) => {
56+
console.log('Spot market account update:', decodeName(data.name));
57+
spotMarketUpdateCount++;
58+
if (perpMarketUpdateCount >= 10 && spotMarketUpdateCount >= 10 && oraclePriceUpdateCount >= 10 && userAccountUpdateCount >= 2) {
59+
resolve();
60+
}
61+
});
62+
63+
driftClient.accountSubscriber.eventEmitter.on('oraclePriceUpdate', (data) => {
64+
console.log('Oracle price update:', data.toBase58());
65+
oraclePriceUpdateCount++;
66+
if (perpMarketUpdateCount >= 10 && spotMarketUpdateCount >= 10 && oraclePriceUpdateCount >= 10 && userAccountUpdateCount >= 2) {
67+
resolve();
68+
}
69+
});
70+
71+
driftClient.accountSubscriber.eventEmitter.on('userAccountUpdate', (data) => {
72+
console.log('User account update:', decodeName(data.name));
73+
userAccountUpdateCount++;
74+
if (perpMarketUpdateCount >= 10 && spotMarketUpdateCount >= 10 && oraclePriceUpdateCount >= 10 && userAccountUpdateCount >= 2) {
75+
resolve();
76+
}
77+
});
78+
});
79+
80+
await driftClient.subscribe();
81+
console.log('DriftClient initialized and listening for updates.');
82+
83+
await updatePromise;
84+
console.log('Received required number of updates.');
85+
}
86+
87+
initializeGrpcDriftClientV2().catch(console.error);

‎sdk/src/accounts/grpcAccountSubscriber.ts‎

Lines changed: 9 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -39,13 +39,16 @@ export class grpcAccountSubscriber<T> extends WebSocketAccountSubscriber<T> {
3939
program: Program,
4040
accountPublicKey: PublicKey,
4141
decodeBuffer?: (buffer: Buffer) => U,
42-
resubOpts?: ResubOpts
42+
resubOpts?: ResubOpts,
43+
clientProp?: Client
4344
): Promise<grpcAccountSubscriber<U>> {
44-
const client = await createClient(
45-
grpcConfigs.endpoint,
46-
grpcConfigs.token,
47-
grpcConfigs.channelOptions ?? {}
48-
);
45+
const client = clientProp
46+
? clientProp
47+
: await createClient(
48+
grpcConfigs.endpoint,
49+
grpcConfigs.token,
50+
grpcConfigs.channelOptions ?? {}
51+
);
4952
const commitmentLevel =
5053
// @ts-ignore :: isomorphic exported enum fails typescript but will work at runtime
5154
grpcConfigs.commitmentLevel ?? CommitmentLevel.CONFIRMED;

‎sdk/src/accounts/grpcDriftClientAccountSubscriber.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@ import { grpcAccountSubscriber } from './grpcAccountSubscriber';
1212
import { PerpMarketAccount, SpotMarketAccount, StateAccount } from '../types';
1313
import { getOracleId } from '../oracles/oracleId';
1414

15-
export class gprcDriftClientAccountSubscriber extends WebSocketDriftClientAccountSubscriber {
15+
export class grpcDriftClientAccountSubscriber extends WebSocketDriftClientAccountSubscriber {
1616
private grpcConfigs: GrpcConfigs;
1717

1818
constructor(

0 commit comments

Comments
 (0)