fix: simplify retry logic, avoid retrying on top of next push attempt

Author: tejasbadadare

apps/price_pusher/src/solana/command.ts

@@ -232,7 +232,8 @@ export default {
         jitoClients,
         jitoBundleSize,
         updatesPerJitoBundle,
-        60000, // Default max retry time of 60 seconds
+        // Set max retry time to pushing frequency, since we want to stop retrying before the next push attempt
+        pushingFrequency * 1000,
         lookupTableAccount,
       );
 

apps/price_pusher/src/solana/solana.ts

@@ -169,7 +169,7 @@ export class SolanaPricePusherJito implements IPricePusher {
     private searcherClients: SearcherClient[],
     private jitoBundleSize: number,
     private updatesPerJitoBundle: number,
-    private maxRetryTimeMs: number = 60000, // Default to 60 seconds max retry time
+    private maxRetryTimeMs: number,
     private addressLookupTableAccount?: AddressLookupTableAccount,
   ) {}
 
@@ -195,10 +195,6 @@ export class SolanaPricePusherJito implements IPricePusher {
     }
   }
 
-  private async sleep(ms: number): Promise<void> {
-    return new Promise((resolve) => setTimeout(resolve, ms));
-  }
-
   async updatePriceFeed(priceIds: string[]): Promise<void> {
     const recentJitoTip = await this.getRecentJitoTipLamports();
     const jitoTip =
@@ -244,41 +240,15 @@ export class SolanaPricePusherJito implements IPricePusher {
         jitoBundleSize: this.jitoBundleSize,
       });
 
-      try {
-        this.logger.info("Sending Jito transactions...");
-        await sendTransactionsJito(
-          transactions,
-          this.searcherClients,
-          this.pythSolanaReceiver.wallet,
-          { maxRetryTimeMs: this.maxRetryTimeMs },
-        );
-      } catch (err: any) {
-        if (err.code === 8 && err.details?.includes("Rate limit exceeded")) {
-          this.logger.warn("Rate limit hit, waiting before retry...");
-          await this.sleep(1100); // Wait slightly more than 1 second
-          try {
-            this.logger.info("Sending Jito transactions...");
-            await sendTransactionsJito(
-              transactions,
-              this.searcherClients,
-              this.pythSolanaReceiver.wallet,
-              { maxRetryTimeMs: this.maxRetryTimeMs },
-            );
-          } catch (retryErr: any) {
-            this.logger.error("Failed after rate limit retry");
-            throw retryErr;
-          }
-        } else {
-          this.logger.error(
-            { err },
-            "Failed to send transactions via all JITO endpoints",
-          );
-          throw err;
-        }
-      }
-
-      // Add a delay between bundles to avoid rate limiting
-      await this.sleep(1100);
+      await sendTransactionsJito(
+        transactions,
+        this.searcherClients,
+        this.pythSolanaReceiver.wallet,
+        {
+          maxRetryTimeMs: this.maxRetryTimeMs,
+        },
+        this.logger,
+      );
     }
   }
 }

pnpm-lock.yaml

@@ -49,6 +49,7 @@
     "@coral-xyz/anchor": "^0.29.0",
     "@solana/web3.js": "^1.90.0",
     "bs58": "^5.0.0",
-    "jito-ts": "^3.0.1"
+    "jito-ts": "^3.0.1",
+    "ts-log": "^2.2.7"
   }
 }

target_chains/solana/sdk/js/solana_utils/package.json

@@ -1,3 +1,4 @@
+import { dummyLogger, Logger } from "ts-log";
 import { Wallet } from "@coral-xyz/anchor";
 import {
   PublicKey,
@@ -45,8 +46,10 @@ export async function sendTransactionsJito(
   searcherClients: SearcherClient | SearcherClient[],
   wallet: Wallet,
   options: {
-    maxRetryTimeMs?: number;
+    maxRetryTimeMs?: number; // Max time to retry sending transactions
+    delayBetweenCyclesMs?: number; // Delay between cycles of sending transactions to all searcher clients
   } = {},
+  logger: Logger = dummyLogger, // Optional logger to track progress of retries
 ): Promise<string> {
   const clients = Array.isArray(searcherClients)
     ? searcherClients
@@ -57,6 +60,8 @@ export async function sendTransactionsJito(
   }
 
   const maxRetryTimeMs = options.maxRetryTimeMs || 60000; // Default to 60 seconds
+  const delayBetweenCyclesMs = options.delayBetweenCyclesMs || 1000; // Default to 1 second
+
   const startTime = Date.now();
 
   const signedTransactions = [];
@@ -80,22 +85,54 @@ export async function sendTransactionsJito(
   const bundle = new Bundle(signedTransactions, 2);
 
   let lastError: Error | null = null;
-  let clientIndex = 0;
+  let totalAttempts = 0;
 
   while (Date.now() - startTime < maxRetryTimeMs) {
-    const currentClient = clients[clientIndex];
-    try {
-      await currentClient.sendBundle(bundle);
-      return firstTransactionSignature;
-    } catch (err: any) {
-      lastError = err;
-      clientIndex = (clientIndex + 1) % clients.length;
-      await new Promise((resolve) => setTimeout(resolve, 500));
+    // Try all clients in this cycle
+    for (let i = 0; i < clients.length; i++) {
+      const currentClient = clients[i];
+      totalAttempts++;
+
+      try {
+        await currentClient.sendBundle(bundle);
+        logger.info(
+          { clientIndex: i, totalAttempts },
+          `Successfully sent bundle to Jito client after ${totalAttempts} attempts`,
+        );
+        return firstTransactionSignature;
+      } catch (err: any) {
+        lastError = err;
+        logger.error(
+          { clientIndex: i, totalAttempts, err: err.message },
+          `Attempt ${totalAttempts}: Error sending bundle to Jito client ${i}`,
+        );
+      }
+
+      // Check if we've run out of time
+      if (Date.now() - startTime >= maxRetryTimeMs) {
+        break;
+      }
+    }
+
+    // If we've tried all clients and still have time, wait before next cycle
+    const timeRemaining = maxRetryTimeMs - (Date.now() - startTime);
+    if (timeRemaining > delayBetweenCyclesMs) {
+      await new Promise((resolve) => setTimeout(resolve, delayBetweenCyclesMs));
     }
   }
 
-  throw (
-    lastError ||
-    new Error("Failed to send transactions via JITO after maximum retry time")
+  const totalTimeMs = Date.now() - startTime;
+  const errorMsg = `Failed to send transactions via JITO after ${totalAttempts} attempts over ${totalTimeMs}ms (max: ${maxRetryTimeMs}ms)`;
+
+  logger.error(
+    {
+      totalAttempts,
+      totalTimeMs,
+      maxRetryTimeMs,
+      lastError: lastError?.message,
+    },
+    errorMsg,
   );
+
+  throw lastError || new Error(errorMsg);
 }