From c117aa3778863c248e20a9e613d45d6b33279378 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Thu, 8 May 2025 12:19:48 +0000 Subject: [PATCH 1/5] feat: support multiple JITO endpoints with round-robin retry Co-Authored-By: Ali Behjati --- apps/price_pusher/src/solana/command.ts | 17 +++++-- apps/price_pusher/src/solana/solana.ts | 50 +++++++++++-------- .../solana/sdk/js/solana_utils/package.json | 2 +- .../solana/sdk/js/solana_utils/src/jito.ts | 37 ++++++++++++-- 4 files changed, 77 insertions(+), 29 deletions(-) diff --git a/apps/price_pusher/src/solana/command.ts b/apps/price_pusher/src/solana/command.ts index 93420ef79d..e9a56f17b3 100644 --- a/apps/price_pusher/src/solana/command.ts +++ b/apps/price_pusher/src/solana/command.ts @@ -50,7 +50,7 @@ export default { default: 50000, } as Options, "jito-endpoint": { - description: "Jito endpoint", + description: "Jito endpoint(s) - comma-separated list of endpoints", type: "string", optional: true, } as Options, @@ -209,7 +209,13 @@ export default { Uint8Array.from(JSON.parse(fs.readFileSync(jitoKeypairFile, "ascii"))), ); - const jitoClient = searcherClient(jitoEndpoint, jitoKeypair); + const jitoEndpoints = jitoEndpoint + .split(",") + .map((endpoint: string) => endpoint.trim()); + const jitoClients: SearcherClient[] = jitoEndpoints.map( + (endpoint: string) => searcherClient(endpoint, jitoKeypair), + ); + solanaPricePusher = new SolanaPricePusherJito( pythSolanaReceiver, hermesClient, @@ -218,13 +224,16 @@ export default { jitoTipLamports, dynamicJitoTips, maxJitoTipLamports, - jitoClient, + jitoClients, jitoBundleSize, updatesPerJitoBundle, + 60000, // Default max retry time of 60 seconds lookupTableAccount, ); - onBundleResult(jitoClient, logger.child({ module: "JitoClient" })); + jitoClients.forEach((client, index) => { + onBundleResult(client, logger.child({ module: `JitoClient-${index}` })); + }); } else { solanaPricePusher = new SolanaPricePusher( pythSolanaReceiver, diff --git a/apps/price_pusher/src/solana/solana.ts b/apps/price_pusher/src/solana/solana.ts index e907283496..ff64a7fc17 100644 --- a/apps/price_pusher/src/solana/solana.ts +++ b/apps/price_pusher/src/solana/solana.ts @@ -165,9 +165,10 @@ export class SolanaPricePusherJito implements IPricePusher { private defaultJitoTipLamports: number, private dynamicJitoTips: boolean, private maxJitoTipLamports: number, - private searcherClient: SearcherClient, + private searcherClients: SearcherClient[], private jitoBundleSize: number, private updatesPerJitoBundle: number, + private maxRetryTimeMs: number = 60000, // Default to 60 seconds max retry time private addressLookupTableAccount?: AddressLookupTableAccount, ) {} @@ -242,27 +243,34 @@ export class SolanaPricePusherJito implements IPricePusher { jitoBundleSize: this.jitoBundleSize, }); - let retries = 60; - while (retries > 0) { - try { - await sendTransactionsJito( - transactions, - this.searcherClient, - this.pythSolanaReceiver.wallet, - ); - break; - } catch (err: any) { - if (err.code === 8 && err.details?.includes("Rate limit exceeded")) { - this.logger.warn("Rate limit hit, waiting before retry..."); - await this.sleep(1100); // Wait slightly more than 1 second - retries--; - if (retries === 0) { - this.logger.error("Max retries reached for rate limit"); - throw err; - } - } else { - throw err; + try { + await sendTransactionsJito( + transactions, + this.searcherClients, + this.pythSolanaReceiver.wallet, + { maxRetryTimeMs: this.maxRetryTimeMs }, + ); + } catch (err: any) { + if (err.code === 8 && err.details?.includes("Rate limit exceeded")) { + this.logger.warn("Rate limit hit, waiting before retry..."); + await this.sleep(1100); // Wait slightly more than 1 second + try { + await sendTransactionsJito( + transactions, + this.searcherClients, + this.pythSolanaReceiver.wallet, + { maxRetryTimeMs: this.maxRetryTimeMs }, + ); + } catch (retryErr: any) { + this.logger.error("Failed after rate limit retry"); + throw retryErr; } + } else { + this.logger.error( + { err }, + "Failed to send transactions via all JITO endpoints", + ); + throw err; } } diff --git a/target_chains/solana/sdk/js/solana_utils/package.json b/target_chains/solana/sdk/js/solana_utils/package.json index d647e92b53..20fea33bb9 100644 --- a/target_chains/solana/sdk/js/solana_utils/package.json +++ b/target_chains/solana/sdk/js/solana_utils/package.json @@ -1,6 +1,6 @@ { "name": "@pythnetwork/solana-utils", - "version": "0.4.4", + "version": "0.4.5", "description": "Utility functions for Solana", "homepage": "https://pyth.network", "main": "lib/index.js", diff --git a/target_chains/solana/sdk/js/solana_utils/src/jito.ts b/target_chains/solana/sdk/js/solana_utils/src/jito.ts index 2a6a5c290e..5db14c1315 100644 --- a/target_chains/solana/sdk/js/solana_utils/src/jito.ts +++ b/target_chains/solana/sdk/js/solana_utils/src/jito.ts @@ -42,9 +42,23 @@ export async function sendTransactionsJito( tx: VersionedTransaction; signers?: Signer[] | undefined; }[], - searcherClient: SearcherClient, + searcherClients: SearcherClient | SearcherClient[], wallet: Wallet, + options: { + maxRetryTimeMs?: number; + } = {}, ): Promise { + const clients = Array.isArray(searcherClients) + ? searcherClients + : [searcherClients]; + + if (clients.length === 0) { + throw new Error("No searcher clients provided"); + } + + const maxRetryTimeMs = options.maxRetryTimeMs || 60000; // Default to 60 seconds + const startTime = Date.now(); + const signedTransactions = []; for (const transaction of transactions) { @@ -64,7 +78,24 @@ export async function sendTransactionsJito( ); const bundle = new Bundle(signedTransactions, 2); - await searcherClient.sendBundle(bundle); - return firstTransactionSignature; + let lastError: Error | null = null; + let clientIndex = 0; + + while (Date.now() - startTime < maxRetryTimeMs) { + const currentClient = clients[clientIndex]; + try { + await currentClient.sendBundle(bundle); + return firstTransactionSignature; + } catch (err: any) { + lastError = err; + clientIndex = (clientIndex + 1) % clients.length; + await new Promise((resolve) => setTimeout(resolve, 500)); + } + } + + throw ( + lastError || + new Error("Failed to send transactions via JITO after maximum retry time") + ); } From e6d98c2e7e62f822cafc719bf1307e722645baef Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Fri, 9 May 2025 18:23:37 +0000 Subject: [PATCH 2/5] fix: rename jito-endpoint to jito-endpoints Co-Authored-By: Ali Behjati --- apps/price_pusher/src/solana/command.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/apps/price_pusher/src/solana/command.ts b/apps/price_pusher/src/solana/command.ts index e9a56f17b3..0b6353cefa 100644 --- a/apps/price_pusher/src/solana/command.ts +++ b/apps/price_pusher/src/solana/command.ts @@ -49,7 +49,7 @@ export default { type: "number", default: 50000, } as Options, - "jito-endpoint": { + "jito-endpoints": { description: "Jito endpoint(s) - comma-separated list of endpoints", type: "string", optional: true, @@ -117,7 +117,7 @@ export default { pythContractAddress, pushingFrequency, pollingFrequency, - jitoEndpoint, + jitoEndpoints, jitoKeypairFile, jitoTipLamports, dynamicJitoTips, @@ -209,10 +209,10 @@ export default { Uint8Array.from(JSON.parse(fs.readFileSync(jitoKeypairFile, "ascii"))), ); - const jitoEndpoints = jitoEndpoint + const jitoEndpointsList = jitoEndpoints .split(",") .map((endpoint: string) => endpoint.trim()); - const jitoClients: SearcherClient[] = jitoEndpoints.map( + const jitoClients: SearcherClient[] = jitoEndpointsList.map( (endpoint: string) => searcherClient(endpoint, jitoKeypair), ); From 216ac293c40fd356b9a152c5dc3d99c2ca077403 Mon Sep 17 00:00:00 2001 From: Tejas Badadare Date: Mon, 9 Jun 2025 12:29:59 -0700 Subject: [PATCH 3/5] fix: update readme, logs, version --- apps/price_pusher/README.md | 2 +- apps/price_pusher/package.json | 2 +- apps/price_pusher/src/solana/command.ts | 7 ++++++- apps/price_pusher/src/solana/solana.ts | 2 ++ 4 files changed, 10 insertions(+), 3 deletions(-) diff --git a/apps/price_pusher/README.md b/apps/price_pusher/README.md index 882b6a872f..f55bb49b66 100644 --- a/apps/price_pusher/README.md +++ b/apps/price_pusher/README.md @@ -159,7 +159,7 @@ pnpm run start solana \ --endpoint https://api.mainnet-beta.solana.com \ --keypair-file ./id.json \ --shard-id 1 \ - --jito-endpoint mainnet.block-engine.jito.wtf \ + --jito-endpoints mainnet.block-engine.jito.wtf,ny.mainnet.block-engine.jito.wtf \ --jito-keypair-file ./jito.json \ --jito-tip-lamports 100000 \ --jito-bundle-size 5 \ diff --git a/apps/price_pusher/package.json b/apps/price_pusher/package.json index a29842140d..ae9f213d9a 100644 --- a/apps/price_pusher/package.json +++ b/apps/price_pusher/package.json @@ -1,6 +1,6 @@ { "name": "@pythnetwork/price-pusher", - "version": "9.3.3", + "version": "9.3.4", "description": "Pyth Price Pusher", "homepage": "https://pyth.network", "main": "lib/index.js", diff --git a/apps/price_pusher/src/solana/command.ts b/apps/price_pusher/src/solana/command.ts index 0b6353cefa..8f82aad30d 100644 --- a/apps/price_pusher/src/solana/command.ts +++ b/apps/price_pusher/src/solana/command.ts @@ -213,7 +213,12 @@ export default { .split(",") .map((endpoint: string) => endpoint.trim()); const jitoClients: SearcherClient[] = jitoEndpointsList.map( - (endpoint: string) => searcherClient(endpoint, jitoKeypair), + (endpoint: string) => { + logger.info( + `Constructing Jito searcher client from endpoint ${endpoint}`, + ); + return searcherClient(endpoint, jitoKeypair); + }, ); solanaPricePusher = new SolanaPricePusherJito( diff --git a/apps/price_pusher/src/solana/solana.ts b/apps/price_pusher/src/solana/solana.ts index ff64a7fc17..fb1611435e 100644 --- a/apps/price_pusher/src/solana/solana.ts +++ b/apps/price_pusher/src/solana/solana.ts @@ -244,6 +244,7 @@ export class SolanaPricePusherJito implements IPricePusher { }); try { + this.logger.info("Sending Jito transactions..."); await sendTransactionsJito( transactions, this.searcherClients, @@ -255,6 +256,7 @@ export class SolanaPricePusherJito implements IPricePusher { this.logger.warn("Rate limit hit, waiting before retry..."); await this.sleep(1100); // Wait slightly more than 1 second try { + this.logger.info("Sending Jito transactions..."); await sendTransactionsJito( transactions, this.searcherClients, From 02bfef5e878e9bf4068b8f809ba76c3724d7dfde Mon Sep 17 00:00:00 2001 From: Tejas Badadare Date: Wed, 11 Jun 2025 16:41:45 -0700 Subject: [PATCH 4/5] fix: simplify retry logic, avoid retrying on top of next push attempt --- apps/price_pusher/src/solana/command.ts | 3 +- apps/price_pusher/src/solana/solana.ts | 50 +++------------ pnpm-lock.yaml | 3 + .../solana/sdk/js/solana_utils/package.json | 3 +- .../solana/sdk/js/solana_utils/src/jito.ts | 63 +++++++++++++++---- 5 files changed, 67 insertions(+), 55 deletions(-) diff --git a/apps/price_pusher/src/solana/command.ts b/apps/price_pusher/src/solana/command.ts index 8f82aad30d..2196bb50cc 100644 --- a/apps/price_pusher/src/solana/command.ts +++ b/apps/price_pusher/src/solana/command.ts @@ -232,7 +232,8 @@ export default { jitoClients, jitoBundleSize, updatesPerJitoBundle, - 60000, // Default max retry time of 60 seconds + // Set max retry time to pushing frequency, since we want to stop retrying before the next push attempt + pushingFrequency * 1000, lookupTableAccount, ); diff --git a/apps/price_pusher/src/solana/solana.ts b/apps/price_pusher/src/solana/solana.ts index ccfeaaea3f..ef3632ab61 100644 --- a/apps/price_pusher/src/solana/solana.ts +++ b/apps/price_pusher/src/solana/solana.ts @@ -169,7 +169,7 @@ export class SolanaPricePusherJito implements IPricePusher { private searcherClients: SearcherClient[], private jitoBundleSize: number, private updatesPerJitoBundle: number, - private maxRetryTimeMs: number = 60000, // Default to 60 seconds max retry time + private maxRetryTimeMs: number, private addressLookupTableAccount?: AddressLookupTableAccount, ) {} @@ -195,10 +195,6 @@ export class SolanaPricePusherJito implements IPricePusher { } } - private async sleep(ms: number): Promise { - return new Promise((resolve) => setTimeout(resolve, ms)); - } - async updatePriceFeed(priceIds: string[]): Promise { const recentJitoTip = await this.getRecentJitoTipLamports(); const jitoTip = @@ -244,41 +240,15 @@ export class SolanaPricePusherJito implements IPricePusher { jitoBundleSize: this.jitoBundleSize, }); - try { - this.logger.info("Sending Jito transactions..."); - await sendTransactionsJito( - transactions, - this.searcherClients, - this.pythSolanaReceiver.wallet, - { maxRetryTimeMs: this.maxRetryTimeMs }, - ); - } catch (err: any) { - if (err.code === 8 && err.details?.includes("Rate limit exceeded")) { - this.logger.warn("Rate limit hit, waiting before retry..."); - await this.sleep(1100); // Wait slightly more than 1 second - try { - this.logger.info("Sending Jito transactions..."); - await sendTransactionsJito( - transactions, - this.searcherClients, - this.pythSolanaReceiver.wallet, - { maxRetryTimeMs: this.maxRetryTimeMs }, - ); - } catch (retryErr: any) { - this.logger.error("Failed after rate limit retry"); - throw retryErr; - } - } else { - this.logger.error( - { err }, - "Failed to send transactions via all JITO endpoints", - ); - throw err; - } - } - - // Add a delay between bundles to avoid rate limiting - await this.sleep(1100); + await sendTransactionsJito( + transactions, + this.searcherClients, + this.pythSolanaReceiver.wallet, + { + maxRetryTimeMs: this.maxRetryTimeMs, + }, + this.logger, + ); } } } diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index c5fa1d6f77..1e5aca631e 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -2889,6 +2889,9 @@ importers: jito-ts: specifier: ^3.0.1 version: 3.0.1(bufferutil@4.0.9)(encoding@0.1.13)(utf-8-validate@5.0.10) + ts-log: + specifier: ^2.2.7 + version: 2.2.7 devDependencies: '@solana/wallet-adapter-react': specifier: ^0.15.28 diff --git a/target_chains/solana/sdk/js/solana_utils/package.json b/target_chains/solana/sdk/js/solana_utils/package.json index 20fea33bb9..e737e8e24a 100644 --- a/target_chains/solana/sdk/js/solana_utils/package.json +++ b/target_chains/solana/sdk/js/solana_utils/package.json @@ -49,6 +49,7 @@ "@coral-xyz/anchor": "^0.29.0", "@solana/web3.js": "^1.90.0", "bs58": "^5.0.0", - "jito-ts": "^3.0.1" + "jito-ts": "^3.0.1", + "ts-log": "^2.2.7" } } diff --git a/target_chains/solana/sdk/js/solana_utils/src/jito.ts b/target_chains/solana/sdk/js/solana_utils/src/jito.ts index 5db14c1315..12a2762f34 100644 --- a/target_chains/solana/sdk/js/solana_utils/src/jito.ts +++ b/target_chains/solana/sdk/js/solana_utils/src/jito.ts @@ -1,3 +1,4 @@ +import { dummyLogger, Logger } from "ts-log"; import { Wallet } from "@coral-xyz/anchor"; import { PublicKey, @@ -45,8 +46,10 @@ export async function sendTransactionsJito( searcherClients: SearcherClient | SearcherClient[], wallet: Wallet, options: { - maxRetryTimeMs?: number; + maxRetryTimeMs?: number; // Max time to retry sending transactions + delayBetweenCyclesMs?: number; // Delay between cycles of sending transactions to all searcher clients } = {}, + logger: Logger = dummyLogger, // Optional logger to track progress of retries ): Promise { const clients = Array.isArray(searcherClients) ? searcherClients @@ -57,6 +60,8 @@ export async function sendTransactionsJito( } const maxRetryTimeMs = options.maxRetryTimeMs || 60000; // Default to 60 seconds + const delayBetweenCyclesMs = options.delayBetweenCyclesMs || 1000; // Default to 1 second + const startTime = Date.now(); const signedTransactions = []; @@ -80,22 +85,54 @@ export async function sendTransactionsJito( const bundle = new Bundle(signedTransactions, 2); let lastError: Error | null = null; - let clientIndex = 0; + let totalAttempts = 0; while (Date.now() - startTime < maxRetryTimeMs) { - const currentClient = clients[clientIndex]; - try { - await currentClient.sendBundle(bundle); - return firstTransactionSignature; - } catch (err: any) { - lastError = err; - clientIndex = (clientIndex + 1) % clients.length; - await new Promise((resolve) => setTimeout(resolve, 500)); + // Try all clients in this cycle + for (let i = 0; i < clients.length; i++) { + const currentClient = clients[i]; + totalAttempts++; + + try { + await currentClient.sendBundle(bundle); + logger.info( + { clientIndex: i, totalAttempts }, + `Successfully sent bundle to Jito client after ${totalAttempts} attempts`, + ); + return firstTransactionSignature; + } catch (err: any) { + lastError = err; + logger.error( + { clientIndex: i, totalAttempts, err: err.message }, + `Attempt ${totalAttempts}: Error sending bundle to Jito client ${i}`, + ); + } + + // Check if we've run out of time + if (Date.now() - startTime >= maxRetryTimeMs) { + break; + } + } + + // If we've tried all clients and still have time, wait before next cycle + const timeRemaining = maxRetryTimeMs - (Date.now() - startTime); + if (timeRemaining > delayBetweenCyclesMs) { + await new Promise((resolve) => setTimeout(resolve, delayBetweenCyclesMs)); } } - throw ( - lastError || - new Error("Failed to send transactions via JITO after maximum retry time") + const totalTimeMs = Date.now() - startTime; + const errorMsg = `Failed to send transactions via JITO after ${totalAttempts} attempts over ${totalTimeMs}ms (max: ${maxRetryTimeMs}ms)`; + + logger.error( + { + totalAttempts, + totalTimeMs, + maxRetryTimeMs, + lastError: lastError?.message, + }, + errorMsg, ); + + throw lastError || new Error(errorMsg); } From a4907a3caab3de8047f43587301eba5d4a75798a Mon Sep 17 00:00:00 2001 From: Tejas Badadare Date: Wed, 11 Jun 2025 16:50:51 -0700 Subject: [PATCH 5/5] chore: bump ver --- apps/price_pusher/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/price_pusher/package.json b/apps/price_pusher/package.json index ae9f213d9a..f48b1e6819 100644 --- a/apps/price_pusher/package.json +++ b/apps/price_pusher/package.json @@ -1,6 +1,6 @@ { "name": "@pythnetwork/price-pusher", - "version": "9.3.4", + "version": "9.3.5", "description": "Pyth Price Pusher", "homepage": "https://pyth.network", "main": "lib/index.js",