diff --git a/apps/price_pusher/README.md b/apps/price_pusher/README.md index 51f0838fac..6ddc76aec7 100644 --- a/apps/price_pusher/README.md +++ b/apps/price_pusher/README.md @@ -159,7 +159,7 @@ pnpm run start solana \ --endpoint https://api.mainnet-beta.solana.com \ --keypair-file ./id.json \ --shard-id 1 \ - --jito-endpoint mainnet.block-engine.jito.wtf \ + --jito-endpoints mainnet.block-engine.jito.wtf,ny.mainnet.block-engine.jito.wtf \ --jito-keypair-file ./jito.json \ --jito-tip-lamports 100000 \ --jito-bundle-size 5 \ diff --git a/apps/price_pusher/package.json b/apps/price_pusher/package.json index ae9f213d9a..f48b1e6819 100644 --- a/apps/price_pusher/package.json +++ b/apps/price_pusher/package.json @@ -1,6 +1,6 @@ { "name": "@pythnetwork/price-pusher", - "version": "9.3.4", + "version": "9.3.5", "description": "Pyth Price Pusher", "homepage": "https://pyth.network", "main": "lib/index.js", diff --git a/apps/price_pusher/src/solana/command.ts b/apps/price_pusher/src/solana/command.ts index 93420ef79d..2196bb50cc 100644 --- a/apps/price_pusher/src/solana/command.ts +++ b/apps/price_pusher/src/solana/command.ts @@ -49,8 +49,8 @@ export default { type: "number", default: 50000, } as Options, - "jito-endpoint": { - description: "Jito endpoint", + "jito-endpoints": { + description: "Jito endpoint(s) - comma-separated list of endpoints", type: "string", optional: true, } as Options, @@ -117,7 +117,7 @@ export default { pythContractAddress, pushingFrequency, pollingFrequency, - jitoEndpoint, + jitoEndpoints, jitoKeypairFile, jitoTipLamports, dynamicJitoTips, @@ -209,7 +209,18 @@ export default { Uint8Array.from(JSON.parse(fs.readFileSync(jitoKeypairFile, "ascii"))), ); - const jitoClient = searcherClient(jitoEndpoint, jitoKeypair); + const jitoEndpointsList = jitoEndpoints + .split(",") + .map((endpoint: string) => endpoint.trim()); + const jitoClients: SearcherClient[] = jitoEndpointsList.map( + (endpoint: string) => { + logger.info( + `Constructing Jito searcher client from endpoint ${endpoint}`, + ); + return searcherClient(endpoint, jitoKeypair); + }, + ); + solanaPricePusher = new SolanaPricePusherJito( pythSolanaReceiver, hermesClient, @@ -218,13 +229,17 @@ export default { jitoTipLamports, dynamicJitoTips, maxJitoTipLamports, - jitoClient, + jitoClients, jitoBundleSize, updatesPerJitoBundle, + // Set max retry time to pushing frequency, since we want to stop retrying before the next push attempt + pushingFrequency * 1000, lookupTableAccount, ); - onBundleResult(jitoClient, logger.child({ module: "JitoClient" })); + jitoClients.forEach((client, index) => { + onBundleResult(client, logger.child({ module: `JitoClient-${index}` })); + }); } else { solanaPricePusher = new SolanaPricePusher( pythSolanaReceiver, diff --git a/apps/price_pusher/src/solana/solana.ts b/apps/price_pusher/src/solana/solana.ts index 1213490cc2..ef3632ab61 100644 --- a/apps/price_pusher/src/solana/solana.ts +++ b/apps/price_pusher/src/solana/solana.ts @@ -166,9 +166,10 @@ export class SolanaPricePusherJito implements IPricePusher { private defaultJitoTipLamports: number, private dynamicJitoTips: boolean, private maxJitoTipLamports: number, - private searcherClient: SearcherClient, + private searcherClients: SearcherClient[], private jitoBundleSize: number, private updatesPerJitoBundle: number, + private maxRetryTimeMs: number, private addressLookupTableAccount?: AddressLookupTableAccount, ) {} @@ -194,10 +195,6 @@ export class SolanaPricePusherJito implements IPricePusher { } } - private async sleep(ms: number): Promise { - return new Promise((resolve) => setTimeout(resolve, ms)); - } - async updatePriceFeed(priceIds: string[]): Promise { const recentJitoTip = await this.getRecentJitoTipLamports(); const jitoTip = @@ -243,32 +240,15 @@ export class SolanaPricePusherJito implements IPricePusher { jitoBundleSize: this.jitoBundleSize, }); - let retries = 60; - while (retries > 0) { - try { - await sendTransactionsJito( - transactions, - this.searcherClient, - this.pythSolanaReceiver.wallet, - ); - break; - } catch (err: any) { - if (err.code === 8 && err.details?.includes("Rate limit exceeded")) { - this.logger.warn("Rate limit hit, waiting before retry..."); - await this.sleep(1100); // Wait slightly more than 1 second - retries--; - if (retries === 0) { - this.logger.error("Max retries reached for rate limit"); - throw err; - } - } else { - throw err; - } - } - } - - // Add a delay between bundles to avoid rate limiting - await this.sleep(1100); + await sendTransactionsJito( + transactions, + this.searcherClients, + this.pythSolanaReceiver.wallet, + { + maxRetryTimeMs: this.maxRetryTimeMs, + }, + this.logger, + ); } } } diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index c5fa1d6f77..1e5aca631e 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -2889,6 +2889,9 @@ importers: jito-ts: specifier: ^3.0.1 version: 3.0.1(bufferutil@4.0.9)(encoding@0.1.13)(utf-8-validate@5.0.10) + ts-log: + specifier: ^2.2.7 + version: 2.2.7 devDependencies: '@solana/wallet-adapter-react': specifier: ^0.15.28 diff --git a/target_chains/solana/sdk/js/solana_utils/package.json b/target_chains/solana/sdk/js/solana_utils/package.json index d647e92b53..e737e8e24a 100644 --- a/target_chains/solana/sdk/js/solana_utils/package.json +++ b/target_chains/solana/sdk/js/solana_utils/package.json @@ -1,6 +1,6 @@ { "name": "@pythnetwork/solana-utils", - "version": "0.4.4", + "version": "0.4.5", "description": "Utility functions for Solana", "homepage": "https://pyth.network", "main": "lib/index.js", @@ -49,6 +49,7 @@ "@coral-xyz/anchor": "^0.29.0", "@solana/web3.js": "^1.90.0", "bs58": "^5.0.0", - "jito-ts": "^3.0.1" + "jito-ts": "^3.0.1", + "ts-log": "^2.2.7" } } diff --git a/target_chains/solana/sdk/js/solana_utils/src/jito.ts b/target_chains/solana/sdk/js/solana_utils/src/jito.ts index 2a6a5c290e..12a2762f34 100644 --- a/target_chains/solana/sdk/js/solana_utils/src/jito.ts +++ b/target_chains/solana/sdk/js/solana_utils/src/jito.ts @@ -1,3 +1,4 @@ +import { dummyLogger, Logger } from "ts-log"; import { Wallet } from "@coral-xyz/anchor"; import { PublicKey, @@ -42,9 +43,27 @@ export async function sendTransactionsJito( tx: VersionedTransaction; signers?: Signer[] | undefined; }[], - searcherClient: SearcherClient, + searcherClients: SearcherClient | SearcherClient[], wallet: Wallet, + options: { + maxRetryTimeMs?: number; // Max time to retry sending transactions + delayBetweenCyclesMs?: number; // Delay between cycles of sending transactions to all searcher clients + } = {}, + logger: Logger = dummyLogger, // Optional logger to track progress of retries ): Promise { + const clients = Array.isArray(searcherClients) + ? searcherClients + : [searcherClients]; + + if (clients.length === 0) { + throw new Error("No searcher clients provided"); + } + + const maxRetryTimeMs = options.maxRetryTimeMs || 60000; // Default to 60 seconds + const delayBetweenCyclesMs = options.delayBetweenCyclesMs || 1000; // Default to 1 second + + const startTime = Date.now(); + const signedTransactions = []; for (const transaction of transactions) { @@ -64,7 +83,56 @@ export async function sendTransactionsJito( ); const bundle = new Bundle(signedTransactions, 2); - await searcherClient.sendBundle(bundle); - return firstTransactionSignature; + let lastError: Error | null = null; + let totalAttempts = 0; + + while (Date.now() - startTime < maxRetryTimeMs) { + // Try all clients in this cycle + for (let i = 0; i < clients.length; i++) { + const currentClient = clients[i]; + totalAttempts++; + + try { + await currentClient.sendBundle(bundle); + logger.info( + { clientIndex: i, totalAttempts }, + `Successfully sent bundle to Jito client after ${totalAttempts} attempts`, + ); + return firstTransactionSignature; + } catch (err: any) { + lastError = err; + logger.error( + { clientIndex: i, totalAttempts, err: err.message }, + `Attempt ${totalAttempts}: Error sending bundle to Jito client ${i}`, + ); + } + + // Check if we've run out of time + if (Date.now() - startTime >= maxRetryTimeMs) { + break; + } + } + + // If we've tried all clients and still have time, wait before next cycle + const timeRemaining = maxRetryTimeMs - (Date.now() - startTime); + if (timeRemaining > delayBetweenCyclesMs) { + await new Promise((resolve) => setTimeout(resolve, delayBetweenCyclesMs)); + } + } + + const totalTimeMs = Date.now() - startTime; + const errorMsg = `Failed to send transactions via JITO after ${totalAttempts} attempts over ${totalTimeMs}ms (max: ${maxRetryTimeMs}ms)`; + + logger.error( + { + totalAttempts, + totalTimeMs, + maxRetryTimeMs, + lastError: lastError?.message, + }, + errorMsg, + ); + + throw lastError || new Error(errorMsg); }