Skip to content

Commit

Permalink
Merge pull request #141 from HerodotusDev/optimize/fetcher
Browse files Browse the repository at this point in the history 
Optimize/fetcher (EVM)
  • Loading branch information
@codyx
codyx authored Feb 6, 2025
2 parents 5de3f1d + fd69cf1 commit 4457117
Show file tree
Hide file tree
Showing 6 changed files with 302 additions and 99 deletions.
151 changes: 88 additions & 63 deletions crates/fetcher/src/lib.rs
View file
Original file line number Diff line number Diff line change
@@ -1,23 +1,34 @@
use std::{
collections::{HashMap, HashSet},
env,
num::ParseIntError,
};

use alloy::hex::FromHexError;
use dry_hint_processor::syscall_handler::{evm, starknet, SyscallHandler};
use eth_trie_proofs::{tx_receipt_trie::TxReceiptsMptHandler, tx_trie::TxsMptHandler};
use futures::StreamExt;
use indexer::models::IndexerError;
use indicatif::{MultiProgress, ProgressBar, ProgressStyle};
use proof_keys::{evm::ProofKeys as EvmProofKeys, starknet::ProofKeys as StarknetProofKeys, ProofKeys};
use proof_keys::{
evm::{FlattenedKey, ProofKeys as EvmProofKeys},
starknet::ProofKeys as StarknetProofKeys,
ProofKeys,
};
use reqwest::Url;
use starknet_types_core::felt::FromStrError;
use thiserror::Error;
use types::proofs::{
evm::{
account::Account, header::Header as EvmHeader, receipt::Receipt, storage::Storage, transaction::Transaction, Proofs as EvmProofs,
use types::{
proofs::{
evm::{
account::Account, header::Header as EvmHeader, receipt::Receipt, storage::Storage, transaction::Transaction,
Proofs as EvmProofs,
},
header::HeaderMmrMeta,
mmr::MmrMeta,
starknet::{header::Header as StarknetHeader, storage::Storage as StarknetStorage, Proofs as StarknetProofs},
},
header::HeaderMmrMeta,
mmr::MmrMeta,
starknet::{header::Header as StarknetHeader, storage::Storage as StarknetStorage, Proofs as StarknetProofs},
RPC_URL_ETHEREUM,
};

pub mod proof_keys;
Expand Down Expand Up @@ -127,7 +138,6 @@ pub struct Fetcher<'a> {
#[cfg(feature = "progress_bars")]
progress_bars: ProgressBars,
}

impl<'a> Fetcher<'a> {
pub fn new(proof_keys: &'a ProofKeys) -> Self {
Self {
Expand All @@ -137,18 +147,13 @@ impl<'a> Fetcher<'a> {
}
}

pub async fn collect_evm_proofs(&self) -> Result<EvmProofs, FetcherError> {
async fn collect_evm_headers_proofs(
&self,
flattened_keys: &HashSet<FlattenedKey>,
) -> Result<HashMap<MmrMeta, Vec<EvmHeader>>, FetcherError> {
let mut headers_with_mmr = HashMap::default();
let mut accounts: HashSet<Account> = HashSet::default();
let mut storages: HashSet<Storage> = HashSet::default();
let mut receipts: HashSet<Receipt> = HashSet::default();
let mut transactions: HashSet<Transaction> = HashSet::default();

// Collect header proofs
let mut header_fut = futures::stream::iter(
self.proof_keys
.evm
.header_keys
flattened_keys
.iter()
.map(|key| EvmProofKeys::fetch_header_proof(key.chain_id, key.block_number)),
)
Expand All @@ -166,23 +171,30 @@ impl<'a> Fetcher<'a> {
self.progress_bars.evm_header.safe_inc();
}

Ok(headers_with_mmr)
}

pub async fn collect_evm_proofs(&self) -> Result<EvmProofs, FetcherError> {
let mut accounts: HashSet<Account> = HashSet::default();
let mut storages: HashSet<Storage> = HashSet::default();
let mut receipts: HashSet<Receipt> = HashSet::default();
let mut transactions: HashSet<Transaction> = HashSet::default();

let flattened_keys = self.proof_keys.evm.to_flattened_keys();

// Collect required header proofs for all keys
let headers_with_mmr = self.collect_evm_headers_proofs(&flattened_keys).await?;

#[cfg(feature = "progress_bars")]
self.progress_bars
.evm_header
.safe_finish_with_message("ethereum header keys - finished");
self.progress_bars.evm_header.safe_finish_with_message("evm header keys - finished");

// Collect account proofs
let mut account_fut = futures::stream::iter(self.proof_keys.evm.account_keys.iter().map(EvmProofKeys::fetch_account_proof))
.buffer_unordered(BUFFER_UNORDERED)
.boxed();

while let Some(result) = account_fut.next().await {
let (header_with_mmr, account) = result?;
headers_with_mmr
.entry(header_with_mmr.mmr_meta)
.and_modify(|headers: &mut Vec<EvmHeader>| headers.extend(header_with_mmr.headers.clone()))
.or_insert(header_with_mmr.headers);
accounts.insert(account);
accounts.insert(result?);

#[cfg(feature = "progress_bars")]
self.progress_bars.evm_account.safe_inc();
Expand All @@ -191,19 +203,15 @@ impl<'a> Fetcher<'a> {
#[cfg(feature = "progress_bars")]
self.progress_bars
.evm_account
.safe_finish_with_message("ethereum account keys - finished");
.safe_finish_with_message("evm account keys - finished");

// Collect storage proofs
let mut storage_fut = futures::stream::iter(self.proof_keys.evm.storage_keys.iter().map(EvmProofKeys::fetch_storage_proof))
.buffer_unordered(BUFFER_UNORDERED)
.boxed();

while let Some(result) = storage_fut.next().await {
let (header_with_mmr, account, storage) = result?;
headers_with_mmr
.entry(header_with_mmr.mmr_meta)
.and_modify(|headers: &mut Vec<EvmHeader>| headers.extend(header_with_mmr.headers.clone()))
.or_insert(header_with_mmr.headers);
let (account, storage) = result?;
accounts.insert(account);
storages.insert(storage);

Expand All @@ -214,56 +222,73 @@ impl<'a> Fetcher<'a> {
#[cfg(feature = "progress_bars")]
self.progress_bars
.evm_storage
.safe_finish_with_message("ethereum storage keys - finished");
.safe_finish_with_message("evm storage keys - finished");

// Collect ransaction receipts proofs
let mut transaction_receipts_fut =
futures::stream::iter(self.proof_keys.evm.receipt_keys.iter().map(EvmProofKeys::fetch_receipt_proof))
.buffer_unordered(BUFFER_UNORDERED)
.boxed();
// For each block, we need to create a mpt_handler
let mut receipt_mpt_handlers: HashMap<u64, TxReceiptsMptHandler> = HashMap::default();

while let Some(Ok((header_with_mmr, transaction_receipt))) = transaction_receipts_fut.next().await {
headers_with_mmr
.entry(header_with_mmr.mmr_meta)
.and_modify(|headers: &mut Vec<EvmHeader>| headers.extend(header_with_mmr.headers.clone()))
.or_insert(header_with_mmr.headers);
receipts.insert(transaction_receipt);
for block_number in self.proof_keys.evm.receipt_keys.iter().map(|key| key.block_number) {
if let std::collections::hash_map::Entry::Vacant(entry) = receipt_mpt_handlers.entry(block_number) {
let mut mpt_handler = TxReceiptsMptHandler::new(Url::parse(&env::var(RPC_URL_ETHEREUM).unwrap()).unwrap())
.map_err(|e| FetcherError::InternalError(e.to_string()))?;

mpt_handler
.build_tx_receipts_tree_from_block(block_number)
.await
.map_err(|e| FetcherError::InternalError(e.to_string()))?;

entry.insert(mpt_handler);
}

#[cfg(feature = "progress_bars")]
self.progress_bars.evm_receipts.safe_inc();
}

receipts.extend(
self.proof_keys
.evm
.receipt_keys
.iter()
.map(|key| EvmProofKeys::compute_receipt_proof(key, receipt_mpt_handlers.get_mut(&key.block_number).unwrap()).unwrap()),
);

#[cfg(feature = "progress_bars")]
self.progress_bars
.evm_receipts
.safe_finish_with_message("ethereum receipt keys - finished");
.safe_finish_with_message("evm receipt keys - finished");

// Collect storage proofs
let mut transaction_keys_fut = futures::stream::iter(
self.proof_keys
.evm
.transaction_keys
.iter()
.map(EvmProofKeys::fetch_transaction_proof),
)
.buffer_unordered(BUFFER_UNORDERED)
.boxed();
// For each tx block, we need to create a mpt_handler
let mut tx_mpt_handlers: HashMap<u64, TxsMptHandler> = HashMap::default();

while let Some(Ok((header_with_mmr, transaction))) = transaction_keys_fut.next().await {
headers_with_mmr
.entry(header_with_mmr.mmr_meta)
.and_modify(|headers: &mut Vec<EvmHeader>| headers.extend(header_with_mmr.headers.clone()))
.or_insert(header_with_mmr.headers);
transactions.insert(transaction);
for key in self.proof_keys.evm.transaction_keys.iter() {
if let std::collections::hash_map::Entry::Vacant(entry) = tx_mpt_handlers.entry(key.block_number) {
let mut mpt_handler = TxsMptHandler::new(Url::parse(&env::var(RPC_URL_ETHEREUM).unwrap()).unwrap())
.map_err(|e| FetcherError::InternalError(e.to_string()))?;

mpt_handler
.build_tx_tree_from_block(key.block_number)
.await
.map_err(|e| FetcherError::InternalError(e.to_string()))?;

entry.insert(mpt_handler);
}

#[cfg(feature = "progress_bars")]
self.progress_bars.evm_transactions.safe_inc();
}

transactions.extend(
self.proof_keys
.evm
.transaction_keys
.iter()
.map(|key| EvmProofKeys::compute_transaction_proof(key, tx_mpt_handlers.get_mut(&key.block_number).unwrap()).unwrap()),
);

#[cfg(feature = "progress_bars")]
self.progress_bars
.evm_transactions
.safe_finish_with_message("ethereum storage keys - finished");
.safe_finish_with_message("evm storage keys - finished");

Ok(EvmProofs {
headers_with_mmr: process_headers(headers_with_mmr),
Expand Down
Loading

0 comments on commit 4457117

Please sign in to comment.