Convert Banks (#9033)

* Store and compute needed state in EpochStakes struct
Co-authored-by: Carl <[email protected]>

Author: carllin

ledger/src/snapshot_utils.rs

@@ -8,7 +8,7 @@ use solana_measure::measure::Measure;
 use solana_runtime::{
     accounts_db::{SnapshotStorage, SnapshotStorages},
     bank::{
-        self, deserialize_from_snapshot, Bank, BankRcSerialize, BankSlotDelta,
+        self, bank_1_0::Bank1_0, deserialize_from_snapshot, Bank, BankRcSerialize, BankSlotDelta,
         MAX_SNAPSHOT_DATA_FILE_SIZE,
     },
 };
@@ -30,7 +30,8 @@ pub const TAR_SNAPSHOTS_DIR: &str = "snapshots";
 pub const TAR_ACCOUNTS_DIR: &str = "accounts";
 pub const TAR_VERSION_FILE: &str = "version";
 
-pub const SNAPSHOT_VERSION: &str = "1.0.0";
+pub const SNAPSHOT_VERSION_1_0: &str = "1.0.0";
+pub const SNAPSHOT_VERSION: &str = "1.1.0";
 
 #[derive(PartialEq, Ord, Eq, Debug)]
 pub struct SlotSnapshotPaths {
@@ -593,6 +594,10 @@ where
         MAX_SNAPSHOT_DATA_FILE_SIZE,
         |stream| {
             let mut bank: Bank = match snapshot_version {
+                SNAPSHOT_VERSION_1_0 => {
+                    let bank_1_0: Bank1_0 = deserialize_from_snapshot(stream.by_ref())?;
+                    bank_1_0.convert_to_current()
+                }
                 SNAPSHOT_VERSION => deserialize_from_snapshot(stream.by_ref())?,
                 _ => {
                     return Err(get_io_error(&format!(
@@ -602,6 +607,7 @@ where
                 }
             };
             info!("Rebuilding accounts...");
+
             let rc = bank::BankRc::from_stream(
                 account_paths,
                 bank.slot(),

runtime/src/bank/bank_1_0.rs

@@ -0,0 +1,220 @@
+//! The `bank` module tracks client accounts and the progress of on-chain
+//! programs. It offers a high-level API that signs transactions
+//! on behalf of the caller, and a low-level API for when they have
+//! already been signed and verified.
+use crate::{
+    bank::{Bank, BankRc, EnteredEpochCallback, StatusCacheRc},
+    blockhash_queue::BlockhashQueue,
+    epoch_stakes::EpochStakes,
+    message_processor::MessageProcessor,
+    rent_collector::RentCollector,
+    serde_utils::{
+        deserialize_atomicbool, deserialize_atomicu64, serialize_atomicbool, serialize_atomicu64,
+    },
+    stakes::Stakes,
+    storage_utils::StorageAccounts,
+};
+use serde::{Deserialize, Serialize};
+use solana_sdk::{
+    clock::{Epoch, Slot, UnixTimestamp},
+    epoch_schedule::EpochSchedule,
+    fee_calculator::{FeeCalculator, FeeRateGovernor},
+    hard_forks::HardForks,
+    hash::Hash,
+    inflation::Inflation,
+    pubkey::Pubkey,
+};
+use std::{
+    collections::HashMap,
+    sync::atomic::{AtomicBool, AtomicU64},
+    sync::{Arc, RwLock},
+};
+
+/// Manager for the state of all accounts and programs after processing its entries.
+#[derive(Deserialize, Serialize)]
+pub struct Bank1_0 {
+    /// References to accounts, parent and signature status
+    #[serde(skip)]
+    pub rc: BankRc,
+
+    #[serde(skip)]
+    pub src: StatusCacheRc,
+
+    /// FIFO queue of `recent_blockhash` items
+    pub blockhash_queue: RwLock<BlockhashQueue>,
+
+    /// The set of parents including this bank
+    pub ancestors: HashMap<Slot, usize>,
+
+    /// Hash of this Bank's state. Only meaningful after freezing.
+    pub hash: RwLock<Hash>,
+
+    /// Hash of this Bank's parent's state
+    pub parent_hash: Hash,
+
+    /// parent's slot
+    pub parent_slot: Slot,
+
+    /// slots to hard fork at
+    pub hard_forks: Arc<RwLock<HardForks>>,
+
+    /// The number of transactions processed without error
+    #[serde(serialize_with = "serialize_atomicu64")]
+    #[serde(deserialize_with = "deserialize_atomicu64")]
+    pub transaction_count: AtomicU64,
+
+    /// Bank tick height
+    #[serde(serialize_with = "serialize_atomicu64")]
+    #[serde(deserialize_with = "deserialize_atomicu64")]
+    pub tick_height: AtomicU64,
+
+    /// The number of signatures from valid transactions in this slot
+    #[serde(serialize_with = "serialize_atomicu64")]
+    #[serde(deserialize_with = "deserialize_atomicu64")]
+    pub signature_count: AtomicU64,
+
+    /// Total capitalization, used to calculate inflation
+    #[serde(serialize_with = "serialize_atomicu64")]
+    #[serde(deserialize_with = "deserialize_atomicu64")]
+    pub capitalization: AtomicU64,
+
+    // Bank max_tick_height
+    pub max_tick_height: u64,
+
+    /// The number of hashes in each tick. None value means hashing is disabled.
+    pub hashes_per_tick: Option<u64>,
+
+    /// The number of ticks in each slot.
+    pub ticks_per_slot: u64,
+
+    /// length of a slot in ns
+    pub ns_per_slot: u128,
+
+    /// genesis time, used for computed clock
+    pub genesis_creation_time: UnixTimestamp,
+
+    /// The number of slots per year, used for inflation
+    pub slots_per_year: f64,
+
+    /// The number of slots per Storage segment
+    pub slots_per_segment: u64,
+
+    /// Bank slot (i.e. block)
+    pub slot: Slot,
+
+    /// Bank epoch
+    pub epoch: Epoch,
+
+    /// Bank block_height
+    pub block_height: u64,
+
+    /// The pubkey to send transactions fees to.
+    pub collector_id: Pubkey,
+
+    /// Fees that have been collected
+    #[serde(serialize_with = "serialize_atomicu64")]
+    #[serde(deserialize_with = "deserialize_atomicu64")]
+    pub collector_fees: AtomicU64,
+
+    /// Latest transaction fees for transactions processed by this bank
+    pub fee_calculator: FeeCalculator,
+
+    /// Track cluster signature throughput and adjust fee rate
+    pub fee_rate_governor: FeeRateGovernor,
+
+    /// Rent that have been collected
+    #[serde(serialize_with = "serialize_atomicu64")]
+    #[serde(deserialize_with = "deserialize_atomicu64")]
+    pub collected_rent: AtomicU64,
+
+    /// latest rent collector, knows the epoch
+    pub rent_collector: RentCollector,
+
+    /// initialized from genesis
+    pub epoch_schedule: EpochSchedule,
+
+    /// inflation specs
+    pub inflation: Arc<RwLock<Inflation>>,
+
+    /// cache of vote_account and stake_account state for this fork
+    pub stakes: RwLock<Stakes>,
+
+    /// cache of validator and archiver storage accounts for this fork
+    pub storage_accounts: RwLock<StorageAccounts>,
+
+    /// staked nodes on epoch boundaries, saved off when a bank.slot() is at
+    ///   a leader schedule calculation boundary
+    pub epoch_stakes: HashMap<Epoch, Stakes>,
+
+    /// A boolean reflecting whether any entries were recorded into the PoH
+    /// stream for the slot == self.slot
+    #[serde(serialize_with = "serialize_atomicbool")]
+    #[serde(deserialize_with = "deserialize_atomicbool")]
+    pub is_delta: AtomicBool,
+
+    /// The Message processor
+    pub message_processor: MessageProcessor,
+
+    /// Callback to be notified when a bank enters a new Epoch
+    /// (used to adjust cluster features over time)
+    #[serde(skip)]
+    pub entered_epoch_callback: Arc<RwLock<Option<EnteredEpochCallback>>>,
+
+    /// Last time when the cluster info vote listener has synced with this bank
+    #[serde(skip)]
+    pub last_vote_sync: AtomicU64,
+
+    /// Rewards that were paid out immediately after this bank was created
+    #[serde(skip)]
+    pub rewards: Option<Vec<(Pubkey, i64)>>,
+}
+
+impl Bank1_0 {
+    pub fn convert_to_current(self) -> Bank {
+        let old_epoch_stakes = self.epoch_stakes;
+        let epoch_stakes = old_epoch_stakes
+            .iter()
+            .map(|(epoch, stakes)| (*epoch, EpochStakes::new(&stakes, *epoch)))
+            .collect();
+        Bank {
+            rc: self.rc,
+            src: self.src,
+            blockhash_queue: self.blockhash_queue,
+            ancestors: self.ancestors,
+            hash: self.hash,
+            parent_hash: self.parent_hash,
+            parent_slot: self.parent_slot,
+            hard_forks: self.hard_forks,
+            transaction_count: self.transaction_count,
+            tick_height: self.tick_height,
+            signature_count: self.signature_count,
+            capitalization: self.capitalization,
+            max_tick_height: self.max_tick_height,
+            hashes_per_tick: self.hashes_per_tick,
+            ticks_per_slot: self.ticks_per_slot,
+            ns_per_slot: self.ns_per_slot,
+            genesis_creation_time: self.genesis_creation_time,
+            slots_per_year: self.slots_per_year,
+            slots_per_segment: self.slots_per_segment,
+            slot: self.slot,
+            epoch: self.epoch,
+            block_height: self.block_height,
+            collector_id: self.collector_id,
+            collector_fees: self.collector_fees,
+            fee_calculator: self.fee_calculator,
+            fee_rate_governor: self.fee_rate_governor,
+            collected_rent: self.collected_rent,
+            rent_collector: self.rent_collector,
+            epoch_schedule: self.epoch_schedule,
+            inflation: self.inflation,
+            stakes: self.stakes,
+            storage_accounts: self.storage_accounts,
+            epoch_stakes,
+            is_delta: self.is_delta,
+            message_processor: self.message_processor,
+            entered_epoch_callback: self.entered_epoch_callback,
+            last_vote_sync: self.last_vote_sync,
+            rewards: self.rewards,
+        }
+    }
+}

runtime/src/bank.rs renamed to runtime/src/bank/mod.rs

@@ -6,6 +6,7 @@ use crate::{
     accounts::{Accounts, TransactionAccounts, TransactionLoadResult, TransactionLoaders},
     accounts_db::{AccountsDBSerialize, ErrorCounters, SnapshotStorage, SnapshotStorages},
     blockhash_queue::BlockhashQueue,
+    epoch_stakes::{EpochStakes, NodeVoteAccounts},
     message_processor::{MessageProcessor, ProcessInstruction},
     nonce_utils,
     rent_collector::RentCollector,
@@ -60,6 +61,8 @@ use std::{
     sync::{Arc, RwLock, RwLockReadGuard},
 };
 
+pub mod bank_1_0;
+
 pub const SECONDS_PER_YEAR: f64 = 365.25 * 24.0 * 60.0 * 60.0;
 pub const MAX_SNAPSHOT_DATA_FILE_SIZE: u64 = 32 * 1024 * 1024 * 1024; // 32 GiB
 
@@ -323,7 +326,7 @@ pub struct Bank {
 
     /// staked nodes on epoch boundaries, saved off when a bank.slot() is at
     ///   a leader schedule calculation boundary
-    epoch_stakes: HashMap<Epoch, Stakes>,
+    epoch_stakes: HashMap<Epoch, EpochStakes>,
 
     /// A boolean reflecting whether any entries were recorded into the PoH
     /// stream for the slot == self.slot
@@ -380,7 +383,8 @@ impl Bank {
         {
             let stakes = bank.stakes.read().unwrap();
             for epoch in 0..=bank.get_leader_schedule_epoch(bank.slot) {
-                bank.epoch_stakes.insert(epoch, stakes.clone());
+                bank.epoch_stakes
+                    .insert(epoch, EpochStakes::new(&stakes, epoch));
             }
             bank.update_stake_history(None);
         }
@@ -592,8 +596,24 @@ impl Bank {
                 epoch >= leader_schedule_epoch.saturating_sub(MAX_LEADER_SCHEDULE_STAKES)
             });
 
+            let vote_stakes: HashMap<_, _> = self
+                .stakes
+                .read()
+                .unwrap()
+                .vote_accounts()
+                .iter()
+                .map(|(epoch, (stake, _))| (*epoch, *stake))
+                .collect();
+            let new_epoch_stakes =
+                EpochStakes::new(&self.stakes.read().unwrap(), leader_schedule_epoch);
+            info!(
+                "new epoch stakes, epoch: {}, stakes: {:#?}, total_stake: {}",
+                leader_schedule_epoch,
+                vote_stakes,
+                new_epoch_stakes.total_stake(),
+            );
             self.epoch_stakes
-                .insert(leader_schedule_epoch, self.stakes.read().unwrap().clone());
+                .insert(leader_schedule_epoch, new_epoch_stakes);
         }
     }
 
@@ -2056,10 +2076,55 @@ impl Bank {
         self.stakes.read().unwrap().vote_accounts().clone()
     }
 
+    /// Get the EpochStakes for a given epoch
+    pub fn epoch_stakes(&self, epoch: Epoch) -> Option<&EpochStakes> {
+        self.epoch_stakes.get(&epoch)
+    }
+
     /// vote accounts for the specific epoch along with the stake
     ///   attributed to each account
     pub fn epoch_vote_accounts(&self, epoch: Epoch) -> Option<&HashMap<Pubkey, (u64, Account)>> {
-        self.epoch_stakes.get(&epoch).map(Stakes::vote_accounts)
+        self.epoch_stakes
+            .get(&epoch)
+            .map(|epoch_stakes| Stakes::vote_accounts(epoch_stakes.stakes()))
+    }
+
+    /// Get the fixed authorized voter for the given vote account for the
+    /// current epoch
+    pub fn epoch_authorized_voter(&self, vote_account: &Pubkey) -> Option<&Pubkey> {
+        self.epoch_stakes
+            .get(&self.epoch)
+            .expect("Epoch stakes for bank's own epoch must exist")
+            .epoch_authorized_voters()
+            .get(vote_account)
+    }
+
+    /// Get the fixed set of vote accounts for the given node id for the
+    /// current epoch
+    pub fn epoch_vote_accounts_for_node_id(&self, node_id: &Pubkey) -> Option<&NodeVoteAccounts> {
+        self.epoch_stakes
+            .get(&self.epoch)
+            .expect("Epoch stakes for bank's own epoch must exist")
+            .node_id_to_vote_accounts()
+            .get(node_id)
+    }
+
+    /// Get the fixed total stake of all vote accounts for current epoch
+    pub fn total_epoch_stake(&self) -> u64 {
+        self.epoch_stakes
+            .get(&self.epoch)
+            .expect("Epoch stakes for bank's own epoch must exist")
+            .total_stake()
+    }
+
+    /// Get the fixed stake of the given vote account for the current epoch
+    pub fn epoch_vote_account_stake(&self, voting_pubkey: &Pubkey) -> u64 {
+        *self
+            .epoch_vote_accounts(self.epoch())
+            .expect("Bank epoch vote accounts must contain entry for the bank's own epoch")
+            .get(voting_pubkey)
+            .map(|(stake, _)| stake)
+            .unwrap_or(&0)
     }
 
     /// given a slot, return the epoch and offset into the epoch this slot falls