Merge branch 'solana_repo' of github.com:wen-coding/solana into solana_repo

Author: wen-coding

accounts-db/Cargo.toml

@@ -10,72 +10,56 @@ license = { workspace = true }
 edition = { workspace = true }
 
 [dependencies]
-ahash = { workspace = true }
+arrayref = { workspace = true }
 bincode = { workspace = true }
 blake3 = { workspace = true }
 bv = { workspace = true, features = ["serde"] }
 bytemuck = { workspace = true }
-bytemuck_derive = { workspace = true }
+byteorder = { workspace = true }
 bzip2 = { workspace = true }
 crossbeam-channel = { workspace = true }
 dashmap = { workspace = true, features = ["rayon", "raw-api"] }
+flate2 = { workspace = true }
+fnv = { workspace = true }
+im = { workspace = true, features = ["rayon", "serde"] }
+index_list = { workspace = true }
 indexmap = { workspace = true }
 itertools = { workspace = true }
+lazy_static = { workspace = true }
 log = { workspace = true }
 lz4 = { workspace = true }
 memmap2 = { workspace = true }
 modular-bitfield = { workspace = true }
+num-derive = { workspace = true }
+num-traits = { workspace = true }
 num_cpus = { workspace = true }
 num_enum = { workspace = true }
-qualifier_attr = { workspace = true, optional = true }
+percentage = { workspace = true }
+qualifier_attr = { workspace = true }
 rand = { workspace = true }
 rayon = { workspace = true }
+regex = { workspace = true }
 seqlock = { workspace = true }
 serde = { workspace = true, features = ["rc"] }
 serde_derive = { workspace = true }
-smallvec = { workspace = true, features = ["const_generics"] }
-solana-account = { workspace = true, features = ["serde"] }
-solana-address-lookup-table-interface = { workspace = true, features = [
-    "bincode",
-    "bytemuck",
-] }
+smallvec = { workspace = true }
 solana-bucket-map = { workspace = true }
-solana-clock = { workspace = true }
-solana-epoch-schedule = { workspace = true }
-solana-fee-calculator = { workspace = true }
-solana-frozen-abi = { workspace = true, optional = true, features = [
-    "frozen-abi",
-] }
-solana-frozen-abi-macro = { workspace = true, optional = true, features = [
-    "frozen-abi",
-] }
-solana-genesis-config = { workspace = true, features = ["serde"] }
-solana-hash = { workspace = true, features = ["serde"] }
-solana-keypair = { workspace = true, optional = true }
-solana-lattice-hash = { workspace = true }
+solana-config-program = { workspace = true }
+solana-frozen-abi = { workspace = true }
+solana-frozen-abi-macro = { workspace = true }
 solana-measure = { workspace = true }
-solana-message = { workspace = true }
 solana-metrics = { workspace = true }
 solana-nohash-hasher = { workspace = true }
-solana-pubkey = { workspace = true }
+solana-program-runtime = { workspace = true }
 solana-rayon-threadlimit = { workspace = true }
-solana-rent = { workspace = true, optional = true }
-solana-rent-collector = { workspace = true }
-solana-reward-info = { workspace = true, features = ["serde"] }
-solana-sha256-hasher = { workspace = true }
-solana-signer = { workspace = true, optional = true }
-solana-slot-hashes = { workspace = true }
-solana-stake-program = { workspace = true, optional = true }
-solana-svm-transaction = { workspace = true }
-solana-system-interface = { workspace = true }
-solana-sysvar = { workspace = true }
-solana-time-utils = { workspace = true }
-solana-transaction = { workspace = true }
-solana-transaction-context = { workspace = true }
-solana-transaction-error = { workspace = true }
-solana-vote-program = { workspace = true, optional = true }
-spl-generic-token = { workspace = true }
+solana-sdk = { workspace = true }
+solana-stake-program = { workspace = true }
+solana-svm = { workspace = true }
+solana-system-program = { workspace = true }
+solana-vote-program = { workspace = true }
 static_assertions = { workspace = true }
+strum = { workspace = true, features = ["derive"] }
+strum_macros = { workspace = true }
 tar = { workspace = true }
 tempfile = { workspace = true }
 thiserror = { workspace = true }
@@ -87,63 +71,26 @@ name = "solana_accounts_db"
 [dev-dependencies]
 assert_matches = { workspace = true }
 criterion = { workspace = true }
+ed25519-dalek = { workspace = true }
 libsecp256k1 = { workspace = true }
 memoffset = { workspace = true }
 rand_chacha = { workspace = true }
-serde_bytes = { workspace = true }
 # See order-crates-for-publishing.py for using this unusual `path = "."`
 solana-accounts-db = { path = ".", features = ["dev-context-only-utils"] }
-solana-compute-budget = { workspace = true }
-solana-instruction = { workspace = true }
 solana-logger = { workspace = true }
-solana-sdk-ids = { workspace = true }
-solana-slot-history = { workspace = true }
+solana-sdk = { workspace = true, features = ["dev-context-only-utils"] }
 static_assertions = { workspace = true }
-strum = { workspace = true, features = ["derive"] }
-strum_macros = { workspace = true }
 test-case = { workspace = true }
 
 [package.metadata.docs.rs]
 targets = ["x86_64-unknown-linux-gnu"]
 
-[features]
-dev-context-only-utils = [
-    "dep:qualifier_attr",
-    "dep:solana-keypair",
-    "dep:solana-rent",
-    "dep:solana-signer",
-    "dep:solana-stake-program",
-    "dep:solana-vote-program",
-    "solana-account/dev-context-only-utils",
-    "solana-pubkey/rand",
-    "solana-transaction/dev-context-only-utils",
-]
-frozen-abi = [
-    "dep:solana-frozen-abi",
-    "dep:solana-frozen-abi-macro",
-    "solana-fee-calculator/frozen-abi",
-    "solana-vote-program/frozen-abi",
-]
+[build-dependencies]
+rustc_version = { workspace = true }
 
-[[bench]]
-name = "bench_accounts_file"
-harness = false
+[features]
+dev-context-only-utils = []
 
 [[bench]]
 name = "bench_hashing"
 harness = false
-
-[[bench]]
-name = "read_only_accounts_cache"
-harness = false
-
-[[bench]]
-name = "bench_serde"
-harness = false
-
-[[bench]]
-name = "bench_lock_accounts"
-harness = false
-
-[lints]
-workspace = true

accounts-db/benches/bench_hashing.rs

@@ -1,13 +1,7 @@
 use {
-    criterion::{criterion_group, criterion_main, BatchSize, BenchmarkId, Criterion, Throughput},
-    rand::seq::SliceRandom,
-    solana_account::AccountSharedData,
-    solana_accounts_db::{
-        accounts_db::AccountsDb,
-        accounts_hash::{AccountHash, AccountsHasher},
-    },
-    solana_hash::Hash,
-    solana_pubkey::Pubkey,
+    criterion::{criterion_group, criterion_main, BenchmarkId, Criterion, Throughput},
+    solana_accounts_db::accounts_db::AccountsDb,
+    solana_sdk::{account::AccountSharedData, pubkey::Pubkey},
 };
 
 const KB: usize = 1024;
@@ -26,7 +20,7 @@ const DATA_SIZES: [usize; 6] = [
 /// part of computing an account's hash.
 ///
 /// Ensure this constant stays in sync with the value of `META_SIZE` in
-/// AccountsDb::hash_account_helper().
+/// AccountsDb::hash_account_data().
 const META_SIZE: usize = 81;
 
 fn bench_hash_account(c: &mut Criterion) {
@@ -39,52 +33,11 @@ fn bench_hash_account(c: &mut Criterion) {
         let num_bytes = META_SIZE.checked_add(data_size).unwrap();
         group.throughput(Throughput::Bytes(num_bytes as u64));
         let account = AccountSharedData::new(lamports, data_size, &owner);
-        group.bench_function(BenchmarkId::new("blake3", data_size), |b| {
+        group.bench_function(BenchmarkId::new("data_size", data_size), |b| {
             b.iter(|| AccountsDb::hash_account(&account, &address));
         });
-        group.bench_function(BenchmarkId::new("lattice", data_size), |b| {
-            b.iter(|| AccountsDb::lt_hash_account(&account, &address));
-        });
-    }
-}
-
-fn bench_accounts_delta_hash(c: &mut Criterion) {
-    const ACCOUNTS_COUNTS: [usize; 4] = [
-        1,      // the smallest count; will bench overhead
-        100,    // number of accounts written per slot on mnb (with *no* rent rewrites)
-        1_000,  // number of accounts written slot on mnb (with rent rewrites)
-        10_000, // reasonable largest number of accounts written per slot
-    ];
-
-    fn create_account_hashes(accounts_count: usize) -> Vec<(Pubkey, AccountHash)> {
-        let mut account_hashes: Vec<_> = std::iter::repeat_with(|| {
-            let address = Pubkey::new_unique();
-            let hash = AccountHash(Hash::new_unique());
-            (address, hash)
-        })
-        .take(accounts_count)
-        .collect();
-
-        // since the accounts delta hash needs to sort the accounts first, ensure we're not
-        // creating a pre-sorted vec.
-        let mut rng = rand::thread_rng();
-        account_hashes.shuffle(&mut rng);
-        account_hashes
-    }
-
-    let mut group = c.benchmark_group("accounts_delta_hash");
-    for accounts_count in ACCOUNTS_COUNTS {
-        group.throughput(Throughput::Elements(accounts_count as u64));
-        let account_hashes = create_account_hashes(accounts_count);
-        group.bench_function(BenchmarkId::new("accounts_count", accounts_count), |b| {
-            b.iter_batched(
-                || account_hashes.clone(),
-                AccountsHasher::accumulate_account_hashes,
-                BatchSize::SmallInput,
-            );
-        });
     }
 }
 
-criterion_group!(benches, bench_hash_account, bench_accounts_delta_hash);
+criterion_group!(benches, bench_hash_account,);
 criterion_main!(benches);

accounts-db/src/lib.rs

@@ -40,6 +40,7 @@ mod rolling_bit_field;
 pub mod shared_buffer_reader;
 pub mod sorted_storages;
 pub mod stake_rewards;
+pub mod starting_snapshot_storages;
 pub mod storable_accounts;
 pub mod tiered_storage;
 pub mod utils;

accounts-db/src/starting_snapshot_storages.rs

@@ -0,0 +1,19 @@
+use {crate::accounts_db::AccountStorageEntry, std::sync::Arc};
+
+/// Snapshot storages that the node loaded from
+///
+/// This is used to support fastboot.  Since fastboot reuses existing storages, we must carefully
+/// handle the storages used to load at startup.  If we do not handle these storages properly,
+/// restarting from the same local state (i.e. bank snapshot) may fail.
+#[derive(Debug)]
+pub enum StartingSnapshotStorages {
+    /// Starting from genesis has no storages yet
+    Genesis,
+    /// Starting from a snapshot archive always extracts the storages from the archive, so no
+    /// special handling is necessary to preserve them.
+    Archive,
+    /// Starting from local state must preserve the loaded storages.  These storages must *not* be
+    /// recycled or removed prior to taking the next snapshot, otherwise restarting from the same
+    /// bank snapshot may fail.
+    Fastboot(Vec<Arc<AccountStorageEntry>>),
+}