removes SIZE_OF_DATA_SHRED_PAYLOAD

Author: behzadnouri

core/benches/shredder.rs

@@ -9,13 +9,16 @@ use {
     solana_entry::entry::{create_ticks, Entry},
     solana_ledger::shred::{
         max_entries_per_n_shred, max_ticks_per_n_shreds, ProcessShredsStats, Shred, ShredFlags,
-        Shredder, MAX_DATA_SHREDS_PER_FEC_BLOCK, SIZE_OF_DATA_SHRED_PAYLOAD,
+        Shredder, MAX_DATA_SHREDS_PER_FEC_BLOCK,
     },
     solana_perf::test_tx,
     solana_sdk::{hash::Hash, packet::PACKET_DATA_SIZE, signature::Keypair},
     test::Bencher,
 };
 
+// Equivalent to ledger::shred::legacy::ShredData::CAPACITY.
+const LEGACY_SHERD_DATA_CAPACITY: usize = 1051;
+
 // Copied these values here to avoid exposing shreds
 // internals only for the sake of benchmarks.
 
@@ -38,12 +41,11 @@ fn make_large_unchained_entries(txs_per_entry: u64, num_entries: u64) -> Vec<Ent
 }
 
 fn make_shreds(num_shreds: usize) -> Vec<Shred> {
-    let shred_size = SIZE_OF_DATA_SHRED_PAYLOAD;
     let txs_per_entry = 128;
     let num_entries = max_entries_per_n_shred(
         &make_test_entry(txs_per_entry),
         2 * num_shreds as u64,
-        Some(shred_size),
+        Some(LEGACY_SHERD_DATA_CAPACITY),
     );
     let entries = make_large_unchained_entries(txs_per_entry, num_entries);
     let shredder = Shredder::new(1, 0, 0, 0).unwrap();
@@ -73,10 +75,10 @@ fn make_concatenated_shreds(num_shreds: usize) -> Vec<u8> {
 #[bench]
 fn bench_shredder_ticks(bencher: &mut Bencher) {
     let kp = Keypair::new();
-    let shred_size = SIZE_OF_DATA_SHRED_PAYLOAD;
+    let shred_size = LEGACY_SHERD_DATA_CAPACITY;
     let num_shreds = ((1000 * 1000) + (shred_size - 1)) / shred_size;
     // ~1Mb
-    let num_ticks = max_ticks_per_n_shreds(1, Some(SIZE_OF_DATA_SHRED_PAYLOAD)) * num_shreds as u64;
+    let num_ticks = max_ticks_per_n_shreds(1, Some(LEGACY_SHERD_DATA_CAPACITY)) * num_shreds as u64;
     let entries = create_ticks(num_ticks, 0, Hash::default());
     bencher.iter(|| {
         let shredder = Shredder::new(1, 0, 0, 0).unwrap();
@@ -87,7 +89,7 @@ fn bench_shredder_ticks(bencher: &mut Bencher) {
 #[bench]
 fn bench_shredder_large_entries(bencher: &mut Bencher) {
     let kp = Keypair::new();
-    let shred_size = SIZE_OF_DATA_SHRED_PAYLOAD;
+    let shred_size = LEGACY_SHERD_DATA_CAPACITY;
     let num_shreds = ((1000 * 1000) + (shred_size - 1)) / shred_size;
     let txs_per_entry = 128;
     let num_entries = max_entries_per_n_shred(
@@ -106,7 +108,7 @@ fn bench_shredder_large_entries(bencher: &mut Bencher) {
 #[bench]
 fn bench_deshredder(bencher: &mut Bencher) {
     let kp = Keypair::new();
-    let shred_size = SIZE_OF_DATA_SHRED_PAYLOAD;
+    let shred_size = LEGACY_SHERD_DATA_CAPACITY;
     // ~10Mb
     let num_shreds = ((10000 * 1000) + (shred_size - 1)) / shred_size;
     let num_ticks = max_ticks_per_n_shreds(1, Some(shred_size)) * num_shreds as u64;
@@ -121,7 +123,7 @@ fn bench_deshredder(bencher: &mut Bencher) {
 
 #[bench]
 fn bench_deserialize_hdr(bencher: &mut Bencher) {
-    let data = vec![0; SIZE_OF_DATA_SHRED_PAYLOAD];
+    let data = vec![0; LEGACY_SHERD_DATA_CAPACITY];
 
     let shred = Shred::new_from_data(2, 1, 1, &data, ShredFlags::LAST_SHRED_IN_SLOT, 0, 0, 1);
 

core/src/replay_stage.rs

@@ -3180,7 +3180,7 @@ pub(crate) mod tests {
             create_new_tmp_ledger,
             genesis_utils::{create_genesis_config, create_genesis_config_with_leader},
             get_tmp_ledger_path,
-            shred::{Shred, ShredFlags, SIZE_OF_DATA_SHRED_PAYLOAD},
+            shred::{Shred, ShredFlags},
         },
         solana_rpc::{
             optimistically_confirmed_bank_tracker::OptimisticallyConfirmedBank,
@@ -3779,7 +3779,7 @@ pub(crate) mod tests {
     fn test_dead_fork_entry_deserialize_failure() {
         // Insert entry that causes deserialization failure
         let res = check_dead_fork(|_, bank| {
-            let gibberish = [0xa5u8; SIZE_OF_DATA_SHRED_PAYLOAD];
+            let gibberish = [0xa5u8; /*legacy data-shred capacity:*/1051];
             let parent_offset = bank.slot() - bank.parent_slot();
             let shred = Shred::new_from_data(
                 bank.slot(),

ledger/benches/sigverify_shreds.rs

@@ -3,7 +3,7 @@
 extern crate test;
 use {
     solana_ledger::{
-        shred::{Shred, ShredFlags, SIZE_OF_DATA_SHRED_PAYLOAD},
+        shred::{Shred, ShredFlags},
         sigverify_shreds::{sign_shreds_cpu, sign_shreds_gpu, sign_shreds_gpu_pinned_keypair},
     },
     solana_perf::{
@@ -32,7 +32,7 @@ fn bench_sigverify_shreds_sign_gpu(bencher: &mut Bencher) {
             slot,
             0xc0de,
             0xdead,
-            &[5; SIZE_OF_DATA_SHRED_PAYLOAD],
+            &[5; /*legacy data-shred capacity:*/1051],
             ShredFlags::LAST_SHRED_IN_SLOT,
             1,
             2,
@@ -63,7 +63,7 @@ fn bench_sigverify_shreds_sign_cpu(bencher: &mut Bencher) {
             slot,
             0xc0de,
             0xdead,
-            &[5; SIZE_OF_DATA_SHRED_PAYLOAD],
+            &[5; /*legacy data-shred capacity:*/1051],
             ShredFlags::LAST_SHRED_IN_SLOT,
             1,
             2,

ledger/src/shred.rs

@@ -65,7 +65,6 @@ use {
     solana_sdk::{
         clock::Slot,
         hash::{hashv, Hash},
-        packet::PACKET_DATA_SIZE,
         pubkey::Pubkey,
         signature::{Keypair, Signature, Signer},
     },
@@ -98,15 +97,6 @@ pub const SIZE_OF_NONCE: usize = 4;
 const_assert_eq!(SIZE_OF_CODING_SHRED_HEADERS, 89);
 const SIZE_OF_CODING_SHRED_HEADERS: usize =
     SIZE_OF_COMMON_SHRED_HEADER + SIZE_OF_CODING_SHRED_HEADER;
-// XXX This is only true for the legacy shreds;
-// Maximum size of data that a data-shred may contain (excluding headers).
-const_assert_eq!(SIZE_OF_DATA_SHRED_PAYLOAD, 1051);
-// XXX rename DataShred::capacity
-pub const SIZE_OF_DATA_SHRED_PAYLOAD: usize = PACKET_DATA_SIZE
-    - SIZE_OF_COMMON_SHRED_HEADER
-    - SIZE_OF_DATA_SHRED_HEADER
-    - SIZE_OF_CODING_SHRED_HEADERS
-    - SIZE_OF_NONCE;
 const_assert_eq!(SHRED_DATA_OFFSET, 88);
 const SHRED_DATA_OFFSET: usize = SIZE_OF_COMMON_SHRED_HEADER + SIZE_OF_DATA_SHRED_HEADER;
 
@@ -662,7 +652,8 @@ pub fn max_entries_per_n_shred(
     num_shreds: u64,
     shred_data_size: Option<usize>,
 ) -> u64 {
-    let shred_data_size = shred_data_size.unwrap_or(SIZE_OF_DATA_SHRED_PAYLOAD) as u64;
+    let data_buffer_size = ShredData::capacity(/*merkle_proof_size:*/ None).unwrap();
+    let shred_data_size = shred_data_size.unwrap_or(data_buffer_size) as u64;
     let vec_size = bincode::serialized_size(&vec![entry]).unwrap();
     let entry_size = bincode::serialized_size(entry).unwrap();
     let count_size = vec_size - entry_size;
@@ -936,7 +927,7 @@ mod tests {
             let seed = <[u8; 32]>::try_from(bs58_decode(SEED)).unwrap();
             ChaChaRng::from_seed(seed)
         };
-        let mut data = [0u8; SIZE_OF_DATA_SHRED_PAYLOAD];
+        let mut data = [0u8; legacy::ShredData::CAPACITY];
         rng.fill(&mut data[..]);
         let keypair = Keypair::generate(&mut rng);
         let mut shred = Shred::new_from_data(

ledger/src/shredder.rs

@@ -1,7 +1,6 @@
 use {
     crate::shred::{
-        Error, ProcessShredsStats, Shred, ShredFlags, MAX_DATA_SHREDS_PER_FEC_BLOCK,
-        SIZE_OF_DATA_SHRED_PAYLOAD,
+        Error, ProcessShredsStats, Shred, ShredData, ShredFlags, MAX_DATA_SHREDS_PER_FEC_BLOCK,
     },
     lazy_static::lazy_static,
     rayon::{prelude::*, ThreadPool},
@@ -110,9 +109,9 @@ impl Shredder {
         serialize_time.stop();
 
         let mut gen_data_time = Measure::start("shred_gen_data_time");
-        let payload_capacity = SIZE_OF_DATA_SHRED_PAYLOAD;
+        let data_buffer_size = ShredData::capacity(/*merkle_proof_size:*/ None).unwrap();
         // Integer division to ensure we have enough shreds to fit all the data
-        let num_shreds = (serialized_shreds.len() + payload_capacity - 1) / payload_capacity;
+        let num_shreds = (serialized_shreds.len() + data_buffer_size - 1) / data_buffer_size;
         let last_shred_index = next_shred_index + num_shreds as u32 - 1;
         // 1) Generate data shreds
         let make_data_shred = |shred_index: u32, data| {
@@ -141,7 +140,7 @@ impl Shredder {
         };
         let data_shreds: Vec<Shred> = PAR_THREAD_POOL.install(|| {
             serialized_shreds
-                .par_chunks(payload_capacity)
+                .par_chunks(data_buffer_size)
                 .enumerate()
                 .map(|(i, shred_data)| {
                     let shred_index = next_shred_index + i as u32;
@@ -341,7 +340,8 @@ impl Shredder {
             // For backward compatibility. This is needed when the data shred
             // payload is None, so that deserializing to Vec<Entry> results in
             // an empty vector.
-            Ok(vec![0u8; SIZE_OF_DATA_SHRED_PAYLOAD])
+            let data_buffer_size = ShredData::capacity(/*merkle_proof_size:*/ None).unwrap();
+            Ok(vec![0u8; data_buffer_size])
         } else {
             Ok(data)
         }
@@ -401,13 +401,13 @@ mod tests {
             })
             .collect();
 
-        let size = serialized_size(&entries).unwrap();
+        let size = serialized_size(&entries).unwrap() as usize;
         // Integer division to ensure we have enough shreds to fit all the data
-        let payload_capacity = SIZE_OF_DATA_SHRED_PAYLOAD as u64;
-        let num_expected_data_shreds = (size + payload_capacity - 1) / payload_capacity;
+        let data_buffer_size = ShredData::capacity(/*merkle_proof_size:*/ None).unwrap();
+        let num_expected_data_shreds = (size + data_buffer_size - 1) / data_buffer_size;
         let num_expected_coding_shreds = (2 * MAX_DATA_SHREDS_PER_FEC_BLOCK as usize)
-            .saturating_sub(num_expected_data_shreds as usize)
-            .max(num_expected_data_shreds as usize);
+            .saturating_sub(num_expected_data_shreds)
+            .max(num_expected_data_shreds);
         let start_index = 0;
         let (data_shreds, coding_shreds) = shredder.entries_to_shreds(
             &keypair,
@@ -417,14 +417,14 @@ mod tests {
             start_index, // next_code_index
         );
         let next_index = data_shreds.last().unwrap().index() + 1;
-        assert_eq!(next_index as u64, num_expected_data_shreds);
+        assert_eq!(next_index as usize, num_expected_data_shreds);
 
         let mut data_shred_indexes = HashSet::new();
         let mut coding_shred_indexes = HashSet::new();
         for shred in data_shreds.iter() {
             assert_eq!(shred.shred_type(), ShredType::Data);
             let index = shred.index();
-            let is_last = index as u64 == num_expected_data_shreds - 1;
+            let is_last = index as usize == num_expected_data_shreds - 1;
             verify_test_data_shred(
                 shred,
                 index,
@@ -455,7 +455,7 @@ mod tests {
             assert!(coding_shred_indexes.contains(&i));
         }
 
-        assert_eq!(data_shred_indexes.len() as u64, num_expected_data_shreds);
+        assert_eq!(data_shred_indexes.len(), num_expected_data_shreds);
         assert_eq!(coding_shred_indexes.len(), num_expected_coding_shreds);
 
         // Test reassembly
@@ -573,8 +573,8 @@ mod tests {
         let keypair = Arc::new(Keypair::new());
         let shredder = Shredder::new(slot, slot - 5, 0, 0).unwrap();
         // Create enough entries to make > 1 shred
-        let payload_capacity = SIZE_OF_DATA_SHRED_PAYLOAD;
-        let num_entries = max_ticks_per_n_shreds(1, Some(payload_capacity)) + 1;
+        let data_buffer_size = ShredData::capacity(/*merkle_proof_size:*/ None).unwrap();
+        let num_entries = max_ticks_per_n_shreds(1, Some(data_buffer_size)) + 1;
         let entries: Vec<_> = (0..num_entries)
             .map(|_| {
                 let keypair0 = Keypair::new();
@@ -622,9 +622,9 @@ mod tests {
         let entry = Entry::new(&Hash::default(), 1, vec![tx0]);
 
         let num_data_shreds: usize = 5;
-        let payload_capacity = SIZE_OF_DATA_SHRED_PAYLOAD;
+        let data_buffer_size = ShredData::capacity(/*merkle_proof_size:*/ None).unwrap();
         let num_entries =
-            max_entries_per_n_shred(&entry, num_data_shreds as u64, Some(payload_capacity));
+            max_entries_per_n_shred(&entry, num_data_shreds as u64, Some(data_buffer_size));
         let entries: Vec<_> = (0..num_entries)
             .map(|_| {
                 let keypair0 = Keypair::new();

ledger/src/sigverify_shreds.rs

@@ -459,7 +459,7 @@ pub fn sign_shreds_gpu(
 pub mod tests {
     use {
         super::*,
-        crate::shred::{Shred, ShredFlags, SIZE_OF_DATA_SHRED_PAYLOAD},
+        crate::shred::{Shred, ShredFlags},
         solana_sdk::signature::{Keypair, Signer},
     };
 
@@ -636,7 +636,7 @@ pub mod tests {
                 slot,
                 0xc0de,
                 i as u16,
-                &[5; SIZE_OF_DATA_SHRED_PAYLOAD],
+                &[5; /*legacy data-shred capacity:*/1051],
                 ShredFlags::LAST_SHRED_IN_SLOT,
                 1,
                 2,

ledger/tests/shred.rs

@@ -3,7 +3,7 @@ use {
     solana_entry::entry::Entry,
     solana_ledger::shred::{
         max_entries_per_n_shred, verify_test_data_shred, Shred, Shredder,
-        MAX_DATA_SHREDS_PER_FEC_BLOCK, SIZE_OF_DATA_SHRED_PAYLOAD,
+        MAX_DATA_SHREDS_PER_FEC_BLOCK,
     },
     solana_sdk::{
         clock::Slot,
@@ -34,7 +34,7 @@ fn test_multi_fec_block_coding() {
     let num_entries = max_entries_per_n_shred(
         &entry,
         num_data_shreds as u64,
-        Some(SIZE_OF_DATA_SHRED_PAYLOAD),
+        Some(/*legacy data-shred capacity:*/ 1051),
     );
 
     let entries: Vec<_> = (0..num_entries)
@@ -200,7 +200,7 @@ fn setup_different_sized_fec_blocks(
     let num_entries = max_entries_per_n_shred(
         &entry,
         num_shreds_per_iter as u64,
-        Some(SIZE_OF_DATA_SHRED_PAYLOAD),
+        Some(/*legacy data-shred capacity:*/ 1051),
     );
     let entries: Vec<_> = (0..num_entries)
         .map(|_| {