Reduce number of blobs used in tests to speed up CI (#8194)

`beacon-chain-tests` is now regularly taking 1h+ on CI since Fulu fork was added.

This PR attemtpts to reduce the test time by bringing down the number of blobs generated in tests - instead of generating 0..max_blobs, the generator now generates 0..1 blobs by default, and this can be modified by setting `harness.execution_block_generator.set_min_blob_count(n)`.

Note: The blobs are pre-generated and doesn't require too much CPU to generate however processing a larger number of them on the beacon chain does take a lot of time.

This PR also include a few other small improvements
- Our slowest test (`chain_segment_varying_chunk_size`) runs 3x faster in Fulu just by reusing chain segments
- Avoid re-running fork specific tests on all forks
- Fix a bunch of tests that depends on the harness's existing random blob generation, which is fragile


beacon chain test time on test machine is **~2x** faster:

### `unstable`

```
Summary [ 751.586s] 291 tests run: 291 passed (13 slow), 0 skipped
```

### this branch

```
Summary [ 373.792s] 291 tests run: 291 passed (2 slow), 0 skipped
```

The next set of tests to optimise is the ones that use [`get_chain_segment`](https://github.com/sigp/lighthouse/blob/77a9af96de0f693127055e381ece3e98dceea0a8/beacon_node/beacon_chain/tests/block_verification.rs#L45), as it by default build 320 blocks with supernode - an easy optimisation would be to build these blocks with cgc = 8 for tests that only require fullnodes.


  


Co-Authored-By: Jimmy Chen <[email protected]>

Co-Authored-By: Jimmy Chen <[email protected]>

Author: jimmygchen

beacon_node/beacon_chain/src/data_availability_checker/overflow_lru_cache.rs

@@ -1279,7 +1279,7 @@ mod pending_components_tests {
         let mut rng = StdRng::seed_from_u64(0xDEADBEEF0BAD5EEDu64);
         let spec = test_spec::<E>();
         let (block, blobs_vec) =
-            generate_rand_block_and_blobs::<E>(ForkName::Deneb, NumBlobs::Random, &mut rng, &spec);
+            generate_rand_block_and_blobs::<E>(ForkName::Deneb, NumBlobs::Random, &mut rng);
         let max_len = spec.max_blobs_per_block(block.epoch()) as usize;
         let mut blobs: RuntimeFixedVector<Option<Arc<BlobSidecar<E>>>> =
             RuntimeFixedVector::default(max_len);

beacon_node/beacon_chain/src/kzg_utils.rs

@@ -468,7 +468,7 @@ mod test {
 
     #[track_caller]
     fn test_validate_data_columns(kzg: &Kzg, spec: &ChainSpec) {
-        let num_of_blobs = 6;
+        let num_of_blobs = 2;
         let (signed_block, blobs, proofs) =
             create_test_fulu_block_and_blobs::<E>(num_of_blobs, spec);
         let blob_refs = blobs.iter().collect::<Vec<_>>();
@@ -494,7 +494,8 @@ mod test {
 
     #[track_caller]
     fn test_build_data_columns(kzg: &Kzg, spec: &ChainSpec) {
-        let num_of_blobs = 6;
+        // Using at least 2 blobs to make sure we're arranging the data columns correctly.
+        let num_of_blobs = 2;
         let (signed_block, blobs, proofs) =
             create_test_fulu_block_and_blobs::<E>(num_of_blobs, spec);
 
@@ -534,6 +535,7 @@ mod test {
 
     #[track_caller]
     fn test_reconstruct_data_columns(kzg: &Kzg, spec: &ChainSpec) {
+        // Using at least 2 blobs to make sure we're arranging the data columns correctly.
         let num_of_blobs = 2;
         let (signed_block, blobs, proofs) =
             create_test_fulu_block_and_blobs::<E>(num_of_blobs, spec);
@@ -557,6 +559,7 @@ mod test {
 
     #[track_caller]
     fn test_reconstruct_data_columns_unordered(kzg: &Kzg, spec: &ChainSpec) {
+        // Using at least 2 blobs to make sure we're arranging the data columns correctly.
         let num_of_blobs = 2;
         let (signed_block, blobs, proofs) =
             create_test_fulu_block_and_blobs::<E>(num_of_blobs, spec);
@@ -578,7 +581,7 @@ mod test {
 
     #[track_caller]
     fn test_reconstruct_blobs_from_data_columns(kzg: &Kzg, spec: &ChainSpec) {
-        let num_of_blobs = 6;
+        let num_of_blobs = 3;
         let (signed_block, blobs, proofs) =
             create_test_fulu_block_and_blobs::<E>(num_of_blobs, spec);
         let blob_refs = blobs.iter().collect::<Vec<_>>();
@@ -588,7 +591,8 @@ mod test {
 
         // Now reconstruct
         let signed_blinded_block = signed_block.into();
-        let blob_indices = vec![3, 4, 5];
+        // Using at least 2 blobs to make sure we're arranging the data columns correctly.
+        let blob_indices = vec![1, 2];
         let reconstructed_blobs = reconstruct_blobs(
             kzg,
             &column_sidecars.iter().as_slice()[0..column_sidecars.len() / 2],

beacon_node/beacon_chain/src/test_utils.rs

@@ -81,6 +81,10 @@ pub const TEST_DATA_COLUMN_SIDECARS_SSZ: &[u8] =
 // a different value.
 pub const DEFAULT_TARGET_AGGREGATORS: u64 = u64::MAX;
 
+// Minimum and maximum number of blobs to generate in each slot when using the `NumBlobs::Random` option (default).
+const DEFAULT_MIN_BLOBS: usize = 1;
+const DEFAULT_MAX_BLOBS: usize = 2;
+
 static KZG: LazyLock<Arc<Kzg>> = LazyLock::new(|| {
     let kzg = Kzg::new_from_trusted_setup(&get_trusted_setup()).expect("should create kzg");
     Arc::new(kzg)
@@ -172,23 +176,28 @@ fn make_rng() -> Mutex<StdRng> {
     Mutex::new(StdRng::seed_from_u64(0x0DDB1A5E5BAD5EEDu64))
 }
 
-/// Return a `ChainSpec` suitable for test usage.
-///
-/// If the `fork_from_env` feature is enabled, read the fork to use from the FORK_NAME environment
-/// variable. Otherwise use the default spec.
-pub fn test_spec<E: EthSpec>() -> ChainSpec {
-    let mut spec = if cfg!(feature = "fork_from_env") {
+pub fn fork_name_from_env() -> Option<ForkName> {
+    if cfg!(feature = "fork_from_env") {
         let fork_name = std::env::var(FORK_NAME_ENV_VAR).unwrap_or_else(|e| {
             panic!(
                 "{} env var must be defined when using fork_from_env: {:?}",
                 FORK_NAME_ENV_VAR, e
             )
         });
-        let fork = ForkName::from_str(fork_name.as_str()).unwrap();
-        fork.make_genesis_spec(E::default_spec())
+        Some(ForkName::from_str(fork_name.as_str()).unwrap())
     } else {
-        E::default_spec()
-    };
+        None
+    }
+}
+
+/// Return a `ChainSpec` suitable for test usage.
+///
+/// If the `fork_from_env` feature is enabled, read the fork to use from the FORK_NAME environment
+/// variable. Otherwise use the default spec.
+pub fn test_spec<E: EthSpec>() -> ChainSpec {
+    let mut spec = fork_name_from_env()
+        .map(|fork| fork.make_genesis_spec(E::default_spec()))
+        .unwrap_or_else(|| E::default_spec());
 
     // Set target aggregators to a high value by default.
     spec.target_aggregators_per_committee = DEFAULT_TARGET_AGGREGATORS;
@@ -3245,96 +3254,49 @@ pub enum NumBlobs {
     None,
 }
 
+macro_rules! add_blob_transactions {
+    ($message:expr, $payload_type:ty, $num_blobs:expr, $rng:expr, $fork_name:expr) => {{
+        let num_blobs = match $num_blobs {
+            NumBlobs::Random => $rng.random_range(DEFAULT_MIN_BLOBS..=DEFAULT_MAX_BLOBS),
+            NumBlobs::Number(n) => n,
+            NumBlobs::None => 0,
+        };
+        let (bundle, transactions) =
+            execution_layer::test_utils::generate_blobs::<E>(num_blobs, $fork_name).unwrap();
+
+        let payload: &mut $payload_type = &mut $message.body.execution_payload;
+        payload.execution_payload.transactions = <_>::default();
+        for tx in Vec::from(transactions) {
+            payload.execution_payload.transactions.push(tx).unwrap();
+        }
+        $message.body.blob_kzg_commitments = bundle.commitments.clone();
+        bundle
+    }};
+}
+
 pub fn generate_rand_block_and_blobs<E: EthSpec>(
     fork_name: ForkName,
     num_blobs: NumBlobs,
     rng: &mut impl Rng,
-    spec: &ChainSpec,
 ) -> (SignedBeaconBlock<E, FullPayload<E>>, Vec<BlobSidecar<E>>) {
     let inner = map_fork_name!(fork_name, BeaconBlock, <_>::random_for_test(rng));
 
     let mut block = SignedBeaconBlock::from_block(inner, types::Signature::random_for_test(rng));
-    let max_blobs = spec.max_blobs_per_block(block.epoch()) as usize;
     let mut blob_sidecars = vec![];
 
     let bundle = match block {
         SignedBeaconBlock::Deneb(SignedBeaconBlockDeneb {
             ref mut message, ..
-        }) => {
-            // Get either zero blobs or a random number of blobs between 1 and Max Blobs.
-            let payload: &mut FullPayloadDeneb<E> = &mut message.body.execution_payload;
-            let num_blobs = match num_blobs {
-                NumBlobs::Random => rng.random_range(1..=max_blobs),
-                NumBlobs::Number(n) => n,
-                NumBlobs::None => 0,
-            };
-            let (bundle, transactions) =
-                execution_layer::test_utils::generate_blobs::<E>(num_blobs, fork_name).unwrap();
-
-            payload.execution_payload.transactions = <_>::default();
-            for tx in Vec::from(transactions) {
-                payload.execution_payload.transactions.push(tx).unwrap();
-            }
-            message.body.blob_kzg_commitments = bundle.commitments.clone();
-            bundle
-        }
+        }) => add_blob_transactions!(message, FullPayloadDeneb<E>, num_blobs, rng, fork_name),
         SignedBeaconBlock::Electra(SignedBeaconBlockElectra {
             ref mut message, ..
-        }) => {
-            // Get either zero blobs or a random number of blobs between 1 and Max Blobs.
-            let payload: &mut FullPayloadElectra<E> = &mut message.body.execution_payload;
-            let num_blobs = match num_blobs {
-                NumBlobs::Random => rng.random_range(1..=max_blobs),
-                NumBlobs::Number(n) => n,
-                NumBlobs::None => 0,
-            };
-            let (bundle, transactions) =
-                execution_layer::test_utils::generate_blobs::<E>(num_blobs, fork_name).unwrap();
-            payload.execution_payload.transactions = <_>::default();
-            for tx in Vec::from(transactions) {
-                payload.execution_payload.transactions.push(tx).unwrap();
-            }
-            message.body.blob_kzg_commitments = bundle.commitments.clone();
-            bundle
-        }
+        }) => add_blob_transactions!(message, FullPayloadElectra<E>, num_blobs, rng, fork_name),
         SignedBeaconBlock::Fulu(SignedBeaconBlockFulu {
             ref mut message, ..
-        }) => {
-            // Get either zero blobs or a random number of blobs between 1 and Max Blobs.
-            let payload: &mut FullPayloadFulu<E> = &mut message.body.execution_payload;
-            let num_blobs = match num_blobs {
-                NumBlobs::Random => rng.random_range(1..=max_blobs),
-                NumBlobs::Number(n) => n,
-                NumBlobs::None => 0,
-            };
-            let (bundle, transactions) =
-                execution_layer::test_utils::generate_blobs::<E>(num_blobs, fork_name).unwrap();
-            payload.execution_payload.transactions = <_>::default();
-            for tx in Vec::from(transactions) {
-                payload.execution_payload.transactions.push(tx).unwrap();
-            }
-            message.body.blob_kzg_commitments = bundle.commitments.clone();
-            bundle
-        }
+        }) => add_blob_transactions!(message, FullPayloadFulu<E>, num_blobs, rng, fork_name),
         SignedBeaconBlock::Gloas(SignedBeaconBlockGloas {
             ref mut message, ..
-        }) => {
-            // Get either zero blobs or a random number of blobs between 1 and Max Blobs.
-            let payload: &mut FullPayloadGloas<E> = &mut message.body.execution_payload;
-            let num_blobs = match num_blobs {
-                NumBlobs::Random => rng.random_range(1..=max_blobs),
-                NumBlobs::Number(n) => n,
-                NumBlobs::None => 0,
-            };
-            let (bundle, transactions) =
-                execution_layer::test_utils::generate_blobs::<E>(num_blobs, fork_name).unwrap();
-            payload.execution_payload.transactions = <_>::default();
-            for tx in Vec::from(transactions) {
-                payload.execution_payload.transactions.push(tx).unwrap();
-            }
-            message.body.blob_kzg_commitments = bundle.commitments.clone();
-            bundle
-        }
+        }) => add_blob_transactions!(message, FullPayloadGloas<E>, num_blobs, rng, fork_name),
         _ => return (block, blob_sidecars),
     };
 
@@ -3375,7 +3337,7 @@ pub fn generate_rand_block_and_data_columns<E: EthSpec>(
     SignedBeaconBlock<E, FullPayload<E>>,
     DataColumnSidecarList<E>,
 ) {
-    let (block, _blobs) = generate_rand_block_and_blobs(fork_name, num_blobs, rng, spec);
+    let (block, _blobs) = generate_rand_block_and_blobs(fork_name, num_blobs, rng);
     let data_columns = generate_data_column_sidecars_from_block(&block, spec);
     (block, data_columns)
 }

beacon_node/beacon_chain/tests/block_verification.rs

@@ -297,19 +297,20 @@ async fn chain_segment_full_segment() {
 
 #[tokio::test]
 async fn chain_segment_varying_chunk_size() {
-    for chunk_size in &[1, 2, 3, 5, 31, 32, 33, 42] {
+    let (chain_segment, chain_segment_blobs) = get_chain_segment().await;
+    let blocks: Vec<RpcBlock<E>> = chain_segment_blocks(&chain_segment, &chain_segment_blobs)
+        .into_iter()
+        .collect();
+
+    for chunk_size in &[1, 2, 31, 32, 33] {
         let harness = get_harness(VALIDATOR_COUNT, NodeCustodyType::Fullnode);
-        let (chain_segment, chain_segment_blobs) = get_chain_segment().await;
-        let blocks: Vec<RpcBlock<E>> = chain_segment_blocks(&chain_segment, &chain_segment_blobs)
-            .into_iter()
-            .collect();
 
         harness
             .chain
             .slot_clock
             .set_slot(blocks.last().unwrap().slot().as_u64());
 
-        for chunk in blocks.chunks(*chunk_size) {
+        for chunk in blocks.clone().chunks(*chunk_size) {
             harness
                 .chain
                 .process_chain_segment(chunk.to_vec(), NotifyExecutionLayer::Yes)

beacon_node/beacon_chain/tests/events.rs

@@ -1,20 +1,26 @@
 use beacon_chain::blob_verification::GossipVerifiedBlob;
 use beacon_chain::data_column_verification::GossipVerifiedDataColumn;
-use beacon_chain::test_utils::{BeaconChainHarness, generate_data_column_sidecars_from_block};
+use beacon_chain::test_utils::{
+    BeaconChainHarness, fork_name_from_env, generate_data_column_sidecars_from_block, test_spec,
+};
 use eth2::types::{EventKind, SseBlobSidecar, SseDataColumnSidecar};
 use rand::SeedableRng;
 use rand::rngs::StdRng;
 use std::sync::Arc;
 use types::blob_sidecar::FixedBlobSidecarList;
 use types::test_utils::TestRandom;
-use types::{BlobSidecar, DataColumnSidecar, EthSpec, ForkName, MinimalEthSpec, Slot};
+use types::{BlobSidecar, DataColumnSidecar, EthSpec, MinimalEthSpec, Slot};
 
 type E = MinimalEthSpec;
 
 /// Verifies that a blob event is emitted when a gossip verified blob is received via gossip or the publish block API.
 #[tokio::test]
 async fn blob_sidecar_event_on_process_gossip_blob() {
-    let spec = Arc::new(ForkName::Deneb.make_genesis_spec(E::default_spec()));
+    if fork_name_from_env().is_some_and(|f| !f.deneb_enabled() || f.fulu_enabled()) {
+        return;
+    };
+
+    let spec = Arc::new(test_spec::<E>());
     let harness = BeaconChainHarness::builder(E::default())
         .spec(spec)
         .deterministic_keypairs(8)
@@ -48,7 +54,11 @@ async fn blob_sidecar_event_on_process_gossip_blob() {
 /// Verifies that a data column event is emitted when a gossip verified data column is received via gossip or the publish block API.
 #[tokio::test]
 async fn data_column_sidecar_event_on_process_gossip_data_column() {
-    let spec = Arc::new(ForkName::Fulu.make_genesis_spec(E::default_spec()));
+    if fork_name_from_env().is_some_and(|f| !f.fulu_enabled()) {
+        return;
+    };
+
+    let spec = Arc::new(test_spec::<E>());
     let harness = BeaconChainHarness::builder(E::default())
         .spec(spec)
         .deterministic_keypairs(8)
@@ -93,7 +103,11 @@ async fn data_column_sidecar_event_on_process_gossip_data_column() {
 /// Verifies that a blob event is emitted when blobs are received via RPC.
 #[tokio::test]
 async fn blob_sidecar_event_on_process_rpc_blobs() {
-    let spec = Arc::new(ForkName::Deneb.make_genesis_spec(E::default_spec()));
+    if fork_name_from_env().is_some_and(|f| !f.deneb_enabled() || f.fulu_enabled()) {
+        return;
+    };
+
+    let spec = Arc::new(test_spec::<E>());
     let harness = BeaconChainHarness::builder(E::default())
         .spec(spec)
         .deterministic_keypairs(8)
@@ -112,7 +126,7 @@ async fn blob_sidecar_event_on_process_rpc_blobs() {
     let slot = head_state.slot() + 1;
     let ((signed_block, opt_blobs), _) = harness.make_block(head_state, slot).await;
     let (kzg_proofs, blobs) = opt_blobs.unwrap();
-    assert!(blobs.len() > 2);
+    assert_eq!(blobs.len(), 2);
 
     let blob_1 =
         Arc::new(BlobSidecar::new(0, blobs[0].clone(), &signed_block, kzg_proofs[0]).unwrap());
@@ -144,7 +158,11 @@ async fn blob_sidecar_event_on_process_rpc_blobs() {
 
 #[tokio::test]
 async fn data_column_sidecar_event_on_process_rpc_columns() {
-    let spec = Arc::new(ForkName::Fulu.make_genesis_spec(E::default_spec()));
+    if fork_name_from_env().is_some_and(|f| !f.fulu_enabled()) {
+        return;
+    };
+
+    let spec = Arc::new(test_spec::<E>());
     let harness = BeaconChainHarness::builder(E::default())
         .spec(spec.clone())
         .deterministic_keypairs(8)