From 1e9e070eab92b3860144631c0bc26f28ad5e2c05 Mon Sep 17 00:00:00 2001 From: Yash Atreya <44857776+yash-atreya@users.noreply.github.com> Date: Fri, 18 Jul 2025 12:02:51 +0530 Subject: [PATCH 1/8] feat(`benches`): forge test --isolate --- .github/scripts/combine-benchmarks.sh | 8 ++++++++ .github/workflows/benchmarks.yml | 2 +- benches/src/lib.rs | 20 ++++++++++++++++++++ benches/src/main.rs | 3 ++- benches/src/results.rs | 1 + 5 files changed, 32 insertions(+), 2 deletions(-) diff --git a/.github/scripts/combine-benchmarks.sh b/.github/scripts/combine-benchmarks.sh index ccccb3fd53fcf..76e4a4a21814c 100755 --- a/.github/scripts/combine-benchmarks.sh +++ b/.github/scripts/combine-benchmarks.sh @@ -163,6 +163,14 @@ for bench_file in "forge_test_bench.md" "forge_build_bench.md" "forge_coverage_b echo >> "$OUTPUT_DIR/LATEST.md" extract_benchmark_table "$OUTPUT_DIR/$bench_file" "Forge Fuzz Test" >> "$OUTPUT_DIR/LATEST.md" fi + + # Check if Forge Test (Isolated) section exists + if grep -q "^## Forge Test (Isolated)" "$OUTPUT_DIR/$bench_file"; then + echo >> "$OUTPUT_DIR/LATEST.md" + echo "## Forge Test (Isolated)" >> "$OUTPUT_DIR/LATEST.md" + echo >> "$OUTPUT_DIR/LATEST.md" + extract_benchmark_table "$OUTPUT_DIR/$bench_file" "Forge Test (Isolated)" >> "$OUTPUT_DIR/LATEST.md" + fi elif [[ "$bench_file" == "forge_build_bench.md" ]]; then # Extract No Cache table echo "### No Cache" >> "$OUTPUT_DIR/LATEST.md" diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml index c664680b31daa..c35a7d0dbfa6b 100644 --- a/.github/workflows/benchmarks.yml +++ b/.github/workflows/benchmarks.yml @@ -86,7 +86,7 @@ jobs: ./foundry-bench --output-dir ./benches --force-install \ --versions $VERSIONS \ --repos $REPOS \ - --benchmarks forge_test,forge_fuzz_test \ + --benchmarks forge_test,forge_fuzz_test,forge_isolate_test \ --output-file forge_test_bench.md - name: Upload test benchmark results diff --git a/benches/src/lib.rs b/benches/src/lib.rs index aa9c6ad318c54..c8bb1e2329767 100644 --- a/benches/src/lib.rs +++ b/benches/src/lib.rs @@ -355,6 +355,25 @@ impl BenchmarkProject { ) } + /// Benchmark forge test with --isolate flag + pub fn bench_forge_isolate_test( + &self, + version: &str, + runs: u32, + verbose: bool, + ) -> Result { + // Build before running tests + self.hyperfine( + "forge_isolate_test", + version, + "forge test --isolate", + runs, + Some("forge build"), + None, + verbose, + ) + } + /// Get the root path of the project pub fn root(&self) -> &Path { &self.root_path @@ -374,6 +393,7 @@ impl BenchmarkProject { "forge_build_with_cache" => self.bench_forge_build_with_cache(version, runs, verbose), "forge_fuzz_test" => self.bench_forge_fuzz_test(version, runs, verbose), "forge_coverage" => self.bench_forge_coverage(version, runs, verbose), + "forge_isolate_test" => self.bench_forge_isolate_test(version, runs, verbose), _ => eyre::bail!("Unknown benchmark: {}", benchmark), } } diff --git a/benches/src/main.rs b/benches/src/main.rs index f9fb27d5ac04a..920ef546e612b 100644 --- a/benches/src/main.rs +++ b/benches/src/main.rs @@ -10,12 +10,13 @@ use foundry_common::sh_println; use rayon::prelude::*; use std::{fs, path::PathBuf, process::Command, sync::Mutex}; -const ALL_BENCHMARKS: [&str; 5] = [ +const ALL_BENCHMARKS: [&str; 6] = [ "forge_test", "forge_build_no_cache", "forge_build_with_cache", "forge_fuzz_test", "forge_coverage", + "forge_isolate_test", ]; /// Foundry Benchmark Runner diff --git a/benches/src/results.rs b/benches/src/results.rs index 3bc42685a42d5..69d4746c9e1f9 100644 --- a/benches/src/results.rs +++ b/benches/src/results.rs @@ -231,6 +231,7 @@ pub fn format_benchmark_name(name: &str) -> String { "forge_build_with_cache" => "Forge Build (With Cache)", "forge_fuzz_test" => "Forge Fuzz Test", "forge_coverage" => "Forge Coverage", + "forge_isolate_test" => "Forge Test (Isolated)", _ => name, } .to_string() From d168e00ef605fa578f7491a7f1215bfc21619f4b Mon Sep 17 00:00:00 2001 From: Yash Atreya <44857776+yash-atreya@users.noreply.github.com> Date: Fri, 18 Jul 2025 12:04:03 +0530 Subject: [PATCH 2/8] run on pr to test --- .github/workflows/benchmarks.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml index c35a7d0dbfa6b..5b36d4e333a7a 100644 --- a/.github/workflows/benchmarks.yml +++ b/.github/workflows/benchmarks.yml @@ -1,6 +1,8 @@ name: Foundry Benchmarks on: + pull_request: + types: [opened, synchronize, reopened] workflow_dispatch: inputs: pr_number: From 6ed3e99d496f9b39e31b8f769a8b71bd1903d909 Mon Sep 17 00:00:00 2001 From: Yash Atreya <44857776+yash-atreya@users.noreply.github.com> Date: Fri, 18 Jul 2025 15:03:44 +0530 Subject: [PATCH 3/8] bench isolate separately --- .github/scripts/combine-benchmarks.sh | 13 +++------- .github/workflows/benchmarks.yml | 35 +++++++++++++++++++++++++-- 2 files changed, 37 insertions(+), 11 deletions(-) diff --git a/.github/scripts/combine-benchmarks.sh b/.github/scripts/combine-benchmarks.sh index 76e4a4a21814c..2fee3fca1c2e6 100755 --- a/.github/scripts/combine-benchmarks.sh +++ b/.github/scripts/combine-benchmarks.sh @@ -126,7 +126,7 @@ EOF FIRST_FILE=1 SYSTEM_INFO="" -for bench_file in "forge_test_bench.md" "forge_build_bench.md" "forge_coverage_bench.md"; do +for bench_file in "forge_test_bench.md" "forge_build_bench.md" "forge_coverage_bench.md" "forge_isolate_test_bench.md"; do if [ -f "$OUTPUT_DIR/$bench_file" ]; then echo "Processing $bench_file..." @@ -135,6 +135,9 @@ for bench_file in "forge_test_bench.md" "forge_build_bench.md" "forge_coverage_b "forge_test_bench.md") SECTION_NAME="Forge Test" ;; + "forge_isolate_test_bench.md") + SECTION_NAME="Forge Test (Isolated)" + ;; "forge_build_bench.md") SECTION_NAME="Forge Build" ;; @@ -163,14 +166,6 @@ for bench_file in "forge_test_bench.md" "forge_build_bench.md" "forge_coverage_b echo >> "$OUTPUT_DIR/LATEST.md" extract_benchmark_table "$OUTPUT_DIR/$bench_file" "Forge Fuzz Test" >> "$OUTPUT_DIR/LATEST.md" fi - - # Check if Forge Test (Isolated) section exists - if grep -q "^## Forge Test (Isolated)" "$OUTPUT_DIR/$bench_file"; then - echo >> "$OUTPUT_DIR/LATEST.md" - echo "## Forge Test (Isolated)" >> "$OUTPUT_DIR/LATEST.md" - echo >> "$OUTPUT_DIR/LATEST.md" - extract_benchmark_table "$OUTPUT_DIR/$bench_file" "Forge Test (Isolated)" >> "$OUTPUT_DIR/LATEST.md" - fi elif [[ "$bench_file" == "forge_build_bench.md" ]]; then # Extract No Cache table echo "### No Cache" >> "$OUTPUT_DIR/LATEST.md" diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml index 5b36d4e333a7a..4fd51b8dbe50e 100644 --- a/.github/workflows/benchmarks.yml +++ b/.github/workflows/benchmarks.yml @@ -88,7 +88,7 @@ jobs: ./foundry-bench --output-dir ./benches --force-install \ --versions $VERSIONS \ --repos $REPOS \ - --benchmarks forge_test,forge_fuzz_test,forge_isolate_test \ + --benchmarks forge_test,forge_fuzz_test \ --output-file forge_test_bench.md - name: Upload test benchmark results @@ -157,9 +157,40 @@ jobs: name: forge-coverage-results path: benches/forge_coverage_bench.md + forge-isolate-test-bench: + name: Forge Isolate Test Benchmarks + needs: setup + runs-on: foundry-runner + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Benchmark setup + uses: ./.github/actions/benchmark-setup + + - name: Run forge isolate test benchmarks + env: + FOUNDRY_DIR: ${{ github.workspace }}/.foundry + run: | + VERSIONS="${{ github.event.inputs.versions || 'stable,nightly' }}" + # Isolate tests default to Vectorized/solady but can be overridden + REPOS="${{ github.event.inputs.repos || env.VECTORIZED_SOLADY }}" + + ./foundry-bench --output-dir ./benches --force-install \ + --versions $VERSIONS \ + --repos $REPOS \ + --benchmarks forge_isolate_test \ + --output-file forge_isolate_test_bench.md + + - name: Upload isolate test benchmark results + uses: actions/upload-artifact@v4 + with: + name: forge-isolate-test-results + path: benches/forge_isolate_test_bench.md + combine-results: name: Combine and Publish Results - needs: [forge-test-bench, forge-build-bench, forge-coverage-bench] + needs: [forge-test-bench, forge-build-bench, forge-coverage-bench, forge-isolate-test-bench] runs-on: foundry-runner steps: - name: Checkout repository From ab205a0462aa03e3366a414a4bba402c2fec65a1 Mon Sep 17 00:00:00 2001 From: GitHub Action Date: Fri, 18 Jul 2025 09:47:06 +0000 Subject: [PATCH 4/8] chore(`benches`): update benchmark results MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 🤖 Generated with [Foundry Benchmarks](https://github.com/foundry-rs/foundry/actions) Co-Authored-By: github-actions --- benches/LATEST.md | 40 +++++++++++++++++++++++++++------------- 1 file changed, 27 insertions(+), 13 deletions(-) diff --git a/benches/LATEST.md b/benches/LATEST.md index e4f600ad96626..078331727b4c0 100644 --- a/benches/LATEST.md +++ b/benches/LATEST.md @@ -1,6 +1,6 @@ # 📊 Foundry Benchmark Results -**Generated at**: 2025-07-11 11:48:21 UTC +**Generated at**: 2025-07-18 09:47:02 UTC ## Forge Test @@ -11,19 +11,19 @@ ### Foundry Versions - **stable**: forge Version: 1.2.3-stable (a813a2c 2025-06-08) -- **nightly**: forge Version: 1.2.3-nightly (d592b3e 2025-07-11) +- **nightly**: forge Version: 1.3.0-nightly (5eb910c 2025-07-18) | Repository | stable | nightly | |------------|----------|----------| -| ithacaxyz-account | 6.86 s | 6.03 s | -| solady | 7.70 s | 6.94 s | +| ithacaxyz-account | 5.84 s | 6.44 s | +| solady | 6.90 s | 6.98 s | ## Forge Fuzz Test | Repository | stable | nightly | |------------|----------|----------| -| ithacaxyz-account | 6.65 s | 6.28 s | -| solady | 7.87 s | 6.98 s | +| ithacaxyz-account | 6.13 s | 6.58 s | +| solady | 7.72 s | 7.27 s | ## Forge Build @@ -34,21 +34,21 @@ ### Foundry Versions - **stable**: forge Version: 1.2.3-stable (a813a2c 2025-06-08) -- **nightly**: forge Version: 1.2.3-nightly (d592b3e 2025-07-11) +- **nightly**: forge Version: 1.3.0-nightly (5eb910c 2025-07-18) ### No Cache | Repository | stable | nightly | |------------|----------|----------| -| ithacaxyz-account | 2.57 s | 2.61 s | -| solady | 3.95 s | 4.08 s | +| ithacaxyz-account | 2.59 s | 2.82 s | +| solady | 4.03 s | 4.06 s | ### With Cache | Repository | stable | nightly | |------------|----------|----------| -| ithacaxyz-account | 2.58 s | 2.60 s | -| solady | 4.04 s | 4.07 s | +| ithacaxyz-account | 2.59 s | 2.83 s | +| solady | 4.10 s | 3.99 s | ## Forge Coverage @@ -58,11 +58,25 @@ ### Foundry Versions - **stable**: forge Version: 1.2.3-stable (a813a2c 2025-06-08) -- **nightly**: forge Version: 1.2.3-nightly (d592b3e 2025-07-11) +- **nightly**: forge Version: 1.3.0-nightly (5eb910c 2025-07-18) | Repository | stable | nightly | |------------|----------|----------| -| ithacaxyz-account | 21.49 s | 20.85 s | +| ithacaxyz-account | 20.54 s | 21.97 s | + +## Forge Test (Isolated) + +### Repositories Tested + +1. [Vectorized/solady](https://github.com/Vectorized/solady) +### Foundry Versions + +- **stable**: forge Version: 1.2.3-stable (a813a2c 2025-06-08) +- **nightly**: forge Version: 1.3.0-nightly (5eb910c 2025-07-18) + +| Repository | stable | nightly | +|------------|----------|----------| +| solady | 3.24 s | 3.03 s | ## System Information From 51cfe3b86a4d8461ad13e9cf6add2551f64a4c2e Mon Sep 17 00:00:00 2001 From: GitHub Action Date: Wed, 23 Jul 2025 08:26:18 +0000 Subject: [PATCH 5/8] chore(`benches`): update benchmark results MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 🤖 Generated with [Foundry Benchmarks](https://github.com/foundry-rs/foundry/actions) Co-Authored-By: github-actions --- benches/LATEST.md | 62 ++++++++++++++++++++++++++++------------------- 1 file changed, 37 insertions(+), 25 deletions(-) diff --git a/benches/LATEST.md b/benches/LATEST.md index c25abe6fcd130..681cca63ac16b 100644 --- a/benches/LATEST.md +++ b/benches/LATEST.md @@ -1,6 +1,6 @@ # 📊 Foundry Benchmark Results -**Generated at**: 2025-07-17 14:25:08 UTC +**Generated at**: 2025-07-23 08:26:14 UTC ## Forge Test @@ -8,23 +8,22 @@ 1. [ithacaxyz/account](https://github.com/ithacaxyz/account) 2. [Vectorized/solady](https://github.com/Vectorized/solady) - ### Foundry Versions - **stable**: forge Version: 1.2.3-stable (a813a2c 2025-06-08) -- **nightly**: forge Version: 1.3.0-nightly (0af4341 2025-07-17) +- **nightly**: forge Version: 1.3.0-nightly (cb8f3bf 2025-07-23) -| Repository | stable | nightly | -| ----------------- | ------ | ------- | -| ithacaxyz-account | 5.12 s | 5.15 s | -| solady | 2.99 s | 2.87 s | +| Repository | stable | nightly | +|------------|----------|----------| +| ithacaxyz-account | 5.05 s | 5.01 s | +| solady | 2.86 s | 2.72 s | ## Forge Fuzz Test -| Repository | stable | nightly | -| ----------------- | ------ | ------- | -| ithacaxyz-account | 5.45 s | 5.20 s | -| solady | 3.08 s | 3.00 s | +| Repository | stable | nightly | +|------------|----------|----------| +| ithacaxyz-account | 5.53 s | 4.94 s | +| solady | 2.92 s | 2.77 s | ## Forge Build @@ -32,43 +31,56 @@ 1. [ithacaxyz/account](https://github.com/ithacaxyz/account) 2. [Vectorized/solady](https://github.com/Vectorized/solady) - ### Foundry Versions - **stable**: forge Version: 1.2.3-stable (a813a2c 2025-06-08) -- **nightly**: forge Version: 1.3.0-nightly (0af4341 2025-07-17) +- **nightly**: forge Version: 1.3.0-nightly (cb8f3bf 2025-07-23) ### No Cache -| Repository | stable | nightly | -| ----------------- | ------ | ------- | -| ithacaxyz-account | 2.58 s | 2.83 s | -| solady | 3.88 s | 3.98 s | +| Repository | stable | nightly | +|------------|----------|----------| +| ithacaxyz-account | 2.59 s | 2.86 s | +| solady | 3.83 s | 3.97 s | ### With Cache -| Repository | stable | nightly | -| ----------------- | ------- | ------- | -| ithacaxyz-account | 0.210 s | 0.443 s | -| solady | 0.093 s | 0.195 s | +| Repository | stable | nightly | +|------------|----------|----------| +| ithacaxyz-account | 0.197 s | 0.423 s | +| solady | 0.089 s | 0.189 s | ## Forge Coverage ### Repositories Tested 1. [ithacaxyz/account](https://github.com/ithacaxyz/account) +### Foundry Versions + +- **stable**: forge Version: 1.2.3-stable (a813a2c 2025-06-08) +- **nightly**: forge Version: 1.3.0-nightly (cb8f3bf 2025-07-23) +| Repository | stable | nightly | +|------------|----------|----------| +| ithacaxyz-account | 21.90 s | 21.59 s | + +## Forge Test (Isolated) + +### Repositories Tested + +1. [Vectorized/solady](https://github.com/Vectorized/solady) ### Foundry Versions - **stable**: forge Version: 1.2.3-stable (a813a2c 2025-06-08) -- **nightly**: forge Version: 1.3.0-nightly (0af4341 2025-07-17) +- **nightly**: forge Version: 1.3.0-nightly (cb8f3bf 2025-07-23) -| Repository | stable | nightly | -| ----------------- | ------- | ------- | -| ithacaxyz-account | 22.03 s | 21.98 s | +| Repository | stable | nightly | +|------------|----------|----------| +| solady | 3.03 s | 2.81 s | ## System Information + - **OS**: linux - **CPU**: 8 - **Rustc**: rustc 1.88.0 (6b00bc388 2025-06-23) From df74703e6be92f462af261a53ad25c68660f0c8b Mon Sep 17 00:00:00 2001 From: Yash Atreya <44857776+yash-atreya@users.noreply.github.com> Date: Wed, 23 Jul 2025 14:20:54 +0530 Subject: [PATCH 6/8] reorder --- .github/scripts/combine-benchmarks.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/scripts/combine-benchmarks.sh b/.github/scripts/combine-benchmarks.sh index 2fee3fca1c2e6..a9bd27308459d 100755 --- a/.github/scripts/combine-benchmarks.sh +++ b/.github/scripts/combine-benchmarks.sh @@ -126,7 +126,7 @@ EOF FIRST_FILE=1 SYSTEM_INFO="" -for bench_file in "forge_test_bench.md" "forge_build_bench.md" "forge_coverage_bench.md" "forge_isolate_test_bench.md"; do +for bench_file in "forge_test_bench.md" "forge_isolate_test_bench.md" "forge_build_bench.md" "forge_coverage_bench.md"; do if [ -f "$OUTPUT_DIR/$bench_file" ]; then echo "Processing $bench_file..." From 5b5bc6882820fe8aed321b8476ae5465f556acb1 Mon Sep 17 00:00:00 2001 From: GitHub Action Date: Wed, 23 Jul 2025 09:01:20 +0000 Subject: [PATCH 7/8] chore(`benches`): update benchmark results MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 🤖 Generated with [Foundry Benchmarks](https://github.com/foundry-rs/foundry/actions) Co-Authored-By: github-actions --- benches/LATEST.md | 48 +++++++++++++++++++++++------------------------ 1 file changed, 24 insertions(+), 24 deletions(-) diff --git a/benches/LATEST.md b/benches/LATEST.md index 681cca63ac16b..d8e6b9de10807 100644 --- a/benches/LATEST.md +++ b/benches/LATEST.md @@ -1,6 +1,6 @@ # 📊 Foundry Benchmark Results -**Generated at**: 2025-07-23 08:26:14 UTC +**Generated at**: 2025-07-23 09:01:16 UTC ## Forge Test @@ -15,60 +15,60 @@ | Repository | stable | nightly | |------------|----------|----------| -| ithacaxyz-account | 5.05 s | 5.01 s | -| solady | 2.86 s | 2.72 s | +| ithacaxyz-account | 5.44 s | 4.91 s | +| solady | 3.02 s | 2.76 s | ## Forge Fuzz Test | Repository | stable | nightly | |------------|----------|----------| -| ithacaxyz-account | 5.53 s | 4.94 s | -| solady | 2.92 s | 2.77 s | +| ithacaxyz-account | 5.27 s | 4.90 s | +| solady | 2.89 s | 2.82 s | -## Forge Build +## Forge Test (Isolated) ### Repositories Tested -1. [ithacaxyz/account](https://github.com/ithacaxyz/account) -2. [Vectorized/solady](https://github.com/Vectorized/solady) +1. [Vectorized/solady](https://github.com/Vectorized/solady) ### Foundry Versions - **stable**: forge Version: 1.2.3-stable (a813a2c 2025-06-08) - **nightly**: forge Version: 1.3.0-nightly (cb8f3bf 2025-07-23) -### No Cache - | Repository | stable | nightly | |------------|----------|----------| -| ithacaxyz-account | 2.59 s | 2.86 s | -| solady | 3.83 s | 3.97 s | +| solady | 3.21 s | 2.88 s | -### With Cache - -| Repository | stable | nightly | -|------------|----------|----------| -| ithacaxyz-account | 0.197 s | 0.423 s | -| solady | 0.089 s | 0.189 s | - -## Forge Coverage +## Forge Build ### Repositories Tested 1. [ithacaxyz/account](https://github.com/ithacaxyz/account) +2. [Vectorized/solady](https://github.com/Vectorized/solady) ### Foundry Versions - **stable**: forge Version: 1.2.3-stable (a813a2c 2025-06-08) - **nightly**: forge Version: 1.3.0-nightly (cb8f3bf 2025-07-23) +### No Cache + | Repository | stable | nightly | |------------|----------|----------| -| ithacaxyz-account | 21.90 s | 21.59 s | +| ithacaxyz-account | 2.58 s | 2.79 s | +| solady | 3.79 s | 3.89 s | -## Forge Test (Isolated) +### With Cache + +| Repository | stable | nightly | +|------------|----------|----------| +| ithacaxyz-account | 0.209 s | 0.458 s | +| solady | 0.096 s | 0.226 s | + +## Forge Coverage ### Repositories Tested -1. [Vectorized/solady](https://github.com/Vectorized/solady) +1. [ithacaxyz/account](https://github.com/ithacaxyz/account) ### Foundry Versions - **stable**: forge Version: 1.2.3-stable (a813a2c 2025-06-08) @@ -76,7 +76,7 @@ | Repository | stable | nightly | |------------|----------|----------| -| solady | 3.03 s | 2.81 s | +| ithacaxyz-account | 21.88 s | 22.52 s | ## System Information From 75b2e8f8da31ee0ef768f0c734b91c2a6b38cc09 Mon Sep 17 00:00:00 2001 From: Yash Atreya <44857776+yash-atreya@users.noreply.github.com> Date: Wed, 23 Jul 2025 15:15:35 +0530 Subject: [PATCH 8/8] nit --- .github/workflows/benchmarks.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml index efbe67fe66527..45f76cff8e11e 100644 --- a/.github/workflows/benchmarks.yml +++ b/.github/workflows/benchmarks.yml @@ -1,8 +1,6 @@ name: Foundry Benchmarks on: - pull_request: - types: [opened, synchronize, reopened] workflow_dispatch: inputs: pr_number: