diff --git a/.github/scripts/combine-benchmarks.sh b/.github/scripts/combine-benchmarks.sh index ccccb3fd53fcf..a9bd27308459d 100755 --- a/.github/scripts/combine-benchmarks.sh +++ b/.github/scripts/combine-benchmarks.sh @@ -126,7 +126,7 @@ EOF FIRST_FILE=1 SYSTEM_INFO="" -for bench_file in "forge_test_bench.md" "forge_build_bench.md" "forge_coverage_bench.md"; do +for bench_file in "forge_test_bench.md" "forge_isolate_test_bench.md" "forge_build_bench.md" "forge_coverage_bench.md"; do if [ -f "$OUTPUT_DIR/$bench_file" ]; then echo "Processing $bench_file..." @@ -135,6 +135,9 @@ for bench_file in "forge_test_bench.md" "forge_build_bench.md" "forge_coverage_b "forge_test_bench.md") SECTION_NAME="Forge Test" ;; + "forge_isolate_test_bench.md") + SECTION_NAME="Forge Test (Isolated)" + ;; "forge_build_bench.md") SECTION_NAME="Forge Build" ;; diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml index 699d6287fb476..45f76cff8e11e 100644 --- a/.github/workflows/benchmarks.yml +++ b/.github/workflows/benchmarks.yml @@ -83,6 +83,20 @@ jobs: --benchmarks forge_test,forge_fuzz_test \ --output-file forge_test_bench.md + - name: Run forge isolate test benchmarks + env: + FOUNDRY_DIR: ${{ github.workspace }}/.foundry + run: | + VERSIONS="${{ github.event.inputs.versions || 'stable,nightly' }}" + # Isolate tests default to Vectorized/solady but can be overridden + REPOS="${{ github.event.inputs.repos || env.VECTORIZED_SOLADY }}" + + ./target/release/foundry-bench --output-dir ./benches --force-install \ + --versions $VERSIONS \ + --repos $REPOS \ + --benchmarks forge_isolate_test \ + --output-file forge_isolate_test_bench.md + - name: Run forge build benchmarks env: FOUNDRY_DIR: ${{ github.workspace }}/.foundry @@ -124,6 +138,7 @@ jobs: name: benchmark-results path: | benches/forge_test_bench.md + benches/forge_isolate_test_bench.md benches/forge_build_bench.md benches/forge_coverage_bench.md benches/LATEST.md diff --git a/benches/LATEST.md b/benches/LATEST.md index 9a140dda70573..d8e6b9de10807 100644 --- a/benches/LATEST.md +++ b/benches/LATEST.md @@ -1,6 +1,6 @@ # 📊 Foundry Benchmark Results -**Generated at**: 2025-07-17 14:25:08 UTC +**Generated at**: 2025-07-23 09:01:16 UTC ## Forge Test @@ -11,19 +11,33 @@ ### Foundry Versions - **stable**: forge Version: 1.2.3-stable (a813a2c 2025-06-08) -- **nightly**: forge Version: 1.3.0-nightly (0af4341 2025-07-17) +- **nightly**: forge Version: 1.3.0-nightly (cb8f3bf 2025-07-23) | Repository | stable | nightly | |------------|----------|----------| -| ithacaxyz-account | 5.12 s | 5.15 s | -| solady | 2.99 s | 2.87 s | +| ithacaxyz-account | 5.44 s | 4.91 s | +| solady | 3.02 s | 2.76 s | ## Forge Fuzz Test | Repository | stable | nightly | |------------|----------|----------| -| ithacaxyz-account | 5.45 s | 5.20 s | -| solady | 3.08 s | 3.00 s | +| ithacaxyz-account | 5.27 s | 4.90 s | +| solady | 2.89 s | 2.82 s | + +## Forge Test (Isolated) + +### Repositories Tested + +1. [Vectorized/solady](https://github.com/Vectorized/solady) +### Foundry Versions + +- **stable**: forge Version: 1.2.3-stable (a813a2c 2025-06-08) +- **nightly**: forge Version: 1.3.0-nightly (cb8f3bf 2025-07-23) + +| Repository | stable | nightly | +|------------|----------|----------| +| solady | 3.21 s | 2.88 s | ## Forge Build @@ -34,21 +48,21 @@ ### Foundry Versions - **stable**: forge Version: 1.2.3-stable (a813a2c 2025-06-08) -- **nightly**: forge Version: 1.3.0-nightly (0af4341 2025-07-17) +- **nightly**: forge Version: 1.3.0-nightly (cb8f3bf 2025-07-23) ### No Cache | Repository | stable | nightly | |------------|----------|----------| -| ithacaxyz-account | 2.58 s | 2.83 s | -| solady | 3.88 s | 3.98 s | +| ithacaxyz-account | 2.58 s | 2.79 s | +| solady | 3.79 s | 3.89 s | ### With Cache | Repository | stable | nightly | |------------|----------|----------| -| ithacaxyz-account | 0.210 s | 0.443 s | -| solady | 0.093 s | 0.195 s | +| ithacaxyz-account | 0.209 s | 0.458 s | +| solady | 0.096 s | 0.226 s | ## Forge Coverage @@ -58,11 +72,11 @@ ### Foundry Versions - **stable**: forge Version: 1.2.3-stable (a813a2c 2025-06-08) -- **nightly**: forge Version: 1.3.0-nightly (0af4341 2025-07-17) +- **nightly**: forge Version: 1.3.0-nightly (cb8f3bf 2025-07-23) | Repository | stable | nightly | |------------|----------|----------| -| ithacaxyz-account | 22.03 s | 21.98 s | +| ithacaxyz-account | 21.88 s | 22.52 s | ## System Information diff --git a/benches/src/lib.rs b/benches/src/lib.rs index aa9c6ad318c54..c8bb1e2329767 100644 --- a/benches/src/lib.rs +++ b/benches/src/lib.rs @@ -355,6 +355,25 @@ impl BenchmarkProject { ) } + /// Benchmark forge test with --isolate flag + pub fn bench_forge_isolate_test( + &self, + version: &str, + runs: u32, + verbose: bool, + ) -> Result { + // Build before running tests + self.hyperfine( + "forge_isolate_test", + version, + "forge test --isolate", + runs, + Some("forge build"), + None, + verbose, + ) + } + /// Get the root path of the project pub fn root(&self) -> &Path { &self.root_path @@ -374,6 +393,7 @@ impl BenchmarkProject { "forge_build_with_cache" => self.bench_forge_build_with_cache(version, runs, verbose), "forge_fuzz_test" => self.bench_forge_fuzz_test(version, runs, verbose), "forge_coverage" => self.bench_forge_coverage(version, runs, verbose), + "forge_isolate_test" => self.bench_forge_isolate_test(version, runs, verbose), _ => eyre::bail!("Unknown benchmark: {}", benchmark), } } diff --git a/benches/src/main.rs b/benches/src/main.rs index aca0e069d582e..f3361bf30b6e6 100644 --- a/benches/src/main.rs +++ b/benches/src/main.rs @@ -10,12 +10,13 @@ use foundry_common::sh_println; use rayon::prelude::*; use std::{fs, path::PathBuf, process::Command, sync::Mutex}; -const ALL_BENCHMARKS: [&str; 5] = [ +const ALL_BENCHMARKS: [&str; 6] = [ "forge_test", "forge_build_no_cache", "forge_build_with_cache", "forge_fuzz_test", "forge_coverage", + "forge_isolate_test", ]; /// Foundry Benchmark Runner diff --git a/benches/src/results.rs b/benches/src/results.rs index 3bc42685a42d5..69d4746c9e1f9 100644 --- a/benches/src/results.rs +++ b/benches/src/results.rs @@ -231,6 +231,7 @@ pub fn format_benchmark_name(name: &str) -> String { "forge_build_with_cache" => "Forge Build (With Cache)", "forge_fuzz_test" => "Forge Fuzz Test", "forge_coverage" => "Forge Coverage", + "forge_isolate_test" => "Forge Test (Isolated)", _ => name, } .to_string()