From ab6041a2a59f6d6e8009b1ca095b9dd38e491e57 Mon Sep 17 00:00:00 2001 From: Pavlo Khrystenko Date: Thu, 15 May 2025 17:39:32 +0200 Subject: [PATCH 01/10] attempt to fix caching --- crates/compilers/src/buildinfo.rs | 28 ++++--- crates/compilers/src/cache.rs | 7 ++ crates/compilers/src/compile/project.rs | 50 +++++------- .../src/compilers/resolc/compiler.rs | 8 +- crates/compilers/src/report/mod.rs | 77 ++----------------- crates/compilers/tests/project.rs | 20 ++++- 6 files changed, 75 insertions(+), 115 deletions(-) diff --git a/crates/compilers/src/buildinfo.rs b/crates/compilers/src/buildinfo.rs index d71363111..a726291d3 100644 --- a/crates/compilers/src/buildinfo.rs +++ b/crates/compilers/src/buildinfo.rs @@ -22,8 +22,9 @@ pub struct BuildInfo { pub id: String, #[serde(rename = "_format")] pub format: String, - pub solc_version: Version, - pub solc_long_version: Version, + pub input_version: Version, + pub input_version_long: Version, + pub compiler_version: Version, pub input: I, pub output: O, } @@ -92,20 +93,24 @@ impl RawBuildInfo { pub fn new, E: CompilationError, C: CompilerContract>( input: &I, output: &CompilerOutput, + compiler_version: &semver::Version, full_build_info: bool, ) -> Result { - let version = input.version().clone(); + let input_version = input.version().clone(); let build_context = BuildContext::new(input, output)?; let mut hasher = md5::Md5::new(); hasher.update(ETHERS_FORMAT_VERSION); - let solc_short = format!("{}.{}.{}", version.major, version.minor, version.patch); - hasher.update(&solc_short); - hasher.update(version.to_string()); + let input_version_short = + format!("{}.{}.{}", input_version.major, input_version.minor, input_version.patch); + hasher.update(&input_version_short); + hasher.update(&compiler_version.to_string()); + hasher.update(input_version.to_string()); let input = serde_json::to_value(input)?; + hasher.update(&serde_json::to_string(&input)?); hasher.update(&serde_json::to_string(&output)?); @@ -119,8 +124,12 @@ impl RawBuildInfo { if full_build_info { build_info.insert("_format".to_string(), serde_json::to_value(ETHERS_FORMAT_VERSION)?); - build_info.insert("solcVersion".to_string(), serde_json::to_value(&solc_short)?); - build_info.insert("solcLongVersion".to_string(), serde_json::to_value(&version)?); + build_info + .insert("compilerVersion".to_string(), serde_json::to_value(&compiler_version)?); + build_info + .insert("inputVersion".to_string(), serde_json::to_value(&input_version_short)?); + build_info + .insert("inputVersionLong".to_string(), serde_json::to_value(&input_version)?); build_info.insert("input".to_string(), input); build_info.insert("output".to_string(), serde_json::to_value(output)?); } @@ -146,7 +155,8 @@ mod tests { v, ); let output = CompilerOutput::::default(); - let raw_info = RawBuildInfo::new(&input, &output, true).unwrap(); + let raw_info = + RawBuildInfo::new(&input, &output, &semver::Version::new(0, 0, 0), true).unwrap(); let _info: BuildInfo> = serde_json::from_str(&serde_json::to_string(&raw_info).unwrap()).unwrap(); } diff --git a/crates/compilers/src/cache.rs b/crates/compilers/src/cache.rs index 8b5696de2..7abd3ae7c 100644 --- a/crates/compilers/src/cache.rs +++ b/crates/compilers/src/cache.rs @@ -187,6 +187,13 @@ impl CompilerCache { .builds .contains(build_id.file_name().to_string_lossy().trim_end_matches(".json")) { + self.builds.remove( + &build_id + .file_name() + .to_string_lossy() + .trim_end_matches(".json") + .to_owned(), + ); let _ = std::fs::remove_file(build_id.path()); } } diff --git a/crates/compilers/src/compile/project.rs b/crates/compilers/src/compile/project.rs index 7b57f39c8..6acef476b 100644 --- a/crates/compilers/src/compile/project.rs +++ b/crates/compilers/src/compile/project.rs @@ -438,16 +438,23 @@ impl CompilerSources<'_, L, S> { /// Filters out all sources that don't need to be compiled, see [`ArtifactsCache::filter`] fn filter< T: ArtifactOutput, - C: Compiler, + C: Compiler, >( &mut self, cache: &mut ArtifactsCache<'_, T, C>, ) { cache.remove_dirty_sources(); - for versioned_sources in self.sources.values_mut() { - for (version, sources, (profile, _)) in versioned_sources { + for (language, versioned_sources) in self.sources.iter_mut() { + for (version, sources, (profile, settings)) in versioned_sources { + let input = C::Input::build( + sources.clone(), + settings.clone(), + language.clone(), + version.clone(), + ); + let version = cache.project().compiler.compiler_version(&input); trace!("Filtering {} sources for {}", sources.len(), version); - cache.filter(sources, version, profile); + cache.filter(sources, &version, profile); trace!( "Detected {} sources to compile {:?}", sources.dirty().count(), @@ -540,14 +547,14 @@ impl CompilerSources<'_, L, S> { let mut aggregated = AggregatedCompilerOutput::default(); for (input, mut output, profile, actually_dirty) in results { - let version = input.version(); + let version = &project.compiler.compiler_version(&input); // Mark all files as seen by the compiler for file in &actually_dirty { cache.compiler_seen(file); } - let build_info = RawBuildInfo::new(&input, &output, project.build_info)?; + let build_info = RawBuildInfo::new(&input, &output, &version, project.build_info)?; output.retain_files( actually_dirty @@ -573,25 +580,14 @@ fn compile_sequential<'a, C: Compiler>( jobs.into_iter() .map(|(input, profile, actually_dirty)| { let start = Instant::now(); - let versions = { - let compiler_ver = compiler.compiler_version(&input); - if &compiler_ver == input.version() { - vec![input.version().clone()] - } else { - vec![compiler.compiler_version(&input), input.version().clone()] - } - }; + let version = compiler.compiler_version(&input); report::compiler_spawn( &compiler.compiler_name(&input), - versions.as_ref(), + &version, actually_dirty.as_slice(), ); let output = compiler.compile(&input)?; - report::compiler_success( - &compiler.compiler_name(&input), - versions.as_ref(), - &start.elapsed(), - ); + report::compiler_success(&compiler.compiler_name(&input), &version, &start.elapsed()); Ok((input, output, profile, actually_dirty)) }) @@ -617,24 +613,18 @@ fn compile_parallel<'a, C: Compiler>( .map(move |(input, profile, actually_dirty)| { // set the reporter on this thread let _guard = report::set_scoped(&scoped_report); - let versions = { - let compiler_ver = compiler.compiler_version(&input); - if &compiler_ver == input.version() { - vec![input.version().clone()] - } else { - vec![compiler.compiler_version(&input), input.version().clone()] - } - }; + let version = compiler.compiler_version(&input); + let start = Instant::now(); report::compiler_spawn( &compiler.compiler_name(&input), - versions.as_ref(), + &version, actually_dirty.as_slice(), ); compiler.compile(&input).map(move |output| { report::compiler_success( &compiler.compiler_name(&input), - versions.as_ref(), + &version, &start.elapsed(), ); (input, output, profile, actually_dirty) diff --git a/crates/compilers/src/compilers/resolc/compiler.rs b/crates/compilers/src/compilers/resolc/compiler.rs index a14b61397..d776b3bd1 100644 --- a/crates/compilers/src/compilers/resolc/compiler.rs +++ b/crates/compilers/src/compilers/resolc/compiler.rs @@ -2,7 +2,7 @@ use crate::{ error::{Result, SolcError}, resolver::parse::SolData, solc::{Solc, SolcCompiler, SolcSettings}, - Compiler, CompilerVersion, SimpleCompilerName, + Compiler, CompilerInput, CompilerVersion, SimpleCompilerName, }; use foundry_compilers_artifacts::{resolc::ResolcCompilerOutput, Contract, Error, SolcLanguage}; use itertools::Itertools; @@ -35,7 +35,9 @@ impl Compiler for Resolc { type Language = SolcLanguage; fn compiler_version(&self, _input: &Self::Input) -> Version { - self.resolc_version.clone() + let mut v = self.resolc_version.clone(); + v.build = semver::BuildMetadata::new(&format!("Solc.{}", _input.version())).unwrap(); + v } fn compiler_name(&self, _input: &Self::Input) -> std::borrow::Cow<'static, str> { @@ -75,7 +77,7 @@ impl Compiler for Resolc { impl SimpleCompilerName for Resolc { fn compiler_name_default() -> std::borrow::Cow<'static, str> { - "Resolc and Solc".into() + "Resolc".into() } } diff --git a/crates/compilers/src/report/mod.rs b/crates/compilers/src/report/mod.rs index 81e402b1e..4e1229571 100644 --- a/crates/compilers/src/report/mod.rs +++ b/crates/compilers/src/report/mod.rs @@ -111,37 +111,11 @@ pub trait Reporter: 'static + std::fmt::Debug { ) { } - /// Callback invoked right before [Compiler::compile] is called - /// - /// This contains the [Compiler] its [Version] and all files that triggered the compile job. The - /// dirty files are only provided to give a better feedback what was actually compiled. - /// - /// [Compiler]: crate::compilers::Compiler - /// [Compiler::compile]: crate::compilers::Compiler::compile - fn on_multicompiler_spawn( - &self, - _compiler_name: &str, - _versions: &[Version], - _dirty_files: &[PathBuf], - ) { - } - /// Invoked with the `CompilerOutput` if [`Compiler::compile()`] was successful /// /// [`Compiler::compile()`]: crate::compilers::Compiler::compile fn on_compiler_success(&self, _compiler_name: &str, _version: &Version, _duration: &Duration) {} - /// Invoked with the `CompilerOutput` if [`Compiler::compile()`] was successful - /// - /// [`Compiler::compile()`]: crate::compilers::Compiler::compile - fn on_multicompiler_success( - &self, - _compiler_name: &str, - _versions: &[Version], - _duration: &Duration, - ) { - } - /// Invoked before a new compiler version is installed fn on_solc_installation_start(&self, _version: &Version) {} @@ -198,12 +172,12 @@ impl dyn Reporter { } } -pub(crate) fn compiler_spawn(compiler_name: &str, version: &[Version], dirty_files: &[PathBuf]) { - get_default(|r| r.reporter.on_multicompiler_spawn(compiler_name, version, dirty_files)); +pub(crate) fn compiler_spawn(compiler_name: &str, version: &Version, dirty_files: &[PathBuf]) { + get_default(|r| r.reporter.on_compiler_spawn(compiler_name, version, dirty_files)); } -pub(crate) fn compiler_success(compiler_name: &str, version: &[Version], duration: &Duration) { - get_default(|r| r.reporter.on_multicompiler_success(compiler_name, version, duration)); +pub(crate) fn compiler_success(compiler_name: &str, version: &Version, duration: &Duration) { + get_default(|r| r.reporter.on_compiler_success(compiler_name, version, duration)); } #[allow(dead_code)] @@ -348,53 +322,12 @@ impl Reporter for BasicStdoutReporter { /// /// [`Compiler::compile()`]: crate::compilers::Compiler::compile fn on_compiler_spawn(&self, compiler_name: &str, version: &Version, dirty_files: &[PathBuf]) { - println!("Compiling {} files with {} {}", dirty_files.len(), compiler_name, version); - } - - fn on_multicompiler_spawn( - &self, - compiler_name: &str, - versions: &[Version], - dirty_files: &[PathBuf], - ) { - if let [version] = versions { - self.on_compiler_spawn(compiler_name, version, dirty_files); - } else { - let names = compiler_name - .split("and") - .filter(|str| !str.is_empty()) - .map(|x| x.trim()) - .zip(versions) - .map(|(name, version)| format!("{name} v{version}")) - .collect::>() - .join(", "); - println!("Compiling {} files with {}", dirty_files.len(), names); - } + println!("Compiling {} files with {compiler_name} {version}", dirty_files.len()); } fn on_compiler_success(&self, compiler_name: &str, version: &Version, duration: &Duration) { println!("{compiler_name} {version} finished in {duration:.2?}"); } - fn on_multicompiler_success( - &self, - compiler_name: &str, - versions: &[Version], - duration: &Duration, - ) { - if let [version] = versions { - self.on_compiler_success(compiler_name, version, duration); - } else { - let names = compiler_name - .split("and") - .filter(|str| !str.is_empty()) - .map(|x| x.trim()) - .zip(versions) - .map(|(name, version)| format!("{name} {version}")) - .collect::>() - .join(", "); - println!("{names} finished in {duration:.2?}"); - } - } /// Invoked before a new compiler is installed fn on_solc_installation_start(&self, version: &Version) { diff --git a/crates/compilers/tests/project.rs b/crates/compilers/tests/project.rs index 9e02fdde3..e9f44d832 100644 --- a/crates/compilers/tests/project.rs +++ b/crates/compilers/tests/project.rs @@ -4532,6 +4532,7 @@ fn test_output_hash_cache_invalidation() { let root = Path::new(env!("CARGO_MANIFEST_DIR")).join("../../test-data/dapp-sample"); let paths = ProjectPathsConfig::builder().sources(root.join("src")).lib(root.join("lib")); let mut project = TempProject::::new(paths).unwrap(); + project.project_mut().compiler = resolc(); project.project_mut().build_info = true; // First compilation - should compile everything since cache is empty. @@ -4539,10 +4540,27 @@ fn test_output_hash_cache_invalidation() { compiled.assert_success(); assert!(!compiled.is_unchanged(), "First compilation should not be cached"); + project.project_mut().compiler = MultiCompiler { + solidity: SolidityCompiler::Resolc( + Resolc::find_or_install( + &semver::Version::parse("0.1.0-dev.13").unwrap(), + SolcCompiler::default(), + ) + .unwrap(), + ), + ..Default::default() + }; + // Second compilation - should use cache since nothing changed. let compiled = project.compile().unwrap(); compiled.assert_success(); - assert!(compiled.is_unchanged(), "Second compilation should use cache"); + assert!(!compiled.is_unchanged(), "Second compilation should use cache"); + + // Second compilation - should use cache since nothing changed. + let compiled = project.compile().unwrap(); + compiled.assert_success(); + + assert!(compiled.is_unchanged(), "Third compilation should use cache"); // Adding a file to output directory should NOT invalidate cache let artifacts_path = project.project().artifacts_path(); From ba8a610f2a454c3b5e25eb74a255ad1d8016c95b Mon Sep 17 00:00:00 2001 From: Pavlo Khrystenko Date: Thu, 15 May 2025 17:42:28 +0200 Subject: [PATCH 02/10] clippy --- crates/compilers/src/buildinfo.rs | 4 ++-- crates/compilers/src/cache.rs | 9 ++------- crates/compilers/src/compile/project.rs | 10 +++------- crates/compilers/src/compilers/resolc/compiler.rs | 2 +- crates/compilers/src/compilers/solc/mod.rs | 2 +- 5 files changed, 9 insertions(+), 18 deletions(-) diff --git a/crates/compilers/src/buildinfo.rs b/crates/compilers/src/buildinfo.rs index a726291d3..8ee6db692 100644 --- a/crates/compilers/src/buildinfo.rs +++ b/crates/compilers/src/buildinfo.rs @@ -106,7 +106,7 @@ impl RawBuildInfo { let input_version_short = format!("{}.{}.{}", input_version.major, input_version.minor, input_version.patch); hasher.update(&input_version_short); - hasher.update(&compiler_version.to_string()); + hasher.update(compiler_version.to_string()); hasher.update(input_version.to_string()); let input = serde_json::to_value(input)?; @@ -125,7 +125,7 @@ impl RawBuildInfo { if full_build_info { build_info.insert("_format".to_string(), serde_json::to_value(ETHERS_FORMAT_VERSION)?); build_info - .insert("compilerVersion".to_string(), serde_json::to_value(&compiler_version)?); + .insert("compilerVersion".to_string(), serde_json::to_value(compiler_version)?); build_info .insert("inputVersion".to_string(), serde_json::to_value(&input_version_short)?); build_info diff --git a/crates/compilers/src/cache.rs b/crates/compilers/src/cache.rs index 7abd3ae7c..c271a54f6 100644 --- a/crates/compilers/src/cache.rs +++ b/crates/compilers/src/cache.rs @@ -187,13 +187,8 @@ impl CompilerCache { .builds .contains(build_id.file_name().to_string_lossy().trim_end_matches(".json")) { - self.builds.remove( - &build_id - .file_name() - .to_string_lossy() - .trim_end_matches(".json") - .to_owned(), - ); + self.builds + .remove(build_id.file_name().to_string_lossy().trim_end_matches(".json")); let _ = std::fs::remove_file(build_id.path()); } } diff --git a/crates/compilers/src/compile/project.rs b/crates/compilers/src/compile/project.rs index 6acef476b..b006d0301 100644 --- a/crates/compilers/src/compile/project.rs +++ b/crates/compilers/src/compile/project.rs @@ -446,12 +446,8 @@ impl CompilerSources<'_, L, S> { cache.remove_dirty_sources(); for (language, versioned_sources) in self.sources.iter_mut() { for (version, sources, (profile, settings)) in versioned_sources { - let input = C::Input::build( - sources.clone(), - settings.clone(), - language.clone(), - version.clone(), - ); + let input = + C::Input::build(sources.clone(), settings.clone(), *language, version.clone()); let version = cache.project().compiler.compiler_version(&input); trace!("Filtering {} sources for {}", sources.len(), version); cache.filter(sources, &version, profile); @@ -554,7 +550,7 @@ impl CompilerSources<'_, L, S> { cache.compiler_seen(file); } - let build_info = RawBuildInfo::new(&input, &output, &version, project.build_info)?; + let build_info = RawBuildInfo::new(&input, &output, version, project.build_info)?; output.retain_files( actually_dirty diff --git a/crates/compilers/src/compilers/resolc/compiler.rs b/crates/compilers/src/compilers/resolc/compiler.rs index d776b3bd1..bb1451ebf 100644 --- a/crates/compilers/src/compilers/resolc/compiler.rs +++ b/crates/compilers/src/compilers/resolc/compiler.rs @@ -227,7 +227,7 @@ impl Resolc { rvm::VersionManager::new(true).map_err(|e| SolcError::Message(e.to_string()))?; let versions: Vec = version_manager - .list_available(_solc_version.clone()) + .list_available(_solc_version) .map_err(|e| SolcError::Message(e.to_string()))? .into_iter() .filter(|x| _resolc_version.is_none_or(|version| version == x.version())) diff --git a/crates/compilers/src/compilers/solc/mod.rs b/crates/compilers/src/compilers/solc/mod.rs index 8a5741fec..dbb1b3195 100644 --- a/crates/compilers/src/compilers/solc/mod.rs +++ b/crates/compilers/src/compilers/solc/mod.rs @@ -482,7 +482,7 @@ mod tests { SolcLanguage::Solidity, v.clone(), ); - let build_info = RawBuildInfo::new(&input, &out_converted, true).unwrap(); + let build_info = RawBuildInfo::new(&input, &out_converted, &v, true).unwrap(); let mut aggregated = AggregatedCompilerOutput::::default(); aggregated.extend(v, build_info, "default", out_converted); assert!(!aggregated.is_unchanged()); From 49aebbc89418a9bcd81e44d7f882765239c42f11 Mon Sep 17 00:00:00 2001 From: Pavlo Khrystenko Date: Fri, 16 May 2025 10:12:45 +0200 Subject: [PATCH 03/10] review --- crates/compilers/src/buildinfo.rs | 3 +-- crates/compilers/src/cache.rs | 2 -- crates/compilers/src/compilers/resolc/compiler.rs | 10 +++++++++- 3 files changed, 10 insertions(+), 5 deletions(-) diff --git a/crates/compilers/src/buildinfo.rs b/crates/compilers/src/buildinfo.rs index 8ee6db692..f14564f58 100644 --- a/crates/compilers/src/buildinfo.rs +++ b/crates/compilers/src/buildinfo.rs @@ -112,9 +112,8 @@ impl RawBuildInfo { let input = serde_json::to_value(input)?; hasher.update(&serde_json::to_string(&input)?); - hasher.update(&serde_json::to_string(&output)?); - // create the hash for `{_format,solcVersion,solcLongVersion,input}` + // create the hash for `{_format,compilerVersion,inputVersion,inputLongVersion,input}` // N.B. this is not exactly the same as hashing the json representation of these values but // the must efficient one let result = hasher.finalize(); diff --git a/crates/compilers/src/cache.rs b/crates/compilers/src/cache.rs index c271a54f6..8b5696de2 100644 --- a/crates/compilers/src/cache.rs +++ b/crates/compilers/src/cache.rs @@ -187,8 +187,6 @@ impl CompilerCache { .builds .contains(build_id.file_name().to_string_lossy().trim_end_matches(".json")) { - self.builds - .remove(build_id.file_name().to_string_lossy().trim_end_matches(".json")); let _ = std::fs::remove_file(build_id.path()); } } diff --git a/crates/compilers/src/compilers/resolc/compiler.rs b/crates/compilers/src/compilers/resolc/compiler.rs index bb1451ebf..e77098da5 100644 --- a/crates/compilers/src/compilers/resolc/compiler.rs +++ b/crates/compilers/src/compilers/resolc/compiler.rs @@ -36,7 +36,15 @@ impl Compiler for Resolc { fn compiler_version(&self, _input: &Self::Input) -> Version { let mut v = self.resolc_version.clone(); - v.build = semver::BuildMetadata::new(&format!("Solc.{}", _input.version())).unwrap(); + let input_version = _input.version(); + + // Note it shouldn't fail as parsing code assumes that there can be an optional string + // that precludes the version number + v.build = semver::BuildMetadata::new(&format!( + "Solc.{}.{}.{}", + input_version.major, input_version.minor, input_version.patch + )) + .expect("Can't fail"); v } From 27ea82e41ea8827ff85a92fe572b69b9b3e42c50 Mon Sep 17 00:00:00 2001 From: Pavlo Khrystenko Date: Fri, 16 May 2025 10:15:23 +0200 Subject: [PATCH 04/10] more --- crates/compilers/src/compilers/resolc/compiler.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/crates/compilers/src/compilers/resolc/compiler.rs b/crates/compilers/src/compilers/resolc/compiler.rs index e77098da5..0bd4dceb9 100644 --- a/crates/compilers/src/compilers/resolc/compiler.rs +++ b/crates/compilers/src/compilers/resolc/compiler.rs @@ -85,6 +85,7 @@ impl Compiler for Resolc { impl SimpleCompilerName for Resolc { fn compiler_name_default() -> std::borrow::Cow<'static, str> { + // Single `Resolc` is sufficient because we now add `Solc` to `compiler_version` buildMeta. "Resolc".into() } } From a2112a27c9ec40b9221786c840eebffaf5c831f5 Mon Sep 17 00:00:00 2001 From: Pavlo Khrystenko Date: Thu, 22 May 2025 14:53:55 +0200 Subject: [PATCH 05/10] revert changes to `Compiler_version` --- crates/compilers/src/cache.rs | 5 ++ crates/compilers/src/compile/project.rs | 63 ++++++++++++--- .../src/compilers/resolc/compiler.rs | 14 +--- crates/compilers/src/report/mod.rs | 77 +++++++++++++++++-- 4 files changed, 131 insertions(+), 28 deletions(-) diff --git a/crates/compilers/src/cache.rs b/crates/compilers/src/cache.rs index 8b5696de2..42f30e79d 100644 --- a/crates/compilers/src/cache.rs +++ b/crates/compilers/src/cache.rs @@ -170,6 +170,11 @@ impl CompilerCache { { outdated.push(build_id.to_owned()); } + + let path = self.paths.build_infos.join(build_id).with_extension("json"); + if !path.exists() { + outdated.push(build_id.to_owned()); + } } for build_id in outdated { diff --git a/crates/compilers/src/compile/project.rs b/crates/compilers/src/compile/project.rs index b006d0301..0f0ad3497 100644 --- a/crates/compilers/src/compile/project.rs +++ b/crates/compilers/src/compile/project.rs @@ -448,7 +448,10 @@ impl CompilerSources<'_, L, S> { for (version, sources, (profile, settings)) in versioned_sources { let input = C::Input::build(sources.clone(), settings.clone(), *language, version.clone()); - let version = cache.project().compiler.compiler_version(&input); + let version = compound_version( + cache.project().compiler.compiler_version(&input), + &input.version(), + ); trace!("Filtering {} sources for {}", sources.len(), version); cache.filter(sources, &version, profile); trace!( @@ -543,14 +546,16 @@ impl CompilerSources<'_, L, S> { let mut aggregated = AggregatedCompilerOutput::default(); for (input, mut output, profile, actually_dirty) in results { - let version = &project.compiler.compiler_version(&input); - + let version = compound_version( + project.compiler.compiler_version(&input).clone(), + &input.version(), + ); // Mark all files as seen by the compiler for file in &actually_dirty { cache.compiler_seen(file); } - let build_info = RawBuildInfo::new(&input, &output, version, project.build_info)?; + let build_info = RawBuildInfo::new(&input, &output, &version, project.build_info)?; output.retain_files( actually_dirty @@ -576,14 +581,21 @@ fn compile_sequential<'a, C: Compiler>( jobs.into_iter() .map(|(input, profile, actually_dirty)| { let start = Instant::now(); - let version = compiler.compiler_version(&input); + let versions = { + let compiler_ver = compiler.compiler_version(&input); + if &compiler_ver == input.version() { + vec![input.version().clone()] + } else { + vec![compiler.compiler_version(&input), input.version().clone()] + } + }; report::compiler_spawn( &compiler.compiler_name(&input), - &version, + &versions, actually_dirty.as_slice(), ); let output = compiler.compile(&input)?; - report::compiler_success(&compiler.compiler_name(&input), &version, &start.elapsed()); + report::compiler_success(&compiler.compiler_name(&input), &versions, &start.elapsed()); Ok((input, output, profile, actually_dirty)) }) @@ -609,18 +621,24 @@ fn compile_parallel<'a, C: Compiler>( .map(move |(input, profile, actually_dirty)| { // set the reporter on this thread let _guard = report::set_scoped(&scoped_report); - let version = compiler.compiler_version(&input); - + let versions = { + let compiler_ver = compiler.compiler_version(&input); + if &compiler_ver == input.version() { + vec![input.version().clone()] + } else { + vec![compiler.compiler_version(&input), input.version().clone()] + } + }; let start = Instant::now(); report::compiler_spawn( &compiler.compiler_name(&input), - &version, + &versions, actually_dirty.as_slice(), ); compiler.compile(&input).map(move |output| { report::compiler_success( &compiler.compiler_name(&input), - &version, + &versions, &start.elapsed(), ); (input, output, profile, actually_dirty) @@ -630,6 +648,29 @@ fn compile_parallel<'a, C: Compiler>( }) } +fn compound_version(mut compiler_version: Version, input_version: &Version) -> Version { + if compiler_version != *input_version { + let build = if compiler_version.build.is_empty() { + semver::BuildMetadata::new(&format!( + "{}.{}.{}", + input_version.major, input_version.minor, input_version.patch, + )) + .expect("can't fail due to parsing") + } else { + semver::BuildMetadata::new(&format!( + "{}-{}.{}.{}", + compiler_version.build.as_str(), + input_version.major, + input_version.minor, + input_version.patch, + )) + .expect("can't fail due to parsing") + }; + compiler_version.build = build; + }; + compiler_version +} + #[cfg(test)] #[cfg(all(feature = "project-util", feature = "svm-solc"))] mod tests { diff --git a/crates/compilers/src/compilers/resolc/compiler.rs b/crates/compilers/src/compilers/resolc/compiler.rs index d997260dd..06a61ebb2 100644 --- a/crates/compilers/src/compilers/resolc/compiler.rs +++ b/crates/compilers/src/compilers/resolc/compiler.rs @@ -2,7 +2,7 @@ use crate::{ error::{Result, SolcError}, resolver::parse::SolData, solc::{Solc, SolcCompiler, SolcSettings}, - Compiler, CompilerInput, CompilerVersion, SimpleCompilerName, + Compiler, CompilerVersion, SimpleCompilerName, }; use foundry_compilers_artifacts::{resolc::ResolcCompilerOutput, Contract, Error, SolcLanguage}; use itertools::Itertools; @@ -35,17 +35,7 @@ impl Compiler for Resolc { type Language = SolcLanguage; fn compiler_version(&self, _input: &Self::Input) -> Version { - let mut v = self.resolc_version.clone(); - let input_version = _input.version(); - - // Note it shouldn't fail as parsing code assumes that there can be an optional string - // that precludes the version number - v.build = semver::BuildMetadata::new(&format!( - "Solc.{}.{}.{}", - input_version.major, input_version.minor, input_version.patch - )) - .expect("Can't fail"); - v + self.resolc_version.clone() } fn compiler_name(&self, _input: &Self::Input) -> std::borrow::Cow<'static, str> { diff --git a/crates/compilers/src/report/mod.rs b/crates/compilers/src/report/mod.rs index 4e1229571..81e402b1e 100644 --- a/crates/compilers/src/report/mod.rs +++ b/crates/compilers/src/report/mod.rs @@ -111,11 +111,37 @@ pub trait Reporter: 'static + std::fmt::Debug { ) { } + /// Callback invoked right before [Compiler::compile] is called + /// + /// This contains the [Compiler] its [Version] and all files that triggered the compile job. The + /// dirty files are only provided to give a better feedback what was actually compiled. + /// + /// [Compiler]: crate::compilers::Compiler + /// [Compiler::compile]: crate::compilers::Compiler::compile + fn on_multicompiler_spawn( + &self, + _compiler_name: &str, + _versions: &[Version], + _dirty_files: &[PathBuf], + ) { + } + /// Invoked with the `CompilerOutput` if [`Compiler::compile()`] was successful /// /// [`Compiler::compile()`]: crate::compilers::Compiler::compile fn on_compiler_success(&self, _compiler_name: &str, _version: &Version, _duration: &Duration) {} + /// Invoked with the `CompilerOutput` if [`Compiler::compile()`] was successful + /// + /// [`Compiler::compile()`]: crate::compilers::Compiler::compile + fn on_multicompiler_success( + &self, + _compiler_name: &str, + _versions: &[Version], + _duration: &Duration, + ) { + } + /// Invoked before a new compiler version is installed fn on_solc_installation_start(&self, _version: &Version) {} @@ -172,12 +198,12 @@ impl dyn Reporter { } } -pub(crate) fn compiler_spawn(compiler_name: &str, version: &Version, dirty_files: &[PathBuf]) { - get_default(|r| r.reporter.on_compiler_spawn(compiler_name, version, dirty_files)); +pub(crate) fn compiler_spawn(compiler_name: &str, version: &[Version], dirty_files: &[PathBuf]) { + get_default(|r| r.reporter.on_multicompiler_spawn(compiler_name, version, dirty_files)); } -pub(crate) fn compiler_success(compiler_name: &str, version: &Version, duration: &Duration) { - get_default(|r| r.reporter.on_compiler_success(compiler_name, version, duration)); +pub(crate) fn compiler_success(compiler_name: &str, version: &[Version], duration: &Duration) { + get_default(|r| r.reporter.on_multicompiler_success(compiler_name, version, duration)); } #[allow(dead_code)] @@ -322,12 +348,53 @@ impl Reporter for BasicStdoutReporter { /// /// [`Compiler::compile()`]: crate::compilers::Compiler::compile fn on_compiler_spawn(&self, compiler_name: &str, version: &Version, dirty_files: &[PathBuf]) { - println!("Compiling {} files with {compiler_name} {version}", dirty_files.len()); + println!("Compiling {} files with {} {}", dirty_files.len(), compiler_name, version); + } + + fn on_multicompiler_spawn( + &self, + compiler_name: &str, + versions: &[Version], + dirty_files: &[PathBuf], + ) { + if let [version] = versions { + self.on_compiler_spawn(compiler_name, version, dirty_files); + } else { + let names = compiler_name + .split("and") + .filter(|str| !str.is_empty()) + .map(|x| x.trim()) + .zip(versions) + .map(|(name, version)| format!("{name} v{version}")) + .collect::>() + .join(", "); + println!("Compiling {} files with {}", dirty_files.len(), names); + } } fn on_compiler_success(&self, compiler_name: &str, version: &Version, duration: &Duration) { println!("{compiler_name} {version} finished in {duration:.2?}"); } + fn on_multicompiler_success( + &self, + compiler_name: &str, + versions: &[Version], + duration: &Duration, + ) { + if let [version] = versions { + self.on_compiler_success(compiler_name, version, duration); + } else { + let names = compiler_name + .split("and") + .filter(|str| !str.is_empty()) + .map(|x| x.trim()) + .zip(versions) + .map(|(name, version)| format!("{name} {version}")) + .collect::>() + .join(", "); + println!("{names} finished in {duration:.2?}"); + } + } /// Invoked before a new compiler is installed fn on_solc_installation_start(&self, version: &Version) { From 9f32c3151ba645d10ad4064ebced8607b1ff5d8f Mon Sep 17 00:00:00 2001 From: Pavlo Khrystenko Date: Thu, 22 May 2025 15:44:05 +0200 Subject: [PATCH 06/10] fix extra recompiles --- crates/compilers/src/cache.rs | 27 +++++++++++++++---- crates/compilers/src/compile/project.rs | 4 +-- .../src/compilers/resolc/compiler.rs | 4 +-- 3 files changed, 26 insertions(+), 9 deletions(-) diff --git a/crates/compilers/src/cache.rs b/crates/compilers/src/cache.rs index 42f30e79d..1b5597b96 100644 --- a/crates/compilers/src/cache.rs +++ b/crates/compilers/src/cache.rs @@ -170,11 +170,6 @@ impl CompilerCache { { outdated.push(build_id.to_owned()); } - - let path = self.paths.build_infos.join(build_id).with_extension("json"); - if !path.exists() { - outdated.push(build_id.to_owned()); - } } for build_id in outdated { @@ -1262,6 +1257,28 @@ impl<'a, T: ArtifactOutput, C: Compiler> cache .strip_entries_prefix(project.root()) .strip_artifact_files_prefixes(project.artifacts_path()); + let mut additional_removals = vec![]; + for entry in cache + .entries() + .flat_map(|e| e.artifacts.values()) + .flat_map(|a| a.values()) + .flat_map(|a| a.values()) + { + let path = cache.paths.build_infos.join(&entry.build_id).with_extension("json"); + if !path.exists() + && !written_build_infos + .iter() + .map(|x| &x.id) + .collect::>() + .contains(&entry.build_id) + { + additional_removals.push(entry.clone()); + } + } + for entry in additional_removals { + cache.builds.remove(&entry.build_id); + cache.remove(&entry.path); + } cache.write(project.cache_path())?; } diff --git a/crates/compilers/src/compile/project.rs b/crates/compilers/src/compile/project.rs index 0f0ad3497..15d11f878 100644 --- a/crates/compilers/src/compile/project.rs +++ b/crates/compilers/src/compile/project.rs @@ -450,7 +450,7 @@ impl CompilerSources<'_, L, S> { C::Input::build(sources.clone(), settings.clone(), *language, version.clone()); let version = compound_version( cache.project().compiler.compiler_version(&input), - &input.version(), + input.version(), ); trace!("Filtering {} sources for {}", sources.len(), version); cache.filter(sources, &version, profile); @@ -548,7 +548,7 @@ impl CompilerSources<'_, L, S> { for (input, mut output, profile, actually_dirty) in results { let version = compound_version( project.compiler.compiler_version(&input).clone(), - &input.version(), + input.version(), ); // Mark all files as seen by the compiler for file in &actually_dirty { diff --git a/crates/compilers/src/compilers/resolc/compiler.rs b/crates/compilers/src/compilers/resolc/compiler.rs index 06a61ebb2..fbe4e08a4 100644 --- a/crates/compilers/src/compilers/resolc/compiler.rs +++ b/crates/compilers/src/compilers/resolc/compiler.rs @@ -243,13 +243,13 @@ impl Resolc { rvm::VersionManager::new(true).map_err(|e| SolcError::Message(e.to_string()))?; let binary = if let Some(resolc_version) = _resolc_version { if version_manager.is_installed(resolc_version) { - version_manager.get(resolc_version, _solc_version.clone()).ok() + version_manager.get(resolc_version, _solc_version).ok() } else { None } } else { let versions: Vec = version_manager - .list_available(_solc_version.clone()) + .list_available(_solc_version) .map_err(|e| SolcError::Message(e.to_string()))? .into_iter() .collect(); From 014cd16576fbdf231cbed39bf4804a00a8f13160 Mon Sep 17 00:00:00 2001 From: Pavlo Khrystenko Date: Thu, 22 May 2025 15:44:46 +0200 Subject: [PATCH 07/10] noise --- crates/compilers/src/cache.rs | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/crates/compilers/src/cache.rs b/crates/compilers/src/cache.rs index 1b5597b96..a545767f6 100644 --- a/crates/compilers/src/cache.rs +++ b/crates/compilers/src/cache.rs @@ -1258,6 +1258,8 @@ impl<'a, T: ArtifactOutput, C: Compiler> .strip_entries_prefix(project.root()) .strip_artifact_files_prefixes(project.artifacts_path()); let mut additional_removals = vec![]; + let written_builds_set = + written_build_infos.iter().map(|x| &x.id).collect::>(); for entry in cache .entries() .flat_map(|e| e.artifacts.values()) @@ -1265,13 +1267,7 @@ impl<'a, T: ArtifactOutput, C: Compiler> .flat_map(|a| a.values()) { let path = cache.paths.build_infos.join(&entry.build_id).with_extension("json"); - if !path.exists() - && !written_build_infos - .iter() - .map(|x| &x.id) - .collect::>() - .contains(&entry.build_id) - { + if !path.exists() && !written_builds_set.contains(&entry.build_id) { additional_removals.push(entry.clone()); } } From d270589c653d44a68f6c5a4a8bfef78b1354c8dd Mon Sep 17 00:00:00 2001 From: Pavlo Khrystenko Date: Thu, 22 May 2025 17:11:15 +0200 Subject: [PATCH 08/10] fix --- crates/compilers/src/cache.rs | 24 +++++------------------- 1 file changed, 5 insertions(+), 19 deletions(-) diff --git a/crates/compilers/src/cache.rs b/crates/compilers/src/cache.rs index a545767f6..7bc8eb4fc 100644 --- a/crates/compilers/src/cache.rs +++ b/crates/compilers/src/cache.rs @@ -1051,7 +1051,11 @@ impl<'a, T: ArtifactOutput, C: Compiler> if let Ok(cache) = CompilerCache::read_joined(&project.paths) { if cache.paths == paths && preprocessed == cache.preprocessed { // unchanged project paths and same preprocess cache option - return cache; + if cache.builds.iter().all(|x| { + project.paths.build_infos.join(x).with_extension("json").exists() + }) { + return cache; + } } } } @@ -1257,24 +1261,6 @@ impl<'a, T: ArtifactOutput, C: Compiler> cache .strip_entries_prefix(project.root()) .strip_artifact_files_prefixes(project.artifacts_path()); - let mut additional_removals = vec![]; - let written_builds_set = - written_build_infos.iter().map(|x| &x.id).collect::>(); - for entry in cache - .entries() - .flat_map(|e| e.artifacts.values()) - .flat_map(|a| a.values()) - .flat_map(|a| a.values()) - { - let path = cache.paths.build_infos.join(&entry.build_id).with_extension("json"); - if !path.exists() && !written_builds_set.contains(&entry.build_id) { - additional_removals.push(entry.clone()); - } - } - for entry in additional_removals { - cache.builds.remove(&entry.build_id); - cache.remove(&entry.path); - } cache.write(project.cache_path())?; } From 1ff1c8f225771f54dd596a18b514c28d7d86a5e2 Mon Sep 17 00:00:00 2001 From: Pavlo Khrystenko Date: Thu, 22 May 2025 17:32:40 +0200 Subject: [PATCH 09/10] add clear for all invalidated files --- crates/compilers/src/cache.rs | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/crates/compilers/src/cache.rs b/crates/compilers/src/cache.rs index 7bc8eb4fc..8a70fc844 100644 --- a/crates/compilers/src/cache.rs +++ b/crates/compilers/src/cache.rs @@ -20,7 +20,9 @@ use semver::Version; use serde::{de::DeserializeOwned, Deserialize, Serialize}; use std::{ collections::{btree_map::BTreeMap, hash_map, BTreeSet, HashMap, HashSet}, - fs, + ffi::OsStr, + fs::{self, read_dir, DirEntry}, + io, path::{Path, PathBuf}, time::{Duration, UNIX_EPOCH}, }; @@ -1055,6 +1057,9 @@ impl<'a, T: ArtifactOutput, C: Compiler> project.paths.build_infos.join(x).with_extension("json").exists() }) { return cache; + } else { + // clear all artifacts + let _ = std::fs::remove_dir_all(&project.paths.artifacts); } } } From 892937e8f2d5ad974dfdb5f3d3c5c5cb08b4db9e Mon Sep 17 00:00:00 2001 From: Pavlo Khrystenko Date: Thu, 22 May 2025 17:37:52 +0200 Subject: [PATCH 10/10] clippy --- crates/compilers/src/cache.rs | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/crates/compilers/src/cache.rs b/crates/compilers/src/cache.rs index 8a70fc844..2282da138 100644 --- a/crates/compilers/src/cache.rs +++ b/crates/compilers/src/cache.rs @@ -20,9 +20,7 @@ use semver::Version; use serde::{de::DeserializeOwned, Deserialize, Serialize}; use std::{ collections::{btree_map::BTreeMap, hash_map, BTreeSet, HashMap, HashSet}, - ffi::OsStr, - fs::{self, read_dir, DirEntry}, - io, + fs::{self}, path::{Path, PathBuf}, time::{Duration, UNIX_EPOCH}, };