From 59cb00ffbe198c52881750ffb907daed728567e0 Mon Sep 17 00:00:00 2001 From: bit-aloo Date: Sat, 21 Jun 2025 13:25:12 +0530 Subject: [PATCH 01/10] add caching fields inside Bootstrap command --- src/bootstrap/src/utils/exec.rs | 27 ++++++++++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) diff --git a/src/bootstrap/src/utils/exec.rs b/src/bootstrap/src/utils/exec.rs index a7b92441d747e..28b8fc2734e01 100644 --- a/src/bootstrap/src/utils/exec.rs +++ b/src/bootstrap/src/utils/exec.rs @@ -2,15 +2,21 @@ //! //! This module provides a structured way to execute and manage commands efficiently, //! ensuring controlled failure handling and output management. +#![allow(warnings)] + +use std::collections::HashMap; use std::ffi::OsStr; use std::fmt::{Debug, Formatter}; +use std::hash::{Hash, Hasher}; use std::path::Path; use std::process::{Command, CommandArgs, CommandEnvs, ExitStatus, Output, Stdio}; +use std::sync::Mutex; use build_helper::ci::CiEnv; use build_helper::drop_bomb::DropBomb; use super::execution_context::{DeferredCommand, ExecutionContext}; +use crate::PathBuf; /// What should be done when the command fails. #[derive(Debug, Copy, Clone)] @@ -63,6 +69,11 @@ impl OutputMode { /// [allow_failure]: BootstrapCommand::allow_failure /// [delay_failure]: BootstrapCommand::delay_failure pub struct BootstrapCommand { + program: String, + args: Vec, + envs: Vec<(String, String)>, + cwd: Option, + command: Command, pub failure_behavior: BehaviorOnFailure, // Run the command even during dry run @@ -79,6 +90,8 @@ impl<'a> BootstrapCommand { } pub fn arg>(&mut self, arg: S) -> &mut Self { + let arg_str = arg.as_ref().to_string_lossy().into_owned(); + self.args.push(arg_str.clone()); self.command.arg(arg.as_ref()); self } @@ -88,7 +101,9 @@ impl<'a> BootstrapCommand { I: IntoIterator, S: AsRef, { - self.command.args(args); + args.into_iter().for_each(|arg| { + self.arg(arg); + }); self } @@ -97,6 +112,9 @@ impl<'a> BootstrapCommand { K: AsRef, V: AsRef, { + let key_str = key.as_ref().to_string_lossy().into_owned(); + let val_str = val.as_ref().to_string_lossy().into_owned(); + self.envs.push((key_str.clone(), val_str.clone())); self.command.env(key, val); self } @@ -115,6 +133,7 @@ impl<'a> BootstrapCommand { } pub fn current_dir>(&mut self, dir: P) -> &mut Self { + self.cwd = Some(dir.as_ref().to_path_buf()); self.command.current_dir(dir); self } @@ -226,6 +245,10 @@ impl From for BootstrapCommand { let program = command.get_program().to_owned(); Self { + program: program.clone().into_string().unwrap(), + args: Vec::new(), + envs: Vec::new(), + cwd: None, command, failure_behavior: BehaviorOnFailure::Exit, run_in_dry_run: false, @@ -235,6 +258,7 @@ impl From for BootstrapCommand { } /// Represents the current status of `BootstrapCommand`. +#[derive(Clone, PartialEq)] enum CommandStatus { /// The command has started and finished with some status. Finished(ExitStatus), @@ -251,6 +275,7 @@ pub fn command>(program: S) -> BootstrapCommand { } /// Represents the output of an executed process. +#[derive(Clone, PartialEq)] pub struct CommandOutput { status: CommandStatus, stdout: Option>, From 42875e1a3b5c57b2b5059a26f8371ee3e473eed7 Mon Sep 17 00:00:00 2001 From: bit-aloo Date: Sat, 21 Jun 2025 13:26:08 +0530 Subject: [PATCH 02/10] add command cache key, move to osstring, add should cache to bootstrap command --- src/bootstrap/src/utils/exec.rs | 51 +++++++++++++------- src/bootstrap/src/utils/execution_context.rs | 44 ++++++++++++++++- 2 files changed, 76 insertions(+), 19 deletions(-) diff --git a/src/bootstrap/src/utils/exec.rs b/src/bootstrap/src/utils/exec.rs index 28b8fc2734e01..545e059b5cf1a 100644 --- a/src/bootstrap/src/utils/exec.rs +++ b/src/bootstrap/src/utils/exec.rs @@ -5,7 +5,7 @@ #![allow(warnings)] use std::collections::HashMap; -use std::ffi::OsStr; +use std::ffi::{OsStr, OsString}; use std::fmt::{Debug, Formatter}; use std::hash::{Hash, Hasher}; use std::path::Path; @@ -55,6 +55,14 @@ impl OutputMode { } } +#[derive(Clone, Debug, PartialEq, Eq, Hash, Default)] +pub struct CommandCacheKey { + program: OsString, + args: Vec, + envs: Vec<(OsString, OsString)>, + cwd: Option, +} + /// Wrapper around `std::process::Command`. /// /// By default, the command will exit bootstrap if it fails. @@ -69,11 +77,7 @@ impl OutputMode { /// [allow_failure]: BootstrapCommand::allow_failure /// [delay_failure]: BootstrapCommand::delay_failure pub struct BootstrapCommand { - program: String, - args: Vec, - envs: Vec<(String, String)>, - cwd: Option, - + cache_key: CommandCacheKey, command: Command, pub failure_behavior: BehaviorOnFailure, // Run the command even during dry run @@ -81,21 +85,31 @@ pub struct BootstrapCommand { // This field makes sure that each command is executed (or disarmed) before it is dropped, // to avoid forgetting to execute a command. drop_bomb: DropBomb, + should_cache: bool, } impl<'a> BootstrapCommand { #[track_caller] pub fn new>(program: S) -> Self { - Command::new(program).into() + Self { + should_cache: true, + cache_key: CommandCacheKey { + program: program.as_ref().to_os_string(), + ..CommandCacheKey::default() + }, + ..Command::new(program).into() + } } - pub fn arg>(&mut self, arg: S) -> &mut Self { - let arg_str = arg.as_ref().to_string_lossy().into_owned(); - self.args.push(arg_str.clone()); + self.cache_key.args.push(arg.as_ref().to_os_string()); self.command.arg(arg.as_ref()); self } + pub fn should_cache(&self) -> bool { + self.should_cache + } + pub fn args(&mut self, args: I) -> &mut Self where I: IntoIterator, @@ -112,9 +126,7 @@ impl<'a> BootstrapCommand { K: AsRef, V: AsRef, { - let key_str = key.as_ref().to_string_lossy().into_owned(); - let val_str = val.as_ref().to_string_lossy().into_owned(); - self.envs.push((key_str.clone(), val_str.clone())); + self.cache_key.envs.push((key.as_ref().to_os_string(), val.as_ref().to_os_string())); self.command.env(key, val); self } @@ -133,7 +145,7 @@ impl<'a> BootstrapCommand { } pub fn current_dir>(&mut self, dir: P) -> &mut Self { - self.cwd = Some(dir.as_ref().to_path_buf()); + self.cache_key.cwd = Some(dir.as_ref().to_path_buf()); self.command.current_dir(dir); self } @@ -205,6 +217,7 @@ impl<'a> BootstrapCommand { // We don't know what will happen with the returned command, so we need to mark this // command as executed proactively. self.mark_as_executed(); + self.should_cache = false; &mut self.command } @@ -230,6 +243,10 @@ impl<'a> BootstrapCommand { self.env("TERM", "xterm").args(["--color", "always"]); } } + + pub fn cache_key(&self) -> CommandCacheKey { + self.cache_key.clone() + } } impl Debug for BootstrapCommand { @@ -245,10 +262,8 @@ impl From for BootstrapCommand { let program = command.get_program().to_owned(); Self { - program: program.clone().into_string().unwrap(), - args: Vec::new(), - envs: Vec::new(), - cwd: None, + cache_key: CommandCacheKey::default(), + should_cache: false, command, failure_behavior: BehaviorOnFailure::Exit, run_in_dry_run: false, diff --git a/src/bootstrap/src/utils/execution_context.rs b/src/bootstrap/src/utils/execution_context.rs index 66a4be9252e4e..d7ea556c9ff79 100644 --- a/src/bootstrap/src/utils/execution_context.rs +++ b/src/bootstrap/src/utils/execution_context.rs @@ -3,6 +3,8 @@ //! This module provides the [`ExecutionContext`] type, which holds global configuration //! relevant during the execution of commands in bootstrap. This includes dry-run //! mode, verbosity level, and behavior on failure. +#![allow(warnings)] +use std::collections::HashMap; use std::panic::Location; use std::process::Child; use std::sync::{Arc, Mutex}; @@ -10,6 +12,7 @@ use std::sync::{Arc, Mutex}; use crate::core::config::DryRun; #[cfg(feature = "tracing")] use crate::trace_cmd; +use crate::utils::exec::CommandCacheKey; use crate::{BehaviorOnFailure, BootstrapCommand, CommandOutput, OutputMode, exit}; #[derive(Clone, Default)] @@ -18,6 +21,26 @@ pub struct ExecutionContext { verbose: u8, pub fail_fast: bool, delayed_failures: Arc>>, + command_cache: Arc, +} + +#[derive(Default)] +pub struct CommandCache { + cache: Mutex>, +} + +impl CommandCache { + pub fn new() -> Self { + Self { cache: Mutex::new(HashMap::new()) } + } + + pub fn get(&self, key: &CommandCacheKey) -> Option { + self.cache.lock().unwrap().get(key).cloned() + } + + pub fn insert(&self, key: CommandCacheKey, output: CommandOutput) { + self.cache.lock().unwrap().insert(key, output); + } } impl ExecutionContext { @@ -123,7 +146,26 @@ impl ExecutionContext { stdout: OutputMode, stderr: OutputMode, ) -> CommandOutput { - self.start(command, stdout, stderr).wait_for_output(self) + let cache_key = command.cache_key(); + + if command.should_cache() + && let Some(cached_output) = self.command_cache.get(&cache_key) + { + command.mark_as_executed(); + if self.dry_run() && !command.run_always { + return CommandOutput::default(); + } + self.verbose(|| println!("Cache hit: {:?}", command)); + return cached_output; + } + + let output = self.start(command, stdout, stderr).wait_for_output(self); + + if !self.dry_run() || command.run_always && command.should_cache() { + self.command_cache.insert(cache_key, output.clone()); + } + + output } fn fail(&self, message: &str, output: CommandOutput) -> ! { From 08d7b297194502bd92a6e2bb536e50a175cf202f Mon Sep 17 00:00:00 2001 From: bit-aloo Date: Tue, 24 Jun 2025 19:46:30 +0530 Subject: [PATCH 03/10] use metadata for command cache key spawning directly from command --- src/bootstrap/src/utils/exec.rs | 34 +++++++++++--------- src/bootstrap/src/utils/execution_context.rs | 11 ++++--- 2 files changed, 25 insertions(+), 20 deletions(-) diff --git a/src/bootstrap/src/utils/exec.rs b/src/bootstrap/src/utils/exec.rs index 545e059b5cf1a..7b3ae3c9f7581 100644 --- a/src/bootstrap/src/utils/exec.rs +++ b/src/bootstrap/src/utils/exec.rs @@ -77,7 +77,6 @@ pub struct CommandCacheKey { /// [allow_failure]: BootstrapCommand::allow_failure /// [delay_failure]: BootstrapCommand::delay_failure pub struct BootstrapCommand { - cache_key: CommandCacheKey, command: Command, pub failure_behavior: BehaviorOnFailure, // Run the command even during dry run @@ -91,17 +90,9 @@ pub struct BootstrapCommand { impl<'a> BootstrapCommand { #[track_caller] pub fn new>(program: S) -> Self { - Self { - should_cache: true, - cache_key: CommandCacheKey { - program: program.as_ref().to_os_string(), - ..CommandCacheKey::default() - }, - ..Command::new(program).into() - } + Self { should_cache: true, ..Command::new(program).into() } } pub fn arg>(&mut self, arg: S) -> &mut Self { - self.cache_key.args.push(arg.as_ref().to_os_string()); self.command.arg(arg.as_ref()); self } @@ -126,7 +117,6 @@ impl<'a> BootstrapCommand { K: AsRef, V: AsRef, { - self.cache_key.envs.push((key.as_ref().to_os_string(), val.as_ref().to_os_string())); self.command.env(key, val); self } @@ -145,7 +135,6 @@ impl<'a> BootstrapCommand { } pub fn current_dir>(&mut self, dir: P) -> &mut Self { - self.cache_key.cwd = Some(dir.as_ref().to_path_buf()); self.command.current_dir(dir); self } @@ -244,8 +233,8 @@ impl<'a> BootstrapCommand { } } - pub fn cache_key(&self) -> CommandCacheKey { - self.cache_key.clone() + pub fn cache_key(&self) -> Option { + (!self.should_cache).then(|| self.into()) } } @@ -260,9 +249,7 @@ impl From for BootstrapCommand { #[track_caller] fn from(command: Command) -> Self { let program = command.get_program().to_owned(); - Self { - cache_key: CommandCacheKey::default(), should_cache: false, command, failure_behavior: BehaviorOnFailure::Exit, @@ -272,6 +259,21 @@ impl From for BootstrapCommand { } } +impl From<&BootstrapCommand> for CommandCacheKey { + fn from(value: &BootstrapCommand) -> Self { + let command = &value.command; + CommandCacheKey { + program: command.get_program().into(), + args: command.get_args().map(OsStr::to_os_string).collect(), + envs: command + .get_envs() + .filter_map(|(k, v_opt)| v_opt.map(|v| (k.to_owned(), v.to_owned()))) + .collect(), + cwd: command.get_current_dir().map(Path::to_path_buf), + } + } +} + /// Represents the current status of `BootstrapCommand`. #[derive(Clone, PartialEq)] enum CommandStatus { diff --git a/src/bootstrap/src/utils/execution_context.rs b/src/bootstrap/src/utils/execution_context.rs index d7ea556c9ff79..3630ef4dd9797 100644 --- a/src/bootstrap/src/utils/execution_context.rs +++ b/src/bootstrap/src/utils/execution_context.rs @@ -148,21 +148,24 @@ impl ExecutionContext { ) -> CommandOutput { let cache_key = command.cache_key(); - if command.should_cache() - && let Some(cached_output) = self.command_cache.get(&cache_key) + if let Some(cached_output) = cache_key.as_ref().and_then(|key| self.command_cache.get(key)) { command.mark_as_executed(); + if self.dry_run() && !command.run_always { return CommandOutput::default(); } + self.verbose(|| println!("Cache hit: {:?}", command)); return cached_output; } let output = self.start(command, stdout, stderr).wait_for_output(self); - if !self.dry_run() || command.run_always && command.should_cache() { - self.command_cache.insert(cache_key, output.clone()); + if !self.dry_run() || command.run_always { + if let Some(cache_key) = cache_key { + self.command_cache.insert(cache_key, output.clone()); + } } output From 5291c9fcec04436d08fd5689202ff574fd813488 Mon Sep 17 00:00:00 2001 From: bit-aloo Date: Tue, 24 Jun 2025 19:56:31 +0530 Subject: [PATCH 04/10] add do_not_cache method and update the warning on as_command_mut --- src/bootstrap/src/utils/exec.rs | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/src/bootstrap/src/utils/exec.rs b/src/bootstrap/src/utils/exec.rs index 7b3ae3c9f7581..142afec85a6e6 100644 --- a/src/bootstrap/src/utils/exec.rs +++ b/src/bootstrap/src/utils/exec.rs @@ -101,6 +101,11 @@ impl<'a> BootstrapCommand { self.should_cache } + pub fn cache_never(&mut self) -> &mut Self { + self.should_cache = false; + self + } + pub fn args(&mut self, args: I) -> &mut Self where I: IntoIterator, @@ -203,10 +208,11 @@ impl<'a> BootstrapCommand { /// Provides access to the stdlib Command inside. /// FIXME: This function should be eventually removed from bootstrap. pub fn as_command_mut(&mut self) -> &mut Command { - // We don't know what will happen with the returned command, so we need to mark this - // command as executed proactively. + // We proactively mark this command as executed since we can't be certain how the returned + // command will be handled. Caching must also be avoided here, as the inner command could be + // modified externally without us being aware. self.mark_as_executed(); - self.should_cache = false; + self.cache_never(); &mut self.command } From 2cf6552f5dbbb4c09543a6eabe45b7133a3312fe Mon Sep 17 00:00:00 2001 From: bit-aloo Date: Tue, 24 Jun 2025 20:33:18 +0530 Subject: [PATCH 05/10] add new command state in execution context --- src/bootstrap/src/utils/execution_context.rs | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/src/bootstrap/src/utils/execution_context.rs b/src/bootstrap/src/utils/execution_context.rs index 3630ef4dd9797..569fc816b9390 100644 --- a/src/bootstrap/src/utils/execution_context.rs +++ b/src/bootstrap/src/utils/execution_context.rs @@ -29,6 +29,17 @@ pub struct CommandCache { cache: Mutex>, } +enum CommandState<'a> { + Cached(CommandOutput), + Deferred { + process: Option>, + command: &'a mut BootstrapCommand, + stdout: OutputMode, + stderr: OutputMode, + executed_at: &'a Location<'a>, + }, +} + impl CommandCache { pub fn new() -> Self { Self { cache: Mutex::new(HashMap::new()) } From 2e2baed5df07a483d4b91bc237e6a7600f69378d Mon Sep 17 00:00:00 2001 From: bit-aloo Date: Tue, 24 Jun 2025 20:53:52 +0530 Subject: [PATCH 06/10] refactor deferred command and make it compatible with new commandstate, remove extra caching logic from run and re-structure the changes --- src/bootstrap/src/utils/exec.rs | 52 +++---- src/bootstrap/src/utils/execution_context.rs | 138 +++++++++++-------- 2 files changed, 100 insertions(+), 90 deletions(-) diff --git a/src/bootstrap/src/utils/exec.rs b/src/bootstrap/src/utils/exec.rs index 142afec85a6e6..11d786523c4db 100644 --- a/src/bootstrap/src/utils/exec.rs +++ b/src/bootstrap/src/utils/exec.rs @@ -2,15 +2,12 @@ //! //! This module provides a structured way to execute and manage commands efficiently, //! ensuring controlled failure handling and output management. -#![allow(warnings)] -use std::collections::HashMap; use std::ffi::{OsStr, OsString}; use std::fmt::{Debug, Formatter}; -use std::hash::{Hash, Hasher}; +use std::hash::Hash; use std::path::Path; use std::process::{Command, CommandArgs, CommandEnvs, ExitStatus, Output, Stdio}; -use std::sync::Mutex; use build_helper::ci::CiEnv; use build_helper::drop_bomb::DropBomb; @@ -59,7 +56,7 @@ impl OutputMode { pub struct CommandCacheKey { program: OsString, args: Vec, - envs: Vec<(OsString, OsString)>, + envs: Vec<(OsString, Option)>, cwd: Option, } @@ -90,18 +87,14 @@ pub struct BootstrapCommand { impl<'a> BootstrapCommand { #[track_caller] pub fn new>(program: S) -> Self { - Self { should_cache: true, ..Command::new(program).into() } + Command::new(program).into() } pub fn arg>(&mut self, arg: S) -> &mut Self { self.command.arg(arg.as_ref()); self } - pub fn should_cache(&self) -> bool { - self.should_cache - } - - pub fn cache_never(&mut self) -> &mut Self { + pub fn do_not_cache(&mut self) -> &mut Self { self.should_cache = false; self } @@ -111,9 +104,7 @@ impl<'a> BootstrapCommand { I: IntoIterator, S: AsRef, { - args.into_iter().for_each(|arg| { - self.arg(arg); - }); + self.command.args(args); self } @@ -212,7 +203,7 @@ impl<'a> BootstrapCommand { // command will be handled. Caching must also be avoided here, as the inner command could be // modified externally without us being aware. self.mark_as_executed(); - self.cache_never(); + self.do_not_cache(); &mut self.command } @@ -240,7 +231,19 @@ impl<'a> BootstrapCommand { } pub fn cache_key(&self) -> Option { - (!self.should_cache).then(|| self.into()) + if !self.should_cache { + return None; + } + let command = &self.command; + Some(CommandCacheKey { + program: command.get_program().into(), + args: command.get_args().map(OsStr::to_os_string).collect(), + envs: command + .get_envs() + .map(|(k, v)| (k.to_os_string(), v.map(|val| val.to_os_string()))) + .collect(), + cwd: command.get_current_dir().map(Path::to_path_buf), + }) } } @@ -256,7 +259,7 @@ impl From for BootstrapCommand { fn from(command: Command) -> Self { let program = command.get_program().to_owned(); Self { - should_cache: false, + should_cache: true, command, failure_behavior: BehaviorOnFailure::Exit, run_in_dry_run: false, @@ -265,21 +268,6 @@ impl From for BootstrapCommand { } } -impl From<&BootstrapCommand> for CommandCacheKey { - fn from(value: &BootstrapCommand) -> Self { - let command = &value.command; - CommandCacheKey { - program: command.get_program().into(), - args: command.get_args().map(OsStr::to_os_string).collect(), - envs: command - .get_envs() - .filter_map(|(k, v_opt)| v_opt.map(|v| (k.to_owned(), v.to_owned()))) - .collect(), - cwd: command.get_current_dir().map(Path::to_path_buf), - } - } -} - /// Represents the current status of `BootstrapCommand`. #[derive(Clone, PartialEq)] enum CommandStatus { diff --git a/src/bootstrap/src/utils/execution_context.rs b/src/bootstrap/src/utils/execution_context.rs index 569fc816b9390..068a8db218c78 100644 --- a/src/bootstrap/src/utils/execution_context.rs +++ b/src/bootstrap/src/utils/execution_context.rs @@ -3,7 +3,6 @@ //! This module provides the [`ExecutionContext`] type, which holds global configuration //! relevant during the execution of commands in bootstrap. This includes dry-run //! mode, verbosity level, and behavior on failure. -#![allow(warnings)] use std::collections::HashMap; use std::panic::Location; use std::process::Child; @@ -37,14 +36,15 @@ enum CommandState<'a> { stdout: OutputMode, stderr: OutputMode, executed_at: &'a Location<'a>, + cache_key: Option, }, } -impl CommandCache { - pub fn new() -> Self { - Self { cache: Mutex::new(HashMap::new()) } - } +pub struct DeferredCommand<'a> { + state: CommandState<'a>, +} +impl CommandCache { pub fn get(&self, key: &CommandCacheKey) -> Option { self.cache.lock().unwrap().get(key).cloned() } @@ -122,13 +122,33 @@ impl ExecutionContext { stdout: OutputMode, stderr: OutputMode, ) -> DeferredCommand<'a> { + let cache_key = command.cache_key(); + + if let Some(cached_output) = cache_key.as_ref().and_then(|key| self.command_cache.get(key)) + { + command.mark_as_executed(); + + self.verbose(|| println!("Cache hit: {command:?}")); + + return DeferredCommand { state: CommandState::Cached(cached_output) }; + } + command.mark_as_executed(); let created_at = command.get_created_location(); let executed_at = std::panic::Location::caller(); if self.dry_run() && !command.run_in_dry_run { - return DeferredCommand { process: None, stdout, stderr, command, executed_at }; + return DeferredCommand { + state: CommandState::Deferred { + process: None, + command, + stdout, + stderr, + executed_at, + cache_key, + }, + }; } #[cfg(feature = "tracing")] @@ -144,7 +164,16 @@ impl ExecutionContext { let child = cmd.spawn(); - DeferredCommand { process: Some(child), stdout, stderr, command, executed_at } + DeferredCommand { + state: CommandState::Deferred { + process: Some(child), + command, + stdout, + stderr, + executed_at, + cache_key, + }, + } } /// Execute a command and return its output. @@ -157,29 +186,7 @@ impl ExecutionContext { stdout: OutputMode, stderr: OutputMode, ) -> CommandOutput { - let cache_key = command.cache_key(); - - if let Some(cached_output) = cache_key.as_ref().and_then(|key| self.command_cache.get(key)) - { - command.mark_as_executed(); - - if self.dry_run() && !command.run_always { - return CommandOutput::default(); - } - - self.verbose(|| println!("Cache hit: {:?}", command)); - return cached_output; - } - - let output = self.start(command, stdout, stderr).wait_for_output(self); - - if !self.dry_run() || command.run_always { - if let Some(cache_key) = cache_key { - self.command_cache.insert(cache_key, output.clone()); - } - } - - output + self.start(command, stdout, stderr).wait_for_output(self) } fn fail(&self, message: &str, output: CommandOutput) -> ! { @@ -212,25 +219,42 @@ impl AsRef for ExecutionContext { } } -pub struct DeferredCommand<'a> { - process: Option>, - command: &'a mut BootstrapCommand, - stdout: OutputMode, - stderr: OutputMode, - executed_at: &'a Location<'a>, -} - impl<'a> DeferredCommand<'a> { - pub fn wait_for_output(mut self, exec_ctx: impl AsRef) -> CommandOutput { - let exec_ctx = exec_ctx.as_ref(); + pub fn wait_for_output(self, exec_ctx: impl AsRef) -> CommandOutput { + match self.state { + CommandState::Cached(output) => output, + CommandState::Deferred { process, command, stdout, stderr, executed_at, cache_key } => { + let exec_ctx = exec_ctx.as_ref(); + + let output = + Self::finish_process(process, command, stdout, stderr, executed_at, exec_ctx); + + if (!exec_ctx.dry_run() || command.run_always) + && let (Some(cache_key), Some(_)) = (&cache_key, output.status()) + { + exec_ctx.command_cache.insert(cache_key.clone(), output.clone()); + } + + output + } + } + } - let process = match self.process.take() { + pub fn finish_process( + mut process: Option>, + command: &mut BootstrapCommand, + stdout: OutputMode, + stderr: OutputMode, + executed_at: &'a std::panic::Location<'a>, + exec_ctx: &ExecutionContext, + ) -> CommandOutput { + let process = match process.take() { Some(p) => p, None => return CommandOutput::default(), }; - let created_at = self.command.get_created_location(); - let executed_at = self.executed_at; + let created_at = command.get_created_location(); + let executed_at = executed_at; let mut message = String::new(); @@ -238,7 +262,7 @@ impl<'a> DeferredCommand<'a> { Ok(child) => match child.wait_with_output() { Ok(result) if result.status.success() => { // Successful execution - CommandOutput::from_output(result, self.stdout, self.stderr) + CommandOutput::from_output(result, stdout, stderr) } Ok(result) => { // Command ran but failed @@ -247,20 +271,20 @@ impl<'a> DeferredCommand<'a> { writeln!( message, r#" -Command {:?} did not execute successfully. +Command {command:?} did not execute successfully. Expected success, got {} Created at: {created_at} Executed at: {executed_at}"#, - self.command, result.status, + result.status, ) .unwrap(); - let output = CommandOutput::from_output(result, self.stdout, self.stderr); + let output = CommandOutput::from_output(result, stdout, stderr); - if self.stdout.captures() { + if stdout.captures() { writeln!(message, "\nSTDOUT ----\n{}", output.stdout().trim()).unwrap(); } - if self.stderr.captures() { + if stderr.captures() { writeln!(message, "\nSTDERR ----\n{}", output.stderr().trim()).unwrap(); } @@ -272,13 +296,12 @@ Executed at: {executed_at}"#, writeln!( message, - "\n\nCommand {:?} did not execute successfully.\ - \nIt was not possible to execute the command: {e:?}", - self.command + "\n\nCommand {command:?} did not execute successfully.\ + \nIt was not possible to execute the command: {e:?}" ) .unwrap(); - CommandOutput::did_not_start(self.stdout, self.stderr) + CommandOutput::did_not_start(stdout, stderr) } }, Err(e) => { @@ -287,18 +310,17 @@ Executed at: {executed_at}"#, writeln!( message, - "\n\nCommand {:?} did not execute successfully.\ - \nIt was not possible to execute the command: {e:?}", - self.command + "\n\nCommand {command:?} did not execute successfully.\ + \nIt was not possible to execute the command: {e:?}" ) .unwrap(); - CommandOutput::did_not_start(self.stdout, self.stderr) + CommandOutput::did_not_start(stdout, stderr) } }; if !output.is_success() { - match self.command.failure_behavior { + match command.failure_behavior { BehaviorOnFailure::DelayFail => { if exec_ctx.fail_fast { exec_ctx.fail(&message, output); From 28f97108f1a2446bb41cb34b57c98a402631b0b5 Mon Sep 17 00:00:00 2001 From: bit-aloo Date: Tue, 24 Jun 2025 23:00:15 +0530 Subject: [PATCH 07/10] add caching info on bootstrap command --- src/bootstrap/src/utils/exec.rs | 3 +++ src/bootstrap/src/utils/execution_context.rs | 7 +------ 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/src/bootstrap/src/utils/exec.rs b/src/bootstrap/src/utils/exec.rs index 11d786523c4db..29b32f548d1c4 100644 --- a/src/bootstrap/src/utils/exec.rs +++ b/src/bootstrap/src/utils/exec.rs @@ -71,6 +71,9 @@ pub struct CommandCacheKey { /// /// Bootstrap will print a debug log to stdout if the command fails and failure is not allowed. /// +/// By default, command executions are cached based on their workdir, program, arguments, and environment variables. +/// This avoids re-running identical commands unnecessarily, unless caching is explicitly disabled. +/// /// [allow_failure]: BootstrapCommand::allow_failure /// [delay_failure]: BootstrapCommand::delay_failure pub struct BootstrapCommand { diff --git a/src/bootstrap/src/utils/execution_context.rs b/src/bootstrap/src/utils/execution_context.rs index 068a8db218c78..e25dfb501cfb1 100644 --- a/src/bootstrap/src/utils/execution_context.rs +++ b/src/bootstrap/src/utils/execution_context.rs @@ -122,19 +122,15 @@ impl ExecutionContext { stdout: OutputMode, stderr: OutputMode, ) -> DeferredCommand<'a> { + command.mark_as_executed(); let cache_key = command.cache_key(); if let Some(cached_output) = cache_key.as_ref().and_then(|key| self.command_cache.get(key)) { - command.mark_as_executed(); - self.verbose(|| println!("Cache hit: {command:?}")); - return DeferredCommand { state: CommandState::Cached(cached_output) }; } - command.mark_as_executed(); - let created_at = command.get_created_location(); let executed_at = std::panic::Location::caller(); @@ -254,7 +250,6 @@ impl<'a> DeferredCommand<'a> { }; let created_at = command.get_created_location(); - let executed_at = executed_at; let mut message = String::new(); From 18090924f98c095cf001626df293d79246bb271f Mon Sep 17 00:00:00 2001 From: bit-aloo Date: Wed, 25 Jun 2025 00:52:02 +0530 Subject: [PATCH 08/10] make DeferredCommand a must use and move mark_as_executed inside finish process --- src/bootstrap/src/utils/execution_context.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/bootstrap/src/utils/execution_context.rs b/src/bootstrap/src/utils/execution_context.rs index e25dfb501cfb1..3076469f39575 100644 --- a/src/bootstrap/src/utils/execution_context.rs +++ b/src/bootstrap/src/utils/execution_context.rs @@ -40,6 +40,7 @@ enum CommandState<'a> { }, } +#[must_use] pub struct DeferredCommand<'a> { state: CommandState<'a>, } @@ -122,11 +123,11 @@ impl ExecutionContext { stdout: OutputMode, stderr: OutputMode, ) -> DeferredCommand<'a> { - command.mark_as_executed(); let cache_key = command.cache_key(); if let Some(cached_output) = cache_key.as_ref().and_then(|key| self.command_cache.get(key)) { + command.mark_as_executed(); self.verbose(|| println!("Cache hit: {command:?}")); return DeferredCommand { state: CommandState::Cached(cached_output) }; } @@ -244,6 +245,8 @@ impl<'a> DeferredCommand<'a> { executed_at: &'a std::panic::Location<'a>, exec_ctx: &ExecutionContext, ) -> CommandOutput { + command.mark_as_executed(); + let process = match process.take() { Some(p) => p, None => return CommandOutput::default(), From 0eab328414447f000d718bbca0c234bd49e28a6b Mon Sep 17 00:00:00 2001 From: bit-aloo Date: Wed, 25 Jun 2025 16:26:13 +0530 Subject: [PATCH 09/10] move execution context inside exec and prune execution context, use command directly from bootstrap command inside start, and not via as_command_mut --- src/bootstrap/src/core/builder/mod.rs | 3 +- src/bootstrap/src/core/config/config.rs | 3 +- src/bootstrap/src/lib.rs | 4 +- src/bootstrap/src/utils/channel.rs | 2 +- src/bootstrap/src/utils/exec.rs | 344 ++++++++++++++++++- src/bootstrap/src/utils/execution_context.rs | 341 ------------------ src/bootstrap/src/utils/mod.rs | 1 - 7 files changed, 345 insertions(+), 353 deletions(-) delete mode 100644 src/bootstrap/src/utils/execution_context.rs diff --git a/src/bootstrap/src/core/builder/mod.rs b/src/bootstrap/src/core/builder/mod.rs index 7cb7866953a8e..8e9e8b496de7c 100644 --- a/src/bootstrap/src/core/builder/mod.rs +++ b/src/bootstrap/src/core/builder/mod.rs @@ -22,8 +22,7 @@ use crate::core::build_steps::{ use crate::core::config::flags::Subcommand; use crate::core::config::{DryRun, TargetSelection}; use crate::utils::cache::Cache; -use crate::utils::exec::{BootstrapCommand, command}; -use crate::utils::execution_context::ExecutionContext; +use crate::utils::exec::{BootstrapCommand, ExecutionContext, command}; use crate::utils::helpers::{self, LldThreads, add_dylib_path, exe, libdir, linker_args, t}; use crate::{Build, Crate, trace}; diff --git a/src/bootstrap/src/core/config/config.rs b/src/bootstrap/src/core/config/config.rs index b2754a5ad32c2..d1ffdf24acd0e 100644 --- a/src/bootstrap/src/core/config/config.rs +++ b/src/bootstrap/src/core/config/config.rs @@ -47,8 +47,7 @@ use crate::core::config::{ }; use crate::core::download::is_download_ci_available; use crate::utils::channel; -use crate::utils::exec::command; -use crate::utils::execution_context::ExecutionContext; +use crate::utils::exec::{ExecutionContext, command}; use crate::utils::helpers::{exe, get_host_target}; use crate::{GitInfo, OnceLock, TargetSelection, check_ci_llvm, helpers, t}; diff --git a/src/bootstrap/src/lib.rs b/src/bootstrap/src/lib.rs index 26fac6b3b603f..ef5c28272b8e1 100644 --- a/src/bootstrap/src/lib.rs +++ b/src/bootstrap/src/lib.rs @@ -31,12 +31,12 @@ use cc::Tool; use termcolor::{ColorChoice, StandardStream, WriteColor}; use utils::build_stamp::BuildStamp; use utils::channel::GitInfo; -use utils::execution_context::ExecutionContext; +use utils::exec::ExecutionContext; use crate::core::builder; use crate::core::builder::Kind; use crate::core::config::{DryRun, LldMode, LlvmLibunwind, TargetSelection, flags}; -use crate::utils::exec::{BehaviorOnFailure, BootstrapCommand, CommandOutput, OutputMode, command}; +use crate::utils::exec::{BootstrapCommand, command}; use crate::utils::helpers::{ self, dir_is_empty, exe, libdir, set_file_times, split_debuginfo, symlink_dir, }; diff --git a/src/bootstrap/src/utils/channel.rs b/src/bootstrap/src/utils/channel.rs index 16aa9ba0585b2..21b4257e54d0b 100644 --- a/src/bootstrap/src/utils/channel.rs +++ b/src/bootstrap/src/utils/channel.rs @@ -8,7 +8,7 @@ use std::fs; use std::path::Path; -use super::execution_context::ExecutionContext; +use super::exec::ExecutionContext; use super::helpers; use crate::Build; use crate::utils::helpers::t; diff --git a/src/bootstrap/src/utils/exec.rs b/src/bootstrap/src/utils/exec.rs index 29b32f548d1c4..4c38f3a7cc82c 100644 --- a/src/bootstrap/src/utils/exec.rs +++ b/src/bootstrap/src/utils/exec.rs @@ -1,19 +1,29 @@ //! Command Execution Module //! -//! This module provides a structured way to execute and manage commands efficiently, -//! ensuring controlled failure handling and output management. +//! Provides a structured interface for executing and managing commands during bootstrap, +//! with support for controlled failure handling and output management. +//! +//! This module defines the [`ExecutionContext`] type, which encapsulates global configuration +//! relevant to command execution in the bootstrap process. This includes settings such as +//! dry-run mode, verbosity level, and failure behavior. +use std::collections::HashMap; use std::ffi::{OsStr, OsString}; use std::fmt::{Debug, Formatter}; use std::hash::Hash; +use std::panic::Location; use std::path::Path; -use std::process::{Command, CommandArgs, CommandEnvs, ExitStatus, Output, Stdio}; +use std::process::{Child, Command, CommandArgs, CommandEnvs, ExitStatus, Output, Stdio}; +use std::sync::{Arc, Mutex}; use build_helper::ci::CiEnv; use build_helper::drop_bomb::DropBomb; +use build_helper::exit; -use super::execution_context::{DeferredCommand, ExecutionContext}; use crate::PathBuf; +use crate::core::config::DryRun; +#[cfg(feature = "tracing")] +use crate::trace_cmd; /// What should be done when the command fails. #[derive(Debug, Copy, Clone)] @@ -412,3 +422,329 @@ impl FormatShortCmd for Command { line.join(" ") } } + +#[derive(Clone, Default)] +pub struct ExecutionContext { + dry_run: DryRun, + verbose: u8, + pub fail_fast: bool, + delayed_failures: Arc>>, + command_cache: Arc, +} + +#[derive(Default)] +pub struct CommandCache { + cache: Mutex>, +} + +enum CommandState<'a> { + Cached(CommandOutput), + Deferred { + process: Option>, + command: &'a mut BootstrapCommand, + stdout: OutputMode, + stderr: OutputMode, + executed_at: &'a Location<'a>, + cache_key: Option, + }, +} + +#[must_use] +pub struct DeferredCommand<'a> { + state: CommandState<'a>, +} + +impl CommandCache { + pub fn get(&self, key: &CommandCacheKey) -> Option { + self.cache.lock().unwrap().get(key).cloned() + } + + pub fn insert(&self, key: CommandCacheKey, output: CommandOutput) { + self.cache.lock().unwrap().insert(key, output); + } +} + +impl ExecutionContext { + pub fn new() -> Self { + ExecutionContext::default() + } + + pub fn dry_run(&self) -> bool { + match self.dry_run { + DryRun::Disabled => false, + DryRun::SelfCheck | DryRun::UserSelected => true, + } + } + + pub fn get_dry_run(&self) -> &DryRun { + &self.dry_run + } + + pub fn verbose(&self, f: impl Fn()) { + if self.is_verbose() { + f() + } + } + + pub fn is_verbose(&self) -> bool { + self.verbose > 0 + } + + pub fn fail_fast(&self) -> bool { + self.fail_fast + } + + pub fn set_dry_run(&mut self, value: DryRun) { + self.dry_run = value; + } + + pub fn set_verbose(&mut self, value: u8) { + self.verbose = value; + } + + pub fn set_fail_fast(&mut self, value: bool) { + self.fail_fast = value; + } + + pub fn add_to_delay_failure(&self, message: String) { + self.delayed_failures.lock().unwrap().push(message); + } + + pub fn report_failures_and_exit(&self) { + let failures = self.delayed_failures.lock().unwrap(); + if failures.is_empty() { + return; + } + eprintln!("\n{} command(s) did not execute successfully:\n", failures.len()); + for failure in &*failures { + eprintln!(" - {failure}"); + } + exit!(1); + } + + /// Execute a command and return its output. + /// Note: Ideally, you should use one of the BootstrapCommand::run* functions to + /// execute commands. They internally call this method. + #[track_caller] + pub fn start<'a>( + &self, + command: &'a mut BootstrapCommand, + stdout: OutputMode, + stderr: OutputMode, + ) -> DeferredCommand<'a> { + let cache_key = command.cache_key(); + + if let Some(cached_output) = cache_key.as_ref().and_then(|key| self.command_cache.get(key)) + { + command.mark_as_executed(); + self.verbose(|| println!("Cache hit: {command:?}")); + return DeferredCommand { state: CommandState::Cached(cached_output) }; + } + + let created_at = command.get_created_location(); + let executed_at = std::panic::Location::caller(); + + if self.dry_run() && !command.run_always { + return DeferredCommand { + state: CommandState::Deferred { + process: None, + command, + stdout, + stderr, + executed_at, + cache_key, + }, + }; + } + + #[cfg(feature = "tracing")] + let _run_span = trace_cmd!(command); + + self.verbose(|| { + println!("running: {command:?} (created at {created_at}, executed at {executed_at})") + }); + + let cmd = &mut command.command; + cmd.stdout(stdout.stdio()); + cmd.stderr(stderr.stdio()); + + let child = cmd.spawn(); + + DeferredCommand { + state: CommandState::Deferred { + process: Some(child), + command, + stdout, + stderr, + executed_at, + cache_key, + }, + } + } + + /// Execute a command and return its output. + /// Note: Ideally, you should use one of the BootstrapCommand::run* functions to + /// execute commands. They internally call this method. + #[track_caller] + pub fn run( + &self, + command: &mut BootstrapCommand, + stdout: OutputMode, + stderr: OutputMode, + ) -> CommandOutput { + self.start(command, stdout, stderr).wait_for_output(self) + } + + fn fail(&self, message: &str, output: CommandOutput) -> ! { + if self.is_verbose() { + println!("{message}"); + } else { + let (stdout, stderr) = (output.stdout_if_present(), output.stderr_if_present()); + // If the command captures output, the user would not see any indication that + // it has failed. In this case, print a more verbose error, since to provide more + // context. + if stdout.is_some() || stderr.is_some() { + if let Some(stdout) = output.stdout_if_present().take_if(|s| !s.trim().is_empty()) { + println!("STDOUT:\n{stdout}\n"); + } + if let Some(stderr) = output.stderr_if_present().take_if(|s| !s.trim().is_empty()) { + println!("STDERR:\n{stderr}\n"); + } + println!("Command has failed. Rerun with -v to see more details."); + } else { + println!("Command has failed. Rerun with -v to see more details."); + } + } + exit!(1); + } +} + +impl AsRef for ExecutionContext { + fn as_ref(&self) -> &ExecutionContext { + self + } +} + +impl<'a> DeferredCommand<'a> { + pub fn wait_for_output(self, exec_ctx: impl AsRef) -> CommandOutput { + match self.state { + CommandState::Cached(output) => output, + CommandState::Deferred { process, command, stdout, stderr, executed_at, cache_key } => { + let exec_ctx = exec_ctx.as_ref(); + + let output = + Self::finish_process(process, command, stdout, stderr, executed_at, exec_ctx); + + if (!exec_ctx.dry_run() || command.run_always) + && let (Some(cache_key), Some(_)) = (&cache_key, output.status()) + { + exec_ctx.command_cache.insert(cache_key.clone(), output.clone()); + } + + output + } + } + } + + pub fn finish_process( + mut process: Option>, + command: &mut BootstrapCommand, + stdout: OutputMode, + stderr: OutputMode, + executed_at: &'a std::panic::Location<'a>, + exec_ctx: &ExecutionContext, + ) -> CommandOutput { + command.mark_as_executed(); + + let process = match process.take() { + Some(p) => p, + None => return CommandOutput::default(), + }; + + let created_at = command.get_created_location(); + + let mut message = String::new(); + + let output = match process { + Ok(child) => match child.wait_with_output() { + Ok(result) if result.status.success() => { + // Successful execution + CommandOutput::from_output(result, stdout, stderr) + } + Ok(result) => { + // Command ran but failed + use std::fmt::Write; + + writeln!( + message, + r#" +Command {command:?} did not execute successfully. +Expected success, got {} +Created at: {created_at} +Executed at: {executed_at}"#, + result.status, + ) + .unwrap(); + + let output = CommandOutput::from_output(result, stdout, stderr); + + if stdout.captures() { + writeln!(message, "\nSTDOUT ----\n{}", output.stdout().trim()).unwrap(); + } + if stderr.captures() { + writeln!(message, "\nSTDERR ----\n{}", output.stderr().trim()).unwrap(); + } + + output + } + Err(e) => { + // Failed to wait for output + use std::fmt::Write; + + writeln!( + message, + "\n\nCommand {command:?} did not execute successfully.\ + \nIt was not possible to execute the command: {e:?}" + ) + .unwrap(); + + CommandOutput::did_not_start(stdout, stderr) + } + }, + Err(e) => { + // Failed to spawn the command + use std::fmt::Write; + + writeln!( + message, + "\n\nCommand {command:?} did not execute successfully.\ + \nIt was not possible to execute the command: {e:?}" + ) + .unwrap(); + + CommandOutput::did_not_start(stdout, stderr) + } + }; + + if !output.is_success() { + match command.failure_behavior { + BehaviorOnFailure::DelayFail => { + if exec_ctx.fail_fast { + exec_ctx.fail(&message, output); + } + exec_ctx.add_to_delay_failure(message); + } + BehaviorOnFailure::Exit => { + exec_ctx.fail(&message, output); + } + BehaviorOnFailure::Ignore => { + // If failures are allowed, either the error has been printed already + // (OutputMode::Print) or the user used a capture output mode and wants to + // handle the error output on their own. + } + } + } + + output + } +} diff --git a/src/bootstrap/src/utils/execution_context.rs b/src/bootstrap/src/utils/execution_context.rs deleted file mode 100644 index 3076469f39575..0000000000000 --- a/src/bootstrap/src/utils/execution_context.rs +++ /dev/null @@ -1,341 +0,0 @@ -//! Shared execution context for running bootstrap commands. -//! -//! This module provides the [`ExecutionContext`] type, which holds global configuration -//! relevant during the execution of commands in bootstrap. This includes dry-run -//! mode, verbosity level, and behavior on failure. -use std::collections::HashMap; -use std::panic::Location; -use std::process::Child; -use std::sync::{Arc, Mutex}; - -use crate::core::config::DryRun; -#[cfg(feature = "tracing")] -use crate::trace_cmd; -use crate::utils::exec::CommandCacheKey; -use crate::{BehaviorOnFailure, BootstrapCommand, CommandOutput, OutputMode, exit}; - -#[derive(Clone, Default)] -pub struct ExecutionContext { - dry_run: DryRun, - verbose: u8, - pub fail_fast: bool, - delayed_failures: Arc>>, - command_cache: Arc, -} - -#[derive(Default)] -pub struct CommandCache { - cache: Mutex>, -} - -enum CommandState<'a> { - Cached(CommandOutput), - Deferred { - process: Option>, - command: &'a mut BootstrapCommand, - stdout: OutputMode, - stderr: OutputMode, - executed_at: &'a Location<'a>, - cache_key: Option, - }, -} - -#[must_use] -pub struct DeferredCommand<'a> { - state: CommandState<'a>, -} - -impl CommandCache { - pub fn get(&self, key: &CommandCacheKey) -> Option { - self.cache.lock().unwrap().get(key).cloned() - } - - pub fn insert(&self, key: CommandCacheKey, output: CommandOutput) { - self.cache.lock().unwrap().insert(key, output); - } -} - -impl ExecutionContext { - pub fn new() -> Self { - ExecutionContext::default() - } - - pub fn dry_run(&self) -> bool { - match self.dry_run { - DryRun::Disabled => false, - DryRun::SelfCheck | DryRun::UserSelected => true, - } - } - - pub fn get_dry_run(&self) -> &DryRun { - &self.dry_run - } - - pub fn verbose(&self, f: impl Fn()) { - if self.is_verbose() { - f() - } - } - - pub fn is_verbose(&self) -> bool { - self.verbose > 0 - } - - pub fn fail_fast(&self) -> bool { - self.fail_fast - } - - pub fn set_dry_run(&mut self, value: DryRun) { - self.dry_run = value; - } - - pub fn set_verbose(&mut self, value: u8) { - self.verbose = value; - } - - pub fn set_fail_fast(&mut self, value: bool) { - self.fail_fast = value; - } - - pub fn add_to_delay_failure(&self, message: String) { - self.delayed_failures.lock().unwrap().push(message); - } - - pub fn report_failures_and_exit(&self) { - let failures = self.delayed_failures.lock().unwrap(); - if failures.is_empty() { - return; - } - eprintln!("\n{} command(s) did not execute successfully:\n", failures.len()); - for failure in &*failures { - eprintln!(" - {failure}"); - } - exit!(1); - } - - /// Execute a command and return its output. - /// Note: Ideally, you should use one of the BootstrapCommand::run* functions to - /// execute commands. They internally call this method. - #[track_caller] - pub fn start<'a>( - &self, - command: &'a mut BootstrapCommand, - stdout: OutputMode, - stderr: OutputMode, - ) -> DeferredCommand<'a> { - let cache_key = command.cache_key(); - - if let Some(cached_output) = cache_key.as_ref().and_then(|key| self.command_cache.get(key)) - { - command.mark_as_executed(); - self.verbose(|| println!("Cache hit: {command:?}")); - return DeferredCommand { state: CommandState::Cached(cached_output) }; - } - - let created_at = command.get_created_location(); - let executed_at = std::panic::Location::caller(); - - if self.dry_run() && !command.run_in_dry_run { - return DeferredCommand { - state: CommandState::Deferred { - process: None, - command, - stdout, - stderr, - executed_at, - cache_key, - }, - }; - } - - #[cfg(feature = "tracing")] - let _run_span = trace_cmd!(command); - - self.verbose(|| { - println!("running: {command:?} (created at {created_at}, executed at {executed_at})") - }); - - let cmd = command.as_command_mut(); - cmd.stdout(stdout.stdio()); - cmd.stderr(stderr.stdio()); - - let child = cmd.spawn(); - - DeferredCommand { - state: CommandState::Deferred { - process: Some(child), - command, - stdout, - stderr, - executed_at, - cache_key, - }, - } - } - - /// Execute a command and return its output. - /// Note: Ideally, you should use one of the BootstrapCommand::run* functions to - /// execute commands. They internally call this method. - #[track_caller] - pub fn run( - &self, - command: &mut BootstrapCommand, - stdout: OutputMode, - stderr: OutputMode, - ) -> CommandOutput { - self.start(command, stdout, stderr).wait_for_output(self) - } - - fn fail(&self, message: &str, output: CommandOutput) -> ! { - if self.is_verbose() { - println!("{message}"); - } else { - let (stdout, stderr) = (output.stdout_if_present(), output.stderr_if_present()); - // If the command captures output, the user would not see any indication that - // it has failed. In this case, print a more verbose error, since to provide more - // context. - if stdout.is_some() || stderr.is_some() { - if let Some(stdout) = output.stdout_if_present().take_if(|s| !s.trim().is_empty()) { - println!("STDOUT:\n{stdout}\n"); - } - if let Some(stderr) = output.stderr_if_present().take_if(|s| !s.trim().is_empty()) { - println!("STDERR:\n{stderr}\n"); - } - println!("Command has failed. Rerun with -v to see more details."); - } else { - println!("Command has failed. Rerun with -v to see more details."); - } - } - exit!(1); - } -} - -impl AsRef for ExecutionContext { - fn as_ref(&self) -> &ExecutionContext { - self - } -} - -impl<'a> DeferredCommand<'a> { - pub fn wait_for_output(self, exec_ctx: impl AsRef) -> CommandOutput { - match self.state { - CommandState::Cached(output) => output, - CommandState::Deferred { process, command, stdout, stderr, executed_at, cache_key } => { - let exec_ctx = exec_ctx.as_ref(); - - let output = - Self::finish_process(process, command, stdout, stderr, executed_at, exec_ctx); - - if (!exec_ctx.dry_run() || command.run_always) - && let (Some(cache_key), Some(_)) = (&cache_key, output.status()) - { - exec_ctx.command_cache.insert(cache_key.clone(), output.clone()); - } - - output - } - } - } - - pub fn finish_process( - mut process: Option>, - command: &mut BootstrapCommand, - stdout: OutputMode, - stderr: OutputMode, - executed_at: &'a std::panic::Location<'a>, - exec_ctx: &ExecutionContext, - ) -> CommandOutput { - command.mark_as_executed(); - - let process = match process.take() { - Some(p) => p, - None => return CommandOutput::default(), - }; - - let created_at = command.get_created_location(); - - let mut message = String::new(); - - let output = match process { - Ok(child) => match child.wait_with_output() { - Ok(result) if result.status.success() => { - // Successful execution - CommandOutput::from_output(result, stdout, stderr) - } - Ok(result) => { - // Command ran but failed - use std::fmt::Write; - - writeln!( - message, - r#" -Command {command:?} did not execute successfully. -Expected success, got {} -Created at: {created_at} -Executed at: {executed_at}"#, - result.status, - ) - .unwrap(); - - let output = CommandOutput::from_output(result, stdout, stderr); - - if stdout.captures() { - writeln!(message, "\nSTDOUT ----\n{}", output.stdout().trim()).unwrap(); - } - if stderr.captures() { - writeln!(message, "\nSTDERR ----\n{}", output.stderr().trim()).unwrap(); - } - - output - } - Err(e) => { - // Failed to wait for output - use std::fmt::Write; - - writeln!( - message, - "\n\nCommand {command:?} did not execute successfully.\ - \nIt was not possible to execute the command: {e:?}" - ) - .unwrap(); - - CommandOutput::did_not_start(stdout, stderr) - } - }, - Err(e) => { - // Failed to spawn the command - use std::fmt::Write; - - writeln!( - message, - "\n\nCommand {command:?} did not execute successfully.\ - \nIt was not possible to execute the command: {e:?}" - ) - .unwrap(); - - CommandOutput::did_not_start(stdout, stderr) - } - }; - - if !output.is_success() { - match command.failure_behavior { - BehaviorOnFailure::DelayFail => { - if exec_ctx.fail_fast { - exec_ctx.fail(&message, output); - } - exec_ctx.add_to_delay_failure(message); - } - BehaviorOnFailure::Exit => { - exec_ctx.fail(&message, output); - } - BehaviorOnFailure::Ignore => { - // If failures are allowed, either the error has been printed already - // (OutputMode::Print) or the user used a capture output mode and wants to - // handle the error output on their own. - } - } - } - - output - } -} diff --git a/src/bootstrap/src/utils/mod.rs b/src/bootstrap/src/utils/mod.rs index 5a0b90801e73a..169fcec303e90 100644 --- a/src/bootstrap/src/utils/mod.rs +++ b/src/bootstrap/src/utils/mod.rs @@ -8,7 +8,6 @@ pub(crate) mod cc_detect; pub(crate) mod change_tracker; pub(crate) mod channel; pub(crate) mod exec; -pub(crate) mod execution_context; pub(crate) mod helpers; pub(crate) mod job; pub(crate) mod render_tests; From 591df6c1b087c0a12eade854d0e9497c3f8f6d8b Mon Sep 17 00:00:00 2001 From: bit-aloo Date: Wed, 25 Jun 2025 17:00:03 +0530 Subject: [PATCH 10/10] disable caching for cargo commands --- src/bootstrap/src/core/builder/cargo.rs | 5 ++++- src/bootstrap/src/utils/exec.rs | 4 ++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/src/bootstrap/src/core/builder/cargo.rs b/src/bootstrap/src/core/builder/cargo.rs index a878291a33c0d..deb7106f185c4 100644 --- a/src/bootstrap/src/core/builder/cargo.rs +++ b/src/bootstrap/src/core/builder/cargo.rs @@ -131,7 +131,10 @@ impl Cargo { } pub fn into_cmd(self) -> BootstrapCommand { - self.into() + let mut cmd: BootstrapCommand = self.into(); + // Disable caching for commands originating from Cargo-related operations. + cmd.do_not_cache(); + cmd } /// Same as [`Cargo::new`] except this one doesn't configure the linker with diff --git a/src/bootstrap/src/utils/exec.rs b/src/bootstrap/src/utils/exec.rs index 4c38f3a7cc82c..d092765ef762f 100644 --- a/src/bootstrap/src/utils/exec.rs +++ b/src/bootstrap/src/utils/exec.rs @@ -544,7 +544,7 @@ impl ExecutionContext { let created_at = command.get_created_location(); let executed_at = std::panic::Location::caller(); - if self.dry_run() && !command.run_always { + if self.dry_run() && !command.run_in_dry_run { return DeferredCommand { state: CommandState::Deferred { process: None, @@ -635,7 +635,7 @@ impl<'a> DeferredCommand<'a> { let output = Self::finish_process(process, command, stdout, stderr, executed_at, exec_ctx); - if (!exec_ctx.dry_run() || command.run_always) + if (!exec_ctx.dry_run() || command.run_in_dry_run) && let (Some(cache_key), Some(_)) = (&cache_key, output.status()) { exec_ctx.command_cache.insert(cache_key.clone(), output.clone());