diff --git a/Cargo.lock b/Cargo.lock index a9569719..6614c506 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1490,6 +1490,7 @@ dependencies = [ "futures-lite", "nix_rs", "serde", + "serde_json", "thiserror", "tokio", "tracing", diff --git a/crates/nix_health/CHANGELOG.md b/crates/nix_health/CHANGELOG.md index ea1dbf9d..c4e69697 100644 --- a/crates/nix_health/CHANGELOG.md +++ b/crates/nix_health/CHANGELOG.md @@ -2,6 +2,7 @@ ## Unreleased +- The flake config is now fetched from flake outputs using flake-schemas instead of `nix eval` - Remove unused `logging` module - Display Nix installer used diff --git a/crates/nix_health/src/lib.rs b/crates/nix_health/src/lib.rs index 1a2da2cf..14185324 100644 --- a/crates/nix_health/src/lib.rs +++ b/crates/nix_health/src/lib.rs @@ -64,8 +64,8 @@ impl NixHealth { /// override it. pub async fn from_flake(url: &FlakeUrl) -> Result { let cmd = NixCmd::get().await; - let cfg = - OmConfig::::from_flake_url(cmd, url, &["om.health", "nix-health"]).await?; + let cfg = OmConfig::::from_nix(cmd, url, &[&["om", "health"], &["nix-health"]]) + .await?; let (cfg, _rest) = cfg.get_referenced()?; Ok(cfg.clone()) } diff --git a/crates/nix_rs/CHANGELOG.md b/crates/nix_rs/CHANGELOG.md index b6bae2ce..7121e0a0 100644 --- a/crates/nix_rs/CHANGELOG.md +++ b/crates/nix_rs/CHANGELOG.md @@ -1,6 +1,13 @@ # Changelog ## Unreleased + +- **Inherit `stderr`** + - `NixCmd::run_with_args_returning_stdout` now inherits `stderr` from parent process by default. + - `stderr` is no longer captured and passed along with `CommandError::ProcessFailed`. + - `nix_eval_attr` no longer supports missing attribute check. + - Introduce `FilteredFlakeOutputs` to fetch the flake config, avoiding dependency on `nix_eval_attr`. + - Remove `qualified_attr` module, use `find_qualified_attr` from `FilteredFlakeOutputs` instead. - **`flake::schema`** - Don't hardcode flake schema types - **`env`**: diff --git a/crates/nix_rs/src/command.rs b/crates/nix_rs/src/command.rs index a657c135..35058cb6 100644 --- a/crates/nix_rs/src/command.rs +++ b/crates/nix_rs/src/command.rs @@ -16,7 +16,11 @@ use std::{ use serde::{Deserialize, Serialize}; use thiserror::Error; -use tokio::{process::Command, sync::OnceCell}; +use tokio::{ + io::AsyncReadExt, + process::{ChildStdout, Command}, + sync::OnceCell, +}; use tracing::instrument; @@ -118,11 +122,17 @@ impl NixCmd { where T: serde::de::DeserializeOwned, { - let stdout: Vec = self + let mut stdout_handle = self .run_with_returning_stdout(|c| { c.args(args); }) .await?; + let mut stdout = Vec::new(); + + stdout_handle + .read_to_end(&mut stdout) + .await + .map_err(CommandError::ChildProcessError)?; let v = serde_json::from_slice::(&stdout)?; Ok(v) } @@ -133,39 +143,38 @@ impl NixCmd { T: std::str::FromStr, ::Err: std::fmt::Display, { - let stdout = self + let mut stdout_handle = self .run_with_returning_stdout(|c| { c.args(args); }) .await?; + let mut stdout = Vec::new(); + stdout_handle + .read_to_end(&mut stdout) + .await + .map_err(CommandError::ChildProcessError)?; let v = &String::from_utf8_lossy(&stdout); let v = T::from_str(v.trim()).map_err(|e| FromStrError(e.to_string()))?; Ok(v) } /// Like [Self::run_with] but returns stdout as a [`Vec`] - pub async fn run_with_returning_stdout(&self, f: F) -> Result, CommandError> + pub async fn run_with_returning_stdout(&self, f: F) -> Result where F: FnOnce(&mut Command), { let mut cmd = self.command(); f(&mut cmd); + cmd.stdout(Stdio::piped()); trace_cmd(&cmd); - cmd.stdout(Stdio::piped()); - cmd.stderr(Stdio::piped()); - let child = cmd.spawn()?; - let out = child.wait_with_output().await?; + let mut child = cmd.spawn()?; + let exit_status = child.wait().await?; // Wait for the child to finish - if out.status.success() { - Ok(out.stdout) - } else { - let stderr = String::from_utf8(out.stderr)?; - Err(CommandError::ProcessFailed { - stderr: Some(stderr), - exit_code: out.status.code(), - }) - } + let stdout = child.stdout.take().ok_or(CommandError::ProcessFailed { + exit_code: exit_status.code(), + })?; + Ok(stdout) } /// Run Nix with given [Command] customizations, while also tracing the command being run. @@ -181,7 +190,6 @@ impl NixCmd { Ok(()) } else { Err(CommandError::ProcessFailed { - stderr: None, exit_code: status.code(), }) } @@ -264,18 +272,9 @@ pub enum CommandError { ChildProcessError(#[from] std::io::Error), /// Child process exited unsuccessfully - #[error( - "Process exited unsuccessfully. exit_code={:?}{}", - exit_code, - match stderr { - Some(s) => format!(" stderr={}", s), - None => "".to_string() - }, - )] + #[error("Process exited unsuccessfully. exit_code={:?}", exit_code)] ProcessFailed { - /// The stderr of the process, if available. - stderr: Option, - /// The exit code of the process + /// The exit code of the failed process exit_code: Option, }, diff --git a/crates/nix_rs/src/flake/eval.rs b/crates/nix_rs/src/flake/eval.rs index d0d3c699..8d3ad37e 100644 --- a/crates/nix_rs/src/flake/eval.rs +++ b/crates/nix_rs/src/flake/eval.rs @@ -1,4 +1,6 @@ //! Work with `nix eval` +use tokio::io::AsyncReadExt; + use crate::command::{CommandError, NixCmd, NixCmdError}; use super::{command::FlakeOptions, url::FlakeUrl}; @@ -12,13 +14,19 @@ pub async fn nix_eval( where T: serde::de::DeserializeOwned, { - let stdout = nixcmd + let mut stdout_handle = nixcmd .run_with_returning_stdout(|cmd| { cmd.args(["eval", "--json"]); opts.use_in_command(cmd); cmd.arg(url.to_string()); }) .await?; + let mut stdout = Vec::new(); + + stdout_handle + .read_to_end(&mut stdout) + .await + .map_err(CommandError::ChildProcessError)?; let v = serde_json::from_slice::(&stdout)?; Ok(v) } @@ -32,29 +40,6 @@ pub async fn nix_eval_attr(cmd: &NixCmd, url: &FlakeUrl) -> Result, where T: Default + serde::de::DeserializeOwned, { - let result = cmd - .run_with_args_expecting_json(&["eval", &url.0, "--json"]) - .await; - match result { - Ok(v) => Ok(Some(v)), - Err(err) if error_is_missing_attribute(&err) => { - Ok(None) // Attr is missing - } - Err(err) => Err(err), - } -} - -/// Check that [NixCmdError] is a missing attribute error -fn error_is_missing_attribute(err: &NixCmdError) -> bool { - match err { - NixCmdError::CmdError(CommandError::ProcessFailed { stderr, .. }) => { - if let Some(stderr) = stderr { - if stderr.contains("does not provide attribute") { - return true; - } - } - false - } - _ => false, - } + cmd.run_with_args_expecting_json(&["eval", &url.0, "--json"]) + .await } diff --git a/crates/nix_rs/src/flake/outputs.rs b/crates/nix_rs/src/flake/outputs.rs index ce5e66d4..5c0b5bc8 100644 --- a/crates/nix_rs/src/flake/outputs.rs +++ b/crates/nix_rs/src/flake/outputs.rs @@ -1,6 +1,6 @@ //! Nix flake outputs -use serde::{Deserialize, Serialize}; +use serde::{de::DeserializeOwned, Deserialize, Serialize}; use std::collections::HashMap; use super::schema::{FlakeSchemas, Val}; @@ -10,11 +10,19 @@ use super::schema::{FlakeSchemas, Val}; #[serde(untagged)] pub enum FlakeOutputs { /// Terminal value that is not an attrset. + #[serde(serialize_with = "value_serializer")] Val(Val), /// An attrset of nested [FlakeOutputs] Attrset(HashMap), } +fn value_serializer(val: &Val, serializer: S) -> Result +where + S: serde::Serializer, +{ + val.value.serialize(serializer) +} + impl FlakeOutputs { /// Get the terminal value pub fn get_val(&self) -> Option<&Val> { @@ -58,6 +66,25 @@ impl FlakeOutputs { } Some(current) } + + /// Lookup the given paths, returning the first match. + pub fn get_first_by_paths(&self, paths: &[&[&str]]) -> Option<&Self> { + for path in paths { + if let Some(v) = self.get_by_path(path) { + return Some(v); + } + } + None + } + + /// Deserialize the FlakeOutputs into a generic type T + pub fn deserialize(&self) -> Result + where + T: Default + DeserializeOwned + std::fmt::Debug, + { + let json_value = serde_json::to_value(self)?; + serde_json::from_value(json_value) + } } impl From for FlakeOutputs { diff --git a/crates/nix_rs/src/flake/schema.rs b/crates/nix_rs/src/flake/schema.rs index 68e47f74..83cef4f7 100644 --- a/crates/nix_rs/src/flake/schema.rs +++ b/crates/nix_rs/src/flake/schema.rs @@ -151,6 +151,8 @@ pub struct Val { pub derivation_name: Option, /// A short description derived from `meta.description` of the derivation with [Val::derivation_name] pub short_description: Option, + /// The raw value of the flake output + pub value: Option, } impl Default for Val { @@ -159,6 +161,7 @@ impl Default for Val { type_: Type::Unknown, derivation_name: None, short_description: None, + value: None, } } } diff --git a/crates/nix_rs/src/flake/url/mod.rs b/crates/nix_rs/src/flake/url/mod.rs index fc26e68f..37c6d30b 100644 --- a/crates/nix_rs/src/flake/url/mod.rs +++ b/crates/nix_rs/src/flake/url/mod.rs @@ -1,6 +1,5 @@ //! Work with flake URLs pub mod attr; mod core; -pub mod qualified_attr; pub use core::*; diff --git a/crates/nix_rs/src/store/command.rs b/crates/nix_rs/src/store/command.rs index e776f7f3..2c0df3e6 100644 --- a/crates/nix_rs/src/store/command.rs +++ b/crates/nix_rs/src/store/command.rs @@ -1,5 +1,5 @@ //! Rust wrapper for `nix-store` -use std::path::PathBuf; +use std::{path::PathBuf, process::Stdio}; use crate::command::{CommandError, NixCmdError}; use serde::{Deserialize, Serialize}; @@ -42,34 +42,47 @@ impl NixStoreCmd { Ok(all_outs) } - /// Return the derivations used to build the given build output. - pub async fn nix_store_query_deriver( + async fn run_query( &self, - out_paths: &[StorePath], - ) -> Result, NixStoreCmdError> { + args: &[&str], + paths: &[impl AsRef], + ) -> Result, NixStoreCmdError> { let mut cmd = self.command(); - cmd.args(["--query", "--valid-derivers"]) - .args(out_paths.iter().map(StorePath::as_path)); - + cmd.args(args) + .args(paths.iter().map(AsRef::as_ref)) + .stdout(Stdio::piped()); crate::command::trace_cmd(&cmd); - let out = cmd.output().await?; - if out.status.success() { - let drv_paths: Vec = String::from_utf8(out.stdout)? - .lines() - .map(PathBuf::from) - .collect(); - if drv_paths.contains(&PathBuf::from("unknown-deriver")) { - return Err(NixStoreCmdError::UnknownDeriver); + let output = cmd.output().await?; + if !output.status.success() { + return Err(CommandError::ProcessFailed { + exit_code: output.status.code(), } - Ok(drv_paths) - } else { - // TODO(refactor): When upstreaming this module to nix-rs, create a - // nicer and unified way to create `ProcessFailed` - let stderr = Some(String::from_utf8_lossy(&out.stderr).to_string()); - let exit_code = out.status.code(); - Err(CommandError::ProcessFailed { stderr, exit_code }.into()) + .into()); } + + Ok(String::from_utf8(output.stdout)? + .lines() + .map(String::from) + .collect()) + } + + /// Return the derivations used to build the given build output. + pub async fn nix_store_query_deriver( + &self, + out_paths: &[StorePath], + ) -> Result, NixStoreCmdError> { + let drv_paths: Vec = self + .run_query(&["--query", "--valid-derivers"], out_paths) + .await? + .into_iter() + .map(PathBuf::from) + .collect(); + + if drv_paths.contains(&PathBuf::from("unknown-deriver")) { + return Err(NixStoreCmdError::UnknownDeriver); + } + Ok(drv_paths) } /// Given the derivation paths, this function recursively queries and return all @@ -78,24 +91,12 @@ impl NixStoreCmd { &self, drv_paths: &[PathBuf], ) -> Result, NixStoreCmdError> { - let mut cmd = self.command(); - cmd.args(["--query", "--requisites", "--include-outputs"]) - .args(drv_paths); - - crate::command::trace_cmd(&cmd); - - let out = cmd.output().await?; - if out.status.success() { - Ok(String::from_utf8(out.stdout)? - .lines() - .map(|line| StorePath::new(PathBuf::from(line))) - .collect()) - } else { - // TODO(refactor): see above - let stderr = Some(String::from_utf8_lossy(&out.stderr).to_string()); - let exit_code = out.status.code(); - Err(CommandError::ProcessFailed { stderr, exit_code }.into()) - } + Ok(self + .run_query(&["--query", "--requisites", "--include-outputs"], drv_paths) + .await? + .into_iter() + .map(|line| StorePath::new(PathBuf::from(line))) + .collect()) } } diff --git a/crates/nix_rs/src/store/path.rs b/crates/nix_rs/src/store/path.rs index cd24927c..ffa2b785 100644 --- a/crates/nix_rs/src/store/path.rs +++ b/crates/nix_rs/src/store/path.rs @@ -1,5 +1,10 @@ //! Store path management -use std::{convert::Infallible, fmt, path::PathBuf, str::FromStr}; +use std::{ + convert::Infallible, + fmt, + path::{Path, PathBuf}, + str::FromStr, +}; use serde_with::{DeserializeFromStr, SerializeDisplay}; @@ -29,6 +34,12 @@ impl From<&StorePath> for PathBuf { } } +impl AsRef for StorePath { + fn as_ref(&self) -> &Path { + self.as_path() + } +} + impl StorePath { /// Create a new `StorePath` from the given path pub fn new(path: PathBuf) -> Self { diff --git a/crates/nixci/CHANGELOG.md b/crates/nixci/CHANGELOG.md index 04a647f7..c15a47e6 100644 --- a/crates/nixci/CHANGELOG.md +++ b/crates/nixci/CHANGELOG.md @@ -1,5 +1,7 @@ ## Unreleased +- Flake schema + - The flake config is now fetched from flake outputs using flake-schemas instead of `nix eval` - New - Introduced notion of 'steps'. Renamed 'build' to 'run'. - Added a step to run `nix flake check` diff --git a/crates/nixci/src/command/core.rs b/crates/nixci/src/command/core.rs index f6f7049c..511b9603 100644 --- a/crates/nixci/src/command/core.rs +++ b/crates/nixci/src/command/core.rs @@ -48,7 +48,7 @@ impl Command { /// Get the nixci [config::Config] associated with this subcommand async fn get_config(&self, cmd: &NixCmd) -> anyhow::Result> { let url = self.get_flake_ref().to_flake_url().await?; - let cfg = crate::config::core::ci_config_from_flake_url(cmd, &url).await?; + let cfg = crate::config::core::ci_config_from_flake_outputs(cmd, &url).await?; tracing::debug!("Config: {cfg:?}"); Ok(cfg) } diff --git a/crates/nixci/src/config/core.rs b/crates/nixci/src/config/core.rs index 725c54ca..8eef4cab 100644 --- a/crates/nixci/src/config/core.rs +++ b/crates/nixci/src/config/core.rs @@ -13,14 +13,14 @@ use super::subflakes::SubflakesConfig; /// let cfg = Config::from_flake_url(&url).await?; /// ``` /// along with the config. -pub async fn ci_config_from_flake_url( +pub async fn ci_config_from_flake_outputs( cmd: &NixCmd, url: &FlakeUrl, ) -> Result> { - let v = omnix_common::config::OmConfig::::from_flake_url( + let v = omnix_common::config::OmConfig::::from_nix( cmd, url, - &["om.ci", "nixci"], + &[&["om", "ci"], &["nixci"]], ) .await?; Ok(v) @@ -38,7 +38,7 @@ mod tests { "github:srid/haskell-flake/76214cf8b0d77ed763d1f093ddce16febaf07365#default.dev" .to_string(), ); - let cfg = ci_config_from_flake_url(&NixCmd::default(), url) + let cfg = ci_config_from_flake_outputs(&NixCmd::default(), url) .await .unwrap(); let (config, attrs) = cfg.get_referenced().unwrap(); diff --git a/crates/omnix-common/Cargo.toml b/crates/omnix-common/Cargo.toml index e7869250..dffa0529 100644 --- a/crates/omnix-common/Cargo.toml +++ b/crates/omnix-common/Cargo.toml @@ -17,6 +17,7 @@ clap-verbosity-flag = { workspace = true } futures-lite = { workspace = true } nix_rs = { workspace = true } serde = { workspace = true } +serde_json = { workspace = true } thiserror = { workspace = true } tokio = { workspace = true } tracing = { workspace = true } diff --git a/crates/omnix-common/src/config.rs b/crates/omnix-common/src/config.rs index 3618bfe5..1e55b536 100644 --- a/crates/omnix-common/src/config.rs +++ b/crates/omnix-common/src/config.rs @@ -3,11 +3,9 @@ use std::collections::BTreeMap; use nix_rs::{ - command::NixCmd, - flake::url::{ - qualified_attr::{nix_eval_qualified_attr, QualifiedAttrError}, - FlakeUrl, - }, + command::{NixCmd, NixCmdError}, + config::{NixConfig, NixConfigError}, + flake::{outputs::FlakeOutputs, schema::FlakeSchemas, url::FlakeUrl}, }; use serde::de::DeserializeOwned; @@ -27,23 +25,37 @@ pub struct OmConfig { } impl OmConfig { - /// Read the Om configuration from the flake URL - pub async fn from_flake_url( + /// Read the Om configuration from a flake + /// TODO: simplify the implementation, it has too many responsibilities + pub async fn from_nix( cmd: &NixCmd, url: &FlakeUrl, - k: &[S], + k: &[&[&str]], ) -> Result, OmConfigError> where - S: AsRef, - T: Default + DeserializeOwned, + T: Default + DeserializeOwned + std::fmt::Debug, { - let (config, reference) = - nix_eval_qualified_attr::, _>(cmd, url, k).await?; - Ok(OmConfig { - flake_url: url.without_attr(), - reference, - config, - }) + let nix_config = NixConfig::get().await.as_ref().unwrap(); + let schema = + FlakeSchemas::from_nix(cmd, &url.without_attr(), &nix_config.system.value).await?; + let outputs = FlakeOutputs::from(schema); + + let cfg = outputs.get_first_by_paths(k); + match cfg { + Some(cfg) => Ok(OmConfig { + flake_url: url.without_attr(), + reference: url.get_attr().as_list(), + config: cfg.deserialize()?, + }), + None => match url.get_attr().0 { + None => Ok(OmConfig { + flake_url: url.without_attr(), + reference: vec![], + config: Default::default(), + }), + Some(attr) => Err(OmConfigError::MissingConfigAttribute(attr)), + }, + } } /// Get the user-referenced config value `T` @@ -77,11 +89,19 @@ impl OmConfig { /// Error type for OmConfig #[derive(thiserror::Error, Debug)] pub enum OmConfigError { - /// Qualified attribute error - #[error("Qualified attribute error: {0}")] - QualifiedAttrError(#[from] QualifiedAttrError), + /// Serde JSON error + #[error("Serde JSON error: {0}")] + SerdeJson(#[from] serde_json::Error), /// Missing configuration attribute #[error("Missing configuration attribute: {0}")] MissingConfigAttribute(String), + + /// Nix command error + #[error("Nix command error: {0}")] + NixCmd(#[from] NixCmdError), + + /// Nix config error + #[error("Nix config error: {0}")] + NixConfig(#[from] NixConfigError), } diff --git a/doc/src/history.md b/doc/src/history.md index ab36f3d1..99ffa38d 100644 --- a/doc/src/history.md +++ b/doc/src/history.md @@ -4,6 +4,10 @@ ### Enhancements +- `flake-schemas` + - Fetch `om` flake config from flake outputs instead of `nix eval` +- Inherit `stderr` from parent process + - Display progress of all `nix` commands by default - `om init` - Initial working version of `om init` command - `om health` diff --git a/flake.lock b/flake.lock index 13d029ba..3286dcc1 100644 --- a/flake.lock +++ b/flake.lock @@ -90,11 +90,11 @@ "inspect": { "flake": false, "locked": { - "lastModified": 1726418650, - "narHash": "sha256-Aabwu7gHQbKDU8mrbG2le6oqpbBl9oTbFvWEBFgXw/4=", + "lastModified": 1727621122, + "narHash": "sha256-GTxRovvYWYn2/LDvjA73YttGuqvtKaOFZfOR9YxtST0=", "owner": "juspay", "repo": "inspect", - "rev": "279cfa5c434e5bbebb4c07fa6af2deeb84e03bb0", + "rev": "e82e65949d2ba5283865609b8728c50ebe7573e3", "type": "github" }, "original": { diff --git a/nix/flake-schemas/flake.nix b/nix/flake-schemas/flake.nix index 988ef455..768522d1 100644 --- a/nix/flake-schemas/flake.nix +++ b/nix/flake-schemas/flake.nix @@ -33,6 +33,36 @@ }) output); }; + mkTraverseSchema = name: doc: { + version = 1; + inherit doc; + inventory = output: + let + traverse = prefix: attrs: flake-schemas.lib.mkChildren (builtins.mapAttrs + (attrName: value: + if (builtins.typeOf value) != "set" then + { + value = value; + what = "${name} config"; + } + else + traverse (prefix + attrName + ".") value + ) + attrs); + in + traverse "" output; + }; + omSchema = mkTraverseSchema "omnix" '' + Configuration for `omnix`. + ''; + + nixciSchema = mkTraverseSchema "nixci" '' + Configuration for `nixci`. + ''; + + nixHealthSchema = mkTraverseSchema "nix-health" '' + Configuration for `nix-health`. + ''; nixosConfigurationsSchema = { version = 1; doc = '' @@ -87,6 +117,11 @@ nixosConfigurations = nixosConfigurationsSchema; homeConfigurations = homeConfigurationsSchema; darwinConfigurations = darwinConfigurationsSchema; + + # Custom schemas + om = omSchema; + nixci = nixciSchema; + nix-health = nixHealthSchema; }; }; }