Changes for esp-idf-sys cmake build (#14)

* cmd* macros: Add ability to specify argument collection with `@<expression>`

* Add fs utilities

Add dependency `remove_dir_all`
Note: On windows `std::fs::remove_dir_all` doesn't really work (rust-lang/rust#29497).

* Allow errors to be turned into cargo warnings

* Add git utilities

Publicly export the `which` crate
Remove `log` dependency of `cmd*` macros.

* Add initial cmake tools

Add dependency `cmake`
Add initial subset of cmake file API
Add function to extract defined variables in cmake script

* Add cmake-file-API cache object

* Refactor `bindgen`, add `from_cmake` contructor

* cmake: Add toolchains object

* Add type alias `NativeCommandArgs`

* Make `LinkArgsBuilder` methods usable

* Construct `LinkArgsBuilder`, `CInclArgs` from cmake build info

* Add `CfgArgs::try_from_json`

* Don't detect bindgen linker from cache

Fix kconfig json parse bug

* Fix clippy warnings

* Fix `git::Repository::apply`

Add `git::Repository::apply_once`
Add `git::Repository::is_applied`
Publicly export `anyhow` (hidden from docs) because some macros need it.
Fix `cmd_spawn`
Add `cmd` variation that returns `Result<ExitStatus>`

* Improve cmake-file-api error messages

Check the cmake version for support of a specific file-api object kind.

Author: N3xed

Cargo.toml

@@ -28,3 +28,5 @@ bindgen = "0.57"
 xmas-elf = "0.8"
 bitflags = "1.3"
 shlex = "1.0"
+remove_dir_all = "0.7"
+cmake = "0.1"

src/bindgen.rs

@@ -1,55 +1,102 @@
+use std::ffi::OsStr;
 use std::path::{Path, PathBuf};
-use std::process::Command;
 use std::{env, fs};
 
 use anyhow::*;
 
-use crate::pio::project::SconsVariables;
 use crate::utils::OsStrExt;
+use crate::{cargo, cli, cmake, cmd, cmd_output, pio};
 
 pub const VAR_BINDINGS_FILE: &str = "EMBUILD_GENERATED_BINDINGS_FILE";
 
-#[cfg(windows)]
-const EXE_SUFFIX: &str = ".exe";
-
-#[cfg(not(windows))]
-const EXE_SUFFIX: &str = "";
-
-#[cfg(windows)]
-const FS_CASE_INSENSITIVE: bool = true;
-
-#[cfg(not(windows))]
-const FS_CASE_INSENSITIVE: bool = false;
-
 #[derive(Clone, Default, Debug)]
 pub struct Factory {
     pub clang_args: Vec<String>,
     pub linker: Option<PathBuf>,
     pub mcu: Option<String>,
+    pub force_cpp: bool,
+    pub sysroot: Option<PathBuf>,
 }
 
 impl Factory {
-    pub fn from_scons_vars(scons_vars: &SconsVariables) -> Result<Self> {
+    pub fn from_scons_vars(scons_vars: &pio::project::SconsVariables) -> Result<Self> {
+        let clang_args = cli::NativeCommandArgs::new(&scons_vars.incflags)
+            .chain(cli::NativeCommandArgs::new(
+                scons_vars
+                    .clangargs
+                    .as_deref()
+                    .unwrap_or_default(),
+            ))
+            .collect();
+
         Ok(Self {
-            clang_args: Self::get_pio_clang_args(
-                &scons_vars.incflags,
-                scons_vars.clangargs.clone(),
-            ),
+            clang_args,
             linker: Some(scons_vars.full_path(scons_vars.link.clone())?),
             mcu: Some(scons_vars.mcu.clone()),
+            force_cpp: false,
+            sysroot: None,
         })
     }
 
-    pub fn builder(&self) -> Result<bindgen::Builder> {
+    pub fn from_cmake(compile_group: &cmake::codemodel::target::CompileGroup) -> Result<Self> {
+        use crate::cmake::codemodel::Language;
+        assert!(
+            compile_group.language == Language::C || compile_group.language == Language::Cpp,
+            "Generating bindings for languages other than C/C++ is not supported"
+        );
+
+        let clang_args = compile_group
+            .defines
+            .iter()
+            .map(|d| format!("-D{}", d.define))
+            .chain(
+                compile_group
+                    .includes
+                    .iter()
+                    .map(|i| format!("-I{}", &i.path)),
+            )
+            .collect();
+
+        Ok(Self {
+            clang_args,
+            linker: None,
+            force_cpp: compile_group.language == Language::Cpp,
+            mcu: None,
+            sysroot: compile_group.sysroot.as_ref().map(|s| s.path.clone()),
+        })
+    }
+
+    /// Set the linker used to determine the sysroot to be used for generating bindings.
+    pub fn with_linker(mut self, linker: impl Into<PathBuf>) -> Self {
+        self.linker = Some(linker.into());
+        self
+    }
+
+    pub fn builder(self) -> Result<bindgen::Builder> {
         self.create_builder(false)
     }
 
-    pub fn cpp_builder(&self) -> Result<bindgen::Builder> {
+    pub fn cpp_builder(self) -> Result<bindgen::Builder> {
         self.create_builder(true)
     }
 
-    fn create_builder(&self, cpp: bool) -> Result<bindgen::Builder> {
-        let sysroot = self.get_sysroot()?;
+    fn create_builder(self, cpp: bool) -> Result<bindgen::Builder> {
+        let cpp = self.force_cpp || cpp;
+        let sysroot = self
+            .sysroot
+            .clone()
+            .map_or_else(|| try_get_sysroot(&self.linker), Ok)?;
+
+        let sysroot_args = [
+            format!("--sysroot={}", sysroot.try_to_str()?),
+            format!("-I{}", sysroot.join("include").try_to_str()?),
+        ];
+
+        let cpp_args = if cpp {
+            get_cpp_includes(&sysroot)?
+        } else {
+            vec![]
+        };
 
         let builder = bindgen::Builder::default()
             .use_core()
@@ -58,130 +105,25 @@ impl Factory {
             .derive_default(true)
             //.ctypes_prefix(c_types)
             .clang_arg("-D__bindgen")
-            .clang_arg(format!("--sysroot={}", sysroot.display()))
-            .clang_arg(format!("-I{}", sysroot.join("include").try_to_str()?))
+            .clang_args(sysroot_args)
             .clang_args(&["-x", if cpp { "c++" } else { "c" }])
-            .clang_args(if cpp {
-                Self::get_cpp_includes(sysroot)?
-            } else {
-                Vec::new()
-            })
+            .clang_args(cpp_args)
             .clang_args(&self.clang_args);
 
-        eprintln!(
+        log::debug!(
             "Bindgen builder factory flags: {:?}",
             builder.command_line_flags()
         );
 
         Ok(builder)
     }
-
-    fn get_sysroot(&self) -> Result<PathBuf> {
-        let linker = if let Some(linker) = self.linker.as_ref() {
-            linker
-                .clone()
-                .into_os_string()
-                .into_string()
-                .map_err(|_| anyhow!("Cannot convert the linker variable to String"))?
-        } else if let Ok(linker) = env::var("RUSTC_LINKER") {
-            linker
-        } else {
-            bail!("No explicit linker, and env var RUSTC_LINKER not defined either");
-        };
-
-        let gcc = format!("gcc{}", EXE_SUFFIX);
-        let gcc_suffix = format!("-{}", gcc);
-
-        let linker_canonicalized = if FS_CASE_INSENSITIVE {
-            linker.to_lowercase()
-        } else {
-            linker.clone()
-        };
-
-        let linker = if linker_canonicalized == gcc || linker_canonicalized.ends_with(&gcc_suffix) {
-            // For whatever reason, --print-sysroot does not work with GCC
-            // Change it to LD
-            format!("{}ld{}", &linker[0..linker.len() - gcc.len()], EXE_SUFFIX)
-        } else {
-            linker
-        };
-
-        let output = Command::new(linker).arg("--print-sysroot").output()?;
-
-        let path_str = String::from_utf8(output.stdout)?;
-
-        Ok(PathBuf::from(path_str.trim()))
-    }
-
-    fn get_cpp_includes(sysroot: impl AsRef<Path>) -> Result<Vec<String>> {
-        let sysroot = sysroot.as_ref();
-        let cpp_includes_root = sysroot.join("include").join("c++");
-
-        let cpp_version = fs::read_dir(&cpp_includes_root)?
-            .map(|dir_entry_r| dir_entry_r.map(|dir_entry| dir_entry.path()))
-            .fold(None, |ao: Option<PathBuf>, sr: Result<PathBuf, _>| {
-                if let Some(a) = ao.as_ref() {
-                    sr.ok()
-                        .map_or(ao.clone(), |s| if a >= &s { ao.clone() } else { Some(s) })
-                } else {
-                    sr.ok()
-                }
-            });
-
-        if let Some(cpp_version) = cpp_version {
-            let mut cpp_include_paths = vec![
-                format!("-I{}", cpp_version.try_to_str()?),
-                format!("-I{}", cpp_version.join("backward").try_to_str()?),
-            ];
-
-            if let Some(sysroot_last_segment) = fs::canonicalize(sysroot)?.file_name() {
-                cpp_include_paths.push(format!(
-                    "-I{}",
-                    cpp_version.join(sysroot_last_segment).try_to_str()?
-                ));
-            }
-
-            Ok(cpp_include_paths)
-        } else {
-            Ok(Vec::new())
-        }
-    }
-
-    fn get_pio_clang_args(
-        incflags: impl AsRef<str>,
-        extra_args: Option<impl AsRef<str>>,
-    ) -> Vec<String> {
-        let mut result = incflags
-            .as_ref()
-            .split(' ')
-            .map(str::to_string)
-            .collect::<Vec<_>>();
-
-        if let Some(extra_args) = extra_args {
-            result.append(
-                &mut extra_args
-                    .as_ref()
-                    .split(' ')
-                    .map(str::to_string)
-                    .collect::<Vec<_>>(),
-            );
-        }
-
-        result
-    }
 }
 
 pub fn run(builder: bindgen::Builder) -> Result<()> {
     let output_file = PathBuf::from(env::var("OUT_DIR")?).join("bindings.rs");
-
     run_for_file(builder, &output_file)?;
 
-    println!(
-        "cargo:rustc-env={}={}",
-        VAR_BINDINGS_FILE,
-        output_file.display()
-    );
-
+    cargo::set_rustc_env(VAR_BINDINGS_FILE, output_file.try_to_str()?);
     Ok(())
 }
 
@@ -199,12 +141,77 @@ pub fn run_for_file(builder: bindgen::Builder, output_file: impl AsRef<Path>) ->
 
     // Run rustfmt on the generated bindings separately, because custom toolchains often do not have rustfmt
     // Hence why we need to use the rustfmt from the stable buildchain (where the assumption is, it is already installed)
-    Command::new("rustup")
-        .arg("run")
-        .arg("stable")
-        .arg("rustfmt")
-        .arg(output_file)
-        .status()?;
-
+    cmd!("rustup", "run", "stable", "rustfmt", output_file)?;
     Ok(())
 }
+
+fn try_get_sysroot(linker: &Option<impl AsRef<Path>>) -> Result<PathBuf> {
+    let linker = if let Some(ref linker) = linker {
+        linker.as_ref().to_owned()
+    } else if let Some(linker) = env::var_os("RUSTC_LINKER") {
+        PathBuf::from(linker)
+    } else {
+        bail!("Could not determine linker: No explicit linker and `RUSTC_LINKER` not set");
+    };
+
+    let gcc_file_stem = linker
+        .file_stem()
+        .and_then(OsStr::to_str)
+        .filter(|&s| s == "gcc" || s.ends_with("-gcc"));
+
+    // For whatever reason, --print-sysroot does not work with GCC
+    // Change it to LD
+    let linker = if let Some(stem) = gcc_file_stem {
+        let mut ld_linker =
+            linker.with_file_name(format!("{}{}", stem.strip_suffix("gcc").unwrap(), "ld"));
+        if let Some(ext) = linker.extension() {
+            ld_linker.set_extension(ext);
+        }
+        ld_linker
+    } else {
+        linker
+    };
+
+    cmd_output!(linker, "--print-sysroot")
+        .with_context(|| {
+            anyhow!(
+                "Could not determine sysroot from linker '{}'",
+                linker.display()
+            )
+        })
+        .map(PathBuf::from)
+}
+
+fn get_cpp_includes(sysroot: impl AsRef<Path>) -> Result<Vec<String>> {
+    let sysroot = sysroot.as_ref();
+    let cpp_includes_root = sysroot.join("include").join("c++");
+
+    let cpp_version = fs::read_dir(&cpp_includes_root)?
+        .map(|dir_entry_r| dir_entry_r.map(|dir_entry| dir_entry.path()))
+        .fold(None, |ao: Option<PathBuf>, sr: Result<PathBuf, _>| {
+            if let Some(a) = ao.as_ref() {
+                sr.ok()
+                    .map_or(ao.clone(), |s| if a >= &s { ao.clone() } else { Some(s) })
+            } else {
+                sr.ok()
+            }
+        });
+
+    if let Some(cpp_version) = cpp_version {
+        let mut cpp_include_paths = vec![
+            format!("-I{}", cpp_version.try_to_str()?),
+            format!("-I{}", cpp_version.join("backward").try_to_str()?),
+        ];
+
+        if let Some(sysroot_last_segment) = fs::canonicalize(sysroot)?.file_name() {
+            cpp_include_paths.push(format!(
+                "-I{}",
+                cpp_version.join(sysroot_last_segment).try_to_str()?
+            ));
+        }
+
+        Ok(cpp_include_paths)
+    } else {
+        Ok(Vec::new())
+    }
+}

src/build.rs

@@ -137,17 +137,22 @@ pub struct LinkArgsBuilder {
 }
 
 impl LinkArgsBuilder {
-    pub fn force_ldproxy(&mut self, value: bool) -> &mut Self {
+    pub fn force_ldproxy(mut self, value: bool) -> Self {
         self.force_ldproxy = value;
         self
     }
+    
+    pub fn linker(mut self, path: impl Into<PathBuf>) -> Self {
+        self.linker = Some(path.into());
+        self
+    }
 
-    pub fn working_directory(&mut self, dir: impl AsRef<Path>) -> &mut Self {
+    pub fn working_directory(mut self, dir: impl AsRef<Path>) -> Self {
         self.working_directory = Some(dir.as_ref().to_owned());
         self
     }
 
-    pub fn dedup_libs(&mut self, dedup: bool) -> &mut Self {
+    pub fn dedup_libs(mut self, dedup: bool) -> Self {
         self.dedup_libs = dedup;
         self
     }
@@ -166,11 +171,11 @@ impl LinkArgsBuilder {
             .unwrap_or(false);
 
         if self.force_ldproxy && !detected_ldproxy {
-            print_warning(concat!(
-                "The linker arguments force the usage of `ldproxy` but the linker used ",
-                "by cargo is different. Please set the linker to `ldproxy` in your cargo config ",
-                "or set `force_ldproxy` to `false`."
-            ));
+            print_warning(
+                "The linker arguments force the usage of `ldproxy` but the linker used \
+                 by cargo is different. Please set the linker to `ldproxy` in your cargo config \
+                 or set `force_ldproxy` to `false`."
+            );
         }
 
         if self.force_ldproxy || detected_ldproxy {