flowey: vmm_tests: introduce step to verify all tests run on at least one runner

flowey/flowey_lib_common/src/gen_cargo_nextest_list_cmd.rs

@@ -0,0 +1,93 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT License.
+
+//! Run cargo-nextest list subcommand.
+use crate::gen_cargo_nextest_run_cmd;
+use flowey::node::prelude::*;
+use std::collections::BTreeMap;
+
+flowey_request! {
+    pub struct Request {
+        /// What kind of test run this is (inline build vs. from nextest archive).
+        pub run_kind_deps: gen_cargo_nextest_run_cmd::RunKindDeps,
+        /// Working directory the test archive was created from.
+        pub working_dir: ReadVar<PathBuf>,
+        /// Path to `.config/nextest.toml`
+        pub config_file: ReadVar<PathBuf>,
+        /// Nextest profile to use when running the source code
+        pub nextest_profile: String,
+        /// Nextest test filter expression
+        pub nextest_filter_expr: Option<String>,
+        /// Additional env vars set when executing the tests.
+        pub extra_env: Option<ReadVar<BTreeMap<String, String>>>,
+        /// Generated cargo-nextest list command
+        pub command: WriteVar<gen_cargo_nextest_run_cmd::Command>,
+    }
+}
+
+new_flow_node!(struct Node);
+
+impl FlowNode for Node {
+    type Request = Request;
+
+    fn imports(ctx: &mut ImportCtx<'_>) {
+        ctx.import::<gen_cargo_nextest_run_cmd::Node>();
+    }
+
+    fn emit(requests: Vec<Request>, ctx: &mut NodeCtx<'_>) -> anyhow::Result<()> {
+        for Request {
+            run_kind_deps,
+            working_dir,
+            config_file,
+            nextest_profile,
+            nextest_filter_expr,
+            extra_env,
+            command: list_cmd,
+        } in requests
+        {
+            if let gen_cargo_nextest_run_cmd::RunKindDeps::BuildAndRun {
+                params: _,
+                nextest_installed: _,
+                rust_toolchain: _,
+                cargo_flags: _,
+            } = run_kind_deps
+            {
+                anyhow::bail!("BuildAndRun is not supported.")
+            }
+
+            let run_cmd = ctx.reqv(|v| gen_cargo_nextest_run_cmd::Request {
+                run_kind_deps,
+                working_dir,
+                config_file,
+                tool_config_files: Vec::new(), // Ignored
+                nextest_profile,
+                extra_env,
+                nextest_filter_expr,
+                run_ignored: true,
+                fail_fast: None,
+                portable: false,
+                command: v,
+            });
+
+            ctx.emit_rust_step("generate nextest list command", |ctx| {
+                let run_cmd = run_cmd.claim(ctx);
+                let list_cmd = list_cmd.claim(ctx);
+                move |rt| {
+                    let mut cmd = rt.read(run_cmd);
+                    cmd.args = cmd
+                        .args
+                        .into_iter()
+                        .map(|arg| if arg == "run" { "list".into() } else { arg })
+                        .collect();
+                    cmd.args.extend(["--message-format".into(), "json".into()]);
+
+                    rt.write(list_cmd, &cmd);
+                    log::info!("Generated command: {}", cmd);
+
+                    Ok(())
+                }
+            });
+        }
+        Ok(())
+    }
+}

flowey/flowey_lib_common/src/lib.rs

@@ -31,6 +31,7 @@ pub mod download_mdbook_admonish;
 pub mod download_mdbook_mermaid;
 pub mod download_nuget_exe;
 pub mod download_protoc;
+pub mod gen_cargo_nextest_list_cmd;
 pub mod gen_cargo_nextest_run_cmd;
 pub mod gh_download_azure_key_vault_secret;
 pub mod gh_latest_completed_workflow_id;

flowey/flowey_lib_common/src/publish_test_results.rs

@@ -24,6 +24,8 @@ flowey_request! {
         /// To keep making forward progress, I've tweaked this node to accept an
         /// optional... but this ain't great.
         pub junit_xml: ReadVar<Option<PathBuf>>,
+        /// Path to the JSON output from `nextest list --json`.
+        pub nextest_list_json: Option<ReadVar<Option<PathBuf>>>,
         /// Brief string used when publishing the test.
         /// Must be unique to the pipeline.
         pub test_label: String,
@@ -57,6 +59,7 @@ impl FlowNode for Node {
 
         for Request {
             junit_xml,
+            nextest_list_json,
             test_label: label,
             attachments,
             output_dir,
@@ -77,6 +80,13 @@ impl FlowNode for Node {
             let has_junit_xml = junit_xml.map(ctx, |p| p.is_some());
             let junit_xml = junit_xml.map(ctx, |p| p.unwrap_or_default());
 
+            let has_nextest_list = nextest_list_json
+                .as_ref()
+                .map(|v| v.map(ctx, |p| p.is_some()));
+            let nextest_list_json = nextest_list_json
+                .as_ref()
+                .map(|v| v.map(ctx, |p| p.unwrap_or_default()));
+
             match ctx.backend() {
                 FlowBackend::Ado => {
                     use_side_effects.push(ctx.reqv(|v| {
@@ -96,6 +106,25 @@ impl FlowNode for Node {
                         p.absolute().expect("invalid path").display().to_string()
                     });
 
+                    if let Some(nextest_list_json) = nextest_list_json {
+                        let nextest_list_json = nextest_list_json.map(ctx, |p| {
+                            p.absolute().expect("invalid path").display().to_string()
+                        });
+
+                        let has_nextest_list = has_nextest_list.unwrap();
+
+                        use_side_effects.push(
+                            ctx.emit_gh_step(
+                                format!("publish test results: {label} (nextest list JSON)"),
+                                "actions/upload-artifact@v4",
+                            )
+                            .condition(has_nextest_list)
+                            .with("name", format!("{label}-nextest-list-json"))
+                            .with("path", nextest_list_json)
+                            .finish(ctx),
+                        );
+                    }
+
                     // Note: usually flowey's built-in artifact publishing API
                     // should be used instead of this, but here we need to
                     // manually upload the artifact now so that it is still

flowey/flowey_lib_common/src/run_cargo_nextest_run.rs

@@ -6,11 +6,16 @@
 use crate::gen_cargo_nextest_run_cmd::RunKindDeps;
 use flowey::node::prelude::*;
 use std::collections::BTreeMap;
+use std::io::Write;
+use std::process::ExitStatus;
+use std::process::Stdio;
 #[derive(Serialize, Deserialize)]
 pub struct TestResults {
     pub all_tests_passed: bool,
     /// Path to JUnit XML output (if enabled by the nextest profile)
     pub junit_xml: Option<PathBuf>,
+    /// Path to JSON file containing output of `cargo nextest list` command (if running in ci)
+    pub nextest_list_output: Option<PathBuf>,
 }
 
 /// Parameters related to building nextest tests
@@ -135,6 +140,7 @@ impl FlowNode for Node {
         ctx.import::<crate::download_cargo_nextest::Node>();
         ctx.import::<crate::install_cargo_nextest::Node>();
         ctx.import::<crate::install_rust::Node>();
+        ctx.import::<crate::gen_cargo_nextest_list_cmd::Node>();
         ctx.import::<crate::gen_cargo_nextest_run_cmd::Node>();
     }
 
@@ -211,6 +217,30 @@ impl FlowNode for Node {
                 }
             };
 
+            let list_cmd = match &run_kind_deps {
+                RunKindDeps::BuildAndRun { .. } => None,
+                RunKindDeps::RunFromArchive {
+                    archive_file,
+                    nextest_bin,
+                    target,
+                } => {
+                    let list_cmd = ctx.reqv(|v| crate::gen_cargo_nextest_list_cmd::Request {
+                        run_kind_deps: RunKindDeps::RunFromArchive {
+                            archive_file: archive_file.clone(),
+                            nextest_bin: nextest_bin.clone(),
+                            target: target.clone(),
+                        },
+                        working_dir: working_dir.clone(),
+                        config_file: config_file.clone(),
+                        nextest_profile: nextest_profile.clone(),
+                        nextest_filter_expr: None, // Ignored
+                        extra_env: extra_env.clone(),
+                        command: v,
+                    });
+                    Some(list_cmd)
+                }
+            };
+
             let cmd = ctx.reqv(|v| crate::gen_cargo_nextest_run_cmd::Request {
                 run_kind_deps,
                 working_dir: working_dir.clone(),
@@ -227,6 +257,7 @@ impl FlowNode for Node {
 
             let (all_tests_passed_read, all_tests_passed_write) = ctx.new_var();
             let (junit_xml_read, junit_xml_write) = ctx.new_var();
+            let (nextest_list_output_file_read, nextest_list_output_file_write) = ctx.new_var();
 
             ctx.emit_rust_step(format!("run '{friendly_name}' nextest tests"), |ctx| {
                 pre_run_deps.claim(ctx);
@@ -235,12 +266,15 @@ impl FlowNode for Node {
                 let config_file = config_file.claim(ctx);
                 let all_tests_passed_var = all_tests_passed_write.claim(ctx);
                 let junit_xml_write = junit_xml_write.claim(ctx);
+                let nextest_list_output_file_write = nextest_list_output_file_write.claim(ctx);
+                let list_cmd = list_cmd.claim(ctx);
                 let cmd = cmd.claim(ctx);
 
                 move |rt| {
                     let working_dir = rt.read(working_dir);
                     let config_file = rt.read(config_file);
                     let cmd = rt.read(cmd);
+                    let list_cmd = rt.read(list_cmd);
 
                     // first things first - determine if junit is supported by
                     // the profile, and if so, where the output if going to be.
@@ -310,17 +344,7 @@ impl FlowNode for Node {
                     // exit code of the process.
                     //
                     // So we have to use the raw process API instead.
-                    let mut command = std::process::Command::new(&cmd.argv0);
-                    command
-                        .args(&cmd.args)
-                        .envs(&cmd.env)
-                        .current_dir(&working_dir);
-
-                    let mut child = command.spawn().with_context(|| {
-                        format!("failed to spawn '{}'", cmd.argv0.to_string_lossy())
-                    })?;
-
-                    let status = child.wait()?;
+                    let (status, _stdout) = run_command(&cmd, &working_dir, false)?;
 
                     #[cfg(unix)]
                     if let Some((soft, hard)) = old_core_rlimits {
@@ -369,24 +393,52 @@ impl FlowNode for Node {
 
                     rt.write(junit_xml_write, &junit_xml);
 
+                    // run the list command to get all tests in the executable
+                    if let Some(list_cmd) = list_cmd {
+                        let (status, stdout_opt) = run_command(&list_cmd, &working_dir, true)?;
+                        anyhow::ensure!(status.success(), "failed to list tests in executable");
+                        let stdout =
+                            stdout_opt.context("missing stdout from nextest list command")?;
+                        let nextest_list_json = get_nextest_list_output_from_stdout(&stdout)?;
+                        if let Some(ref junit_xml_path) = junit_xml {
+                            anyhow::ensure!(
+                                junit_xml_path.is_file(),
+                                "expected junit xml to exist at {:?}",
+                                junit_xml_path
+                            );
+                            let containing_dir = junit_xml_path
+                                .parent()
+                                .context("junit xml has no parent")?
+                                .to_path_buf();
+                            let output_path = containing_dir.join("nextest_list.json");
+                            let mut file = fs_err::File::create_new(output_path.clone())?;
+                            file.write_all(nextest_list_json.to_string().as_bytes())?;
+
+                            rt.write(nextest_list_output_file_write, &output_path.absolute()?);
+                        }
+                    }
+
                     Ok(())
                 }
             });
 
             ctx.emit_minor_rust_step("write results", |ctx| {
                 let all_tests_passed = all_tests_passed_read.claim(ctx);
                 let junit_xml = junit_xml_read.claim(ctx);
+                let nextest_list_output = nextest_list_output_file_read.claim(ctx);
                 let results = results.claim(ctx);
 
                 move |rt| {
                     let all_tests_passed = rt.read(all_tests_passed);
                     let junit_xml = rt.read(junit_xml);
+                    let nextest_list_output = rt.read(nextest_list_output);
 
                     rt.write(
                         results,
                         &TestResults {
                             all_tests_passed,
                             junit_xml,
+                            nextest_list_output: Some(nextest_list_output),
                         },
                     );
                 }
@@ -421,3 +473,43 @@ impl build_params::NextestBuildParams {
         }
     }
 }
+
+fn run_command(
+    cmd: &crate::gen_cargo_nextest_run_cmd::Command,
+    working_dir: &PathBuf,
+    capture_stdout: bool,
+) -> anyhow::Result<(ExitStatus, Option<String>)> {
+    let mut command = std::process::Command::new(&cmd.argv0);
+    command
+        .args(&cmd.args)
+        .envs(&cmd.env)
+        .current_dir(working_dir);
+
+    if capture_stdout {
+        command.stdout(Stdio::piped());
+    } else {
+        command.stdout(Stdio::inherit());
+    }
+
+    let mut child = command
+        .spawn()
+        .with_context(|| format!("failed to spawn '{}'", cmd.argv0.to_string_lossy()))?;
+
+    if capture_stdout {
+        let output = child.wait_with_output()?;
+        let stdout_str = String::from_utf8_lossy(&output.stdout).into_owned();
+        Ok((output.status, Some(stdout_str)))
+    } else {
+        let status = child.wait()?;
+        Ok((status, None))
+    }
+}
+
+fn get_nextest_list_output_from_stdout(output: &str) -> anyhow::Result<serde_json::Value> {
+    for line in output.lines() {
+        if let Ok(json_value) = serde_json::from_str::<serde_json::Value>(line) {
+            return Ok(json_value);
+        }
+    }
+    anyhow::bail!("failed to find JSON output in nextest list command output");
+}

flowey/flowey_lib_hvlite/src/_jobs/build_and_run_nextest_unit_tests.rs

@@ -68,6 +68,7 @@ impl SimpleFlowNode for Node {
         let junit_xml = results.map(ctx, |r| r.junit_xml);
         let reported_results = ctx.reqv(|v| flowey_lib_common::publish_test_results::Request {
             junit_xml,
+            nextest_list_json: None,
             test_label: junit_test_label,
             attachments: BTreeMap::new(),
             output_dir: artifact_dir,

flowey/flowey_lib_hvlite/src/_jobs/consume_and_test_nextest_unit_tests_archive.rs

@@ -58,6 +58,7 @@ impl SimpleFlowNode for Node {
         let junit_xml = results.map(ctx, |r| r.junit_xml);
         let reported_results = ctx.reqv(|v| flowey_lib_common::publish_test_results::Request {
             junit_xml,
+            nextest_list_json: None,
             test_label: junit_test_label,
             attachments: BTreeMap::new(),
             output_dir: artifact_dir,

flowey/flowey_lib_hvlite/src/_jobs/consume_and_test_nextest_vmm_tests_archive.rs

@@ -205,8 +205,10 @@ impl SimpleFlowNode for Node {
         let test_log_path = test_log_path.depending_on(ctx, &results);
 
         let junit_xml = results.map(ctx, |r| r.junit_xml);
+        let nextest_list_json = results.map(ctx, |r| r.nextest_list_output);
         let reported_results = ctx.reqv(|v| flowey_lib_common::publish_test_results::Request {
             junit_xml,
+            nextest_list_json: Some(nextest_list_json),
             test_label: junit_test_label,
             attachments: BTreeMap::from([("logs".to_string(), (test_log_path, false))]),
             output_dir: artifact_dir,

flowey/flowey_lib_hvlite/src/_jobs/local_build_and_run_nextest_vmm_tests.rs

@@ -931,6 +931,7 @@ impl SimpleFlowNode for Node {
             let published_results =
                 ctx.reqv(|v| flowey_lib_common::publish_test_results::Request {
                     junit_xml,
+                    nextest_list_json: None,
                     test_label,
                     attachments: BTreeMap::new(), // the logs are already there
                     output_dir: Some(ReadVar::from_static(test_content_dir)),