Skip to content

flowey: convert IGVM-related nodes to use typed artifacts #1293

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 2 commits into
base: main
Choose a base branch
from
Open

flowey: convert IGVM-related nodes to use typed artifacts #1293

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
2291b2a
flowey: convert IGVM-related nodes to use typed artifacts
jstarks May 2, 2025
7db52b7
clippy
jstarks May 5, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
562 changes: 258 additions & 304 deletions .github/workflows/openvmm-ci.yaml
View file
Open in desktop

Large diffs are not rendered by default.

559 changes: 256 additions & 303 deletions .github/workflows/openvmm-pr.yaml
View file
Open in desktop

Large diffs are not rendered by default.

21 changes: 10 additions & 11 deletions flowey/flowey_hvlite/src/pipelines/checkin_gates.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -630,9 +630,9 @@ impl IntoPipeline for CheckinGatesCli {
};

let (pub_openhcl_igvm, use_openhcl_igvm) =
pipeline.new_artifact(format!("{arch_tag}-openhcl-igvm"));
pipeline.new_typed_artifact(format!("{arch_tag}-openhcl-igvm"));
let (pub_openhcl_igvm_extras, _use_openhcl_igvm_extras) =
pipeline.new_artifact(format!("{arch_tag}-openhcl-igvm-extras"));
pipeline.new_typed_artifact(format!("{arch_tag}-openhcl-igvm-extras"));

let (pub_openhcl_baseline, _use_openhcl_baseline) =
if matches!(config, PipelineConfig::Ci) {
Expand Down Expand Up @@ -715,11 +715,9 @@ impl IntoPipeline for CheckinGatesCli {
))),
})
.collect(),
artifact_dir_openhcl_igvm: ctx.publish_artifact(pub_openhcl_igvm),
artifact_dir_openhcl_igvm_extras: ctx
.publish_artifact(pub_openhcl_igvm_extras),
openhcl_igvm: ctx.publish_typed_artifact(pub_openhcl_igvm),
openhcl_igvm_extras: ctx.publish_typed_artifact(pub_openhcl_igvm_extras),
artifact_openhcl_verify_size_baseline: publish_baseline_artifact,
done: ctx.new_done_handle(),
}
})
.dep_on(|ctx| flowey_lib_hvlite::build_pipette::Request {
Expand Down Expand Up @@ -1135,6 +1133,7 @@ impl IntoPipeline for CheckinGatesCli {
// of thing that would really benefit from a derive macro.
mod vmm_tests_artifact_builders {
use flowey::pipeline::prelude::*;
use flowey_lib_hvlite::_jobs::build_and_publish_openhcl_igvm_from_recipe::OpenhclIgvmSet;
use flowey_lib_hvlite::_jobs::consume_and_test_nextest_vmm_tests_archive::VmmTestsDepArtifacts;
use flowey_lib_hvlite::build_guest_test_uefi::GuestTestUefiOutput;
use flowey_lib_hvlite::build_openvmm::OpenvmmOutput;
Expand Down Expand Up @@ -1184,7 +1183,7 @@ mod vmm_tests_artifact_builders {
tmk_vmm: Some(ctx.use_typed_artifact(&use_tmk_vmm)),
tmks: Some(ctx.use_typed_artifact(&use_tmks)),
// not currently required, since OpenHCL tests cannot be run on OpenVMM on linux
artifact_dir_openhcl_igvm_files: None,
openhcl_igvm_files: None,
tmk_vmm_linux_musl: None,
}))
}
Expand All @@ -1197,7 +1196,7 @@ mod vmm_tests_artifact_builders {
pub use_pipette_windows: Option<UseTypedArtifact<PipetteOutput>>,
pub use_tmk_vmm: Option<UseTypedArtifact<TmkVmmOutput>>,
// linux build machine
pub use_openhcl_igvm_files: Option<UseArtifact>,
pub use_openhcl_igvm_files: Option<UseTypedArtifact<OpenhclIgvmSet>>,
pub use_pipette_linux_musl: Option<UseTypedArtifact<PipetteOutput>>,
pub use_tmk_vmm_linux_musl: Option<UseTypedArtifact<TmkVmmOutput>>,
// any machine
Expand Down Expand Up @@ -1232,7 +1231,7 @@ mod vmm_tests_artifact_builders {
pipette_windows: Some(ctx.use_typed_artifact(&use_pipette_windows)),
pipette_linux_musl: Some(ctx.use_typed_artifact(&use_pipette_linux_musl)),
guest_test_uefi: Some(ctx.use_typed_artifact(&use_guest_test_uefi)),
artifact_dir_openhcl_igvm_files: Some(ctx.use_artifact(&use_openhcl_igvm_files)),
openhcl_igvm_files: Some(ctx.use_typed_artifact(&use_openhcl_igvm_files)),
tmk_vmm: Some(ctx.use_typed_artifact(&use_tmk_vmm)),
tmk_vmm_linux_musl: Some(ctx.use_typed_artifact(&use_tmk_vmm_linux_musl)),
tmks: Some(ctx.use_typed_artifact(&use_tmks)),
Expand All @@ -1247,7 +1246,7 @@ mod vmm_tests_artifact_builders {
pub use_pipette_windows: Option<UseTypedArtifact<PipetteOutput>>,
pub use_tmk_vmm: Option<UseTypedArtifact<TmkVmmOutput>>,
// linux build machine
pub use_openhcl_igvm_files: Option<UseArtifact>,
pub use_openhcl_igvm_files: Option<UseTypedArtifact<OpenhclIgvmSet>>,
pub use_pipette_linux_musl: Option<UseTypedArtifact<PipetteOutput>>,
pub use_tmk_vmm_linux_musl: Option<UseTypedArtifact<TmkVmmOutput>>,
// any machine
Expand Down Expand Up @@ -1282,7 +1281,7 @@ mod vmm_tests_artifact_builders {
pipette_windows: Some(ctx.use_typed_artifact(&use_pipette_windows)),
pipette_linux_musl: Some(ctx.use_typed_artifact(&use_pipette_linux_musl)),
guest_test_uefi: Some(ctx.use_typed_artifact(&use_guest_test_uefi)),
artifact_dir_openhcl_igvm_files: Some(ctx.use_artifact(&use_openhcl_igvm_files)),
openhcl_igvm_files: Some(ctx.use_typed_artifact(&use_openhcl_igvm_files)),
tmk_vmm: Some(ctx.use_typed_artifact(&use_tmk_vmm)),
tmk_vmm_linux_musl: Some(ctx.use_typed_artifact(&use_tmk_vmm_linux_musl)),
tmks: Some(ctx.use_typed_artifact(&use_tmks)),
Expand Down
234 changes: 156 additions & 78 deletions flowey/flowey_lib_hvlite/src/_jobs/build_and_publish_openhcl_igvm_from_recipe.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,11 +4,12 @@
//! Builds and publishes an a set of OpenHCL IGVM files.

use super::build_and_publish_openvmm_hcl_baseline;
use crate::artifact_openhcl_igvm_from_recipe_extras::OpenhclIgvmExtras;
use crate::build_openhcl_igvm_from_recipe::OpenhclIgvmRecipe;
use crate::build_openvmm_hcl::OpenvmmHclBuildProfile;
use crate::run_cargo_build::common::CommonTriple;
use crate::run_igvmfilegen::IgvmOutput;
use flowey::node::prelude::*;
use std::collections::BTreeMap;

#[derive(Serialize, Deserialize)]
pub struct VmfirmwareigvmDllParams {
Expand All @@ -26,21 +27,40 @@ pub struct OpenhclIgvmBuildParams {
flowey_request! {
pub struct Params {
pub igvm_files: Vec<OpenhclIgvmBuildParams>,
pub artifact_dir_openhcl_igvm: ReadVar<PathBuf>,
pub artifact_dir_openhcl_igvm_extras: ReadVar<PathBuf>,
pub openhcl_igvm: WriteVar<OpenhclIgvmSet>,
pub openhcl_igvm_extras: WriteVar<OpenhclIgvmExtrasSet>,
pub artifact_openhcl_verify_size_baseline: Option<ReadVar<PathBuf>>,
pub done: WriteVar<SideEffect>,
}
}

pub struct OpenhclIgvmSet(pub Vec<(OpenhclIgvmRecipe, IgvmOutput)>);

impl Artifact for OpenhclIgvmSet {}

#[derive(Serialize, Deserialize)]
#[serde(transparent)]
pub struct OpenhclIgvmExtrasSet(pub BTreeMap<OpenhclIgvmRecipe, OpenhclIgvmExtras>);

impl Artifact for OpenhclIgvmExtrasSet {}

#[derive(Serialize, Deserialize)]
pub struct OpenhclIgvmExtras {
#[serde(flatten)]
pub openvmm_hcl_bin: crate::build_openvmm_hcl::OpenvmmHclOutput,
#[serde(rename = "openhcl.bin.map")]
pub openhcl_map: Option<PathBuf>,
#[serde(flatten)]
pub openhcl_boot: crate::build_openhcl_boot::OpenhclBootOutput,
#[serde(flatten)]
pub sidecar: Option<crate::build_sidecar::SidecarOutput>,
}

new_simple_flow_node!(struct Node);

impl SimpleFlowNode for Node {
type Request = Params;

fn imports(ctx: &mut ImportCtx<'_>) {
ctx.import::<crate::artifact_openhcl_igvm_from_recipe_extras::publish::Node>();
ctx.import::<crate::artifact_openhcl_igvm_from_recipe::publish::Node>();
ctx.import::<crate::artifact_openvmm_hcl_sizecheck::publish::Node>();
ctx.import::<crate::build_openhcl_igvm_from_recipe::Node>();
ctx.import::<build_and_publish_openvmm_hcl_baseline::Node>();
Expand All @@ -49,88 +69,146 @@ impl SimpleFlowNode for Node {
fn process_request(request: Self::Request, ctx: &mut NodeCtx<'_>) -> anyhow::Result<()> {
let Params {
igvm_files,
artifact_dir_openhcl_igvm,
artifact_dir_openhcl_igvm_extras,
openhcl_igvm,
openhcl_igvm_extras,
artifact_openhcl_verify_size_baseline,
done,
} = request;

let mut built_igvm_files = Vec::new();
let mut built_extras = Vec::new();

for OpenhclIgvmBuildParams {
profile,
recipe,
custom_target,
} in igvm_files
{
let (read_built_openvmm_hcl, built_openvmm_hcl) = ctx.new_var();
let (read_built_openhcl_boot, built_openhcl_boot) = ctx.new_var();
let (read_built_openhcl_igvm, built_openhcl_igvm) = ctx.new_var();
let (read_built_sidecar, built_sidecar) = ctx.new_var();
ctx.req(crate::build_openhcl_igvm_from_recipe::Request {
custom_target,
profile,
recipe: recipe.clone(),
built_openvmm_hcl,
built_openhcl_boot,
built_openhcl_igvm,
built_sidecar,
});

built_igvm_files.push(read_built_openhcl_igvm.map(ctx, {
let recipe = recipe.clone();
move |x| (recipe, x)
}));

built_extras.push(ctx.emit_minor_rust_stepv(
"collect openhcl component paths",
|ctx| {
let recipe = recipe.clone();
let read_built_openvmm_hcl = read_built_openvmm_hcl.claim(ctx);
let read_built_openhcl_boot = read_built_openhcl_boot.claim(ctx);
let read_built_openhcl_igvm = read_built_openhcl_igvm.claim(ctx);
let read_built_sidecar = read_built_sidecar.claim(ctx);
|rt| OpenhclIgvmExtras {
let output = igvm_files
.into_iter()
.map(
|OpenhclIgvmBuildParams {
profile,
recipe,
custom_target,
}| {
(
recipe,
openvmm_hcl_bin: rt.read(read_built_openvmm_hcl),
openhcl_map: rt.read(read_built_openhcl_igvm).igvm_map,
openhcl_boot: rt.read(read_built_openhcl_boot),
sidecar: rt.read(read_built_sidecar),
}
ctx.reqv(|v| crate::build_openhcl_igvm_from_recipe::Request {
custom_target,
profile,
recipe: recipe.into(),
output: v,
}),
)
},
));
}

let mut did_publish = Vec::new();
)
.collect::<Vec<_>>();

did_publish.push(ctx.reqv(|done| {
crate::artifact_openhcl_igvm_from_recipe::publish::Request {
openhcl_igvm_files: built_igvm_files,
artifact_dir: artifact_dir_openhcl_igvm,
done,
let sizecheck_artifact = artifact_openhcl_verify_size_baseline.map(|artifact_dir| {
ctx.reqv(|v| build_and_publish_openvmm_hcl_baseline::Request {
artifact_dir,
done: v,
})
});

ctx.emit_minor_rust_step("collect openhcl results", |ctx| {
let output = output
.into_iter()
.map(|(r, v)| (r, v.claim(ctx)))
.collect::<Vec<_>>();
let openhcl_igvm = openhcl_igvm.claim(ctx);
let openhcl_igvm_extras = openhcl_igvm_extras.claim(ctx);
sizecheck_artifact.claim(ctx);
move |rt| {
let (base, extras) = output
.into_iter()
.map(|(r, v)| {
let v = rt.read(v);
let extras = OpenhclIgvmExtras {
openvmm_hcl_bin: v.openvmm_hcl,
openhcl_map: v.igvm.igvm_map.clone(),
openhcl_boot: v.openhcl_boot,
sidecar: v.sidecar,
};
let base = v.igvm;
((r, base), (r, extras))
})
.unzip();
rt.write(openhcl_igvm, &OpenhclIgvmSet(base));
rt.write(openhcl_igvm_extras, &OpenhclIgvmExtrasSet(extras));
}
}));
});

did_publish.push(ctx.reqv(|v| {
crate::artifact_openhcl_igvm_from_recipe_extras::publish::Request {
extras: built_extras,
artifact_dir: artifact_dir_openhcl_igvm_extras,
done: v,
Ok(())
}
}

/// Custom logic for serializing and deserializing the [`OpenhclIgvmSet`]` with a
/// directory structure we like.
mod artifact {
use super::OpenhclIgvmSet;
use crate::build_openhcl_igvm_from_recipe::OpenhclIgvmRecipe;
use crate::run_igvmfilegen::IgvmOutput;
use serde::Deserialize;
use serde::Serialize;
use serde_json::Value;
use std::collections::BTreeMap;
use std::path::PathBuf;

impl Serialize for OpenhclIgvmSet {
fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
where
S: serde::Serializer,
{
let mut v = BTreeMap::new();
for (recipe, value) in &self.0 {
let IgvmOutput {
igvm_bin,
igvm_map,
igvm_tdx_json,
igvm_snp_json,
igvm_vbs_json,
} = value;
let Value::String(recipe) = serde_json::to_value(recipe).unwrap() else {
unreachable!()
};
v.insert(format!("{}.bin", recipe), igvm_bin);
if let Some(igvm_map) = igvm_map {
v.insert(format!("{}.bin.map", recipe), igvm_map);
}
if let Some(igvm_tdx_json) = igvm_tdx_json {
v.insert(format!("{}-tdx.json", recipe), igvm_tdx_json);
}
if let Some(igvm_snp_json) = igvm_snp_json {
v.insert(format!("{}-snp.json", recipe), igvm_snp_json);
}
if let Some(igvm_vbs_json) = igvm_vbs_json {
v.insert(format!("{}-vbs.json", recipe), igvm_vbs_json);
}
}
}));

if let Some(sizecheck_artifact) = artifact_openhcl_verify_size_baseline {
did_publish.push(
ctx.reqv(|v| build_and_publish_openvmm_hcl_baseline::Request {
artifact_dir: sizecheck_artifact,
done: v,
}),
);
v.serialize(serializer)
}
}

ctx.emit_side_effect_step(did_publish, [done]);

Ok(())
impl<'de> Deserialize<'de> for OpenhclIgvmSet {
fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
where
D: serde::Deserializer<'de>,
{
let map = BTreeMap::<String, PathBuf>::deserialize(deserializer)?;
let mut r = Vec::new();
for (name, igvm_bin) in &map {
let Some(v) = name.strip_suffix(".bin") else {
continue;
};
let recipe: OpenhclIgvmRecipe =
serde_json::from_value(Value::String(v.to_string()))
.map_err(serde::de::Error::custom)?;
let igvm_map = map.get(&format!("{}.bin.map", v)).cloned();
let igvm_tdx_json = map.get(&format!("{}-tdx.json", v)).cloned();
let igvm_snp_json = map.get(&format!("{}-snp.json", v)).cloned();
let igvm_vbs_json = map.get(&format!("{}-vbs.json", v)).cloned();
let igvm = IgvmOutput {
igvm_bin: igvm_bin.clone(),
igvm_map,
igvm_tdx_json,
igvm_snp_json,
igvm_vbs_json,
};
r.push((recipe, igvm));
}
Ok(OpenhclIgvmSet(r))
}
}
}
3 changes: 2 additions & 1 deletion flowey/flowey_lib_hvlite/src/_jobs/build_and_publish_openvmm_hcl_baseline.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@

use crate::artifact_openvmm_hcl_sizecheck;
use crate::build_openhcl_igvm_from_recipe::OpenhclIgvmRecipe;
use crate::build_openhcl_igvm_from_recipe::RecipeOrCustom;
use crate::build_openvmm_hcl;
use crate::build_openvmm_hcl::OpenvmmHclBuildParams;
use crate::build_openvmm_hcl::OpenvmmHclBuildProfile;
Expand Down Expand Up @@ -35,7 +36,7 @@ impl SimpleFlowNode for Node {
build_params: OpenvmmHclBuildParams {
target: CommonTriple::X86_64_LINUX_MUSL,
profile: OpenvmmHclBuildProfile::OpenvmmHclShip,
features: (OpenhclIgvmRecipe::X64)
features: RecipeOrCustom::Recipe(OpenhclIgvmRecipe::X64)
.recipe_details(OpenvmmHclBuildProfile::OpenvmmHclShip)
.openvmm_hcl_features,
no_split_dbg_info: false,
Expand Down
Loading