flowey_hvlite/pipelines/

checkin_gates.rs

// Copyright (c) Microsoft Corporation.
// Licensed under the MIT License.

//! See [`CheckinGatesCli`]

use crate::pipelines_shared::ado_pools;
use crate::pipelines_shared::gh_pools;
use flowey::node::prelude::AdoResourcesRepositoryId;
use flowey::node::prelude::FlowPlatformLinuxDistro;
use flowey::node::prelude::GhPermission;
use flowey::node::prelude::GhPermissionValue;
use flowey::node::prelude::ReadVar;
use flowey::pipeline::prelude::*;
use flowey_lib_common::git_checkout::RepoSource;
use flowey_lib_hvlite::_jobs::build_and_publish_openhcl_igvm_from_recipe::OpenhclIgvmBuildParams;
use flowey_lib_hvlite::build_openhcl_igvm_from_recipe::OpenhclIgvmRecipe;
use flowey_lib_hvlite::build_openvmm_hcl::OpenvmmHclBuildProfile;
use flowey_lib_hvlite::build_openvmm_hcl::OpenvmmHclFeature;
use flowey_lib_hvlite::common::CommonArch;
use flowey_lib_hvlite::common::CommonPlatform;
use flowey_lib_hvlite::common::CommonProfile;
use flowey_lib_hvlite::common::CommonTriple;
use std::collections::BTreeMap;
use std::collections::BTreeSet;
use std::path::PathBuf;
use target_lexicon::Triple;
use vmm_test_images::KnownTestArtifacts;

// This is a cap for surplus 2 MiB hugetlb pages, not a reservation. Keep it
// generous enough for VMM tests without tying CI provisioning to one test's RAM.
const HUGETLB_2MB_OVERCOMMIT_PAGES: u64 = 4096;

#[derive(Copy, Clone, clap::ValueEnum)]
enum PipelineConfig {
    /// Run on all PRs targeting the OpenVMM GitHub repo.
    Pr,
    /// Run on all commits that land in a branch.
    ///
    /// The key difference between the CI and PR pipelines is whether things are
    /// being built in `release` mode.
    Ci,
    /// Release variant of the `Pr` pipeline.
    PrRelease,
}

/// A unified pipeline defining all checkin gates required to land a commit in
/// the OpenVMM repo.
#[derive(clap::Args)]
pub struct CheckinGatesCli {
    /// Which pipeline configuration to use.
    #[clap(long)]
    config: PipelineConfig,

    #[clap(flatten)]
    local_run_args: Option<crate::pipelines_shared::cfg_common_params::LocalRunArgs>,

    /// Set custom path to search for / download VMM tests disk-images
    #[clap(long)]
    vmm_tests_disk_cache_dir: Option<PathBuf>,
}

impl IntoPipeline for CheckinGatesCli {
    fn into_pipeline(self, backend_hint: PipelineBackendHint) -> anyhow::Result<Pipeline> {
        let Self {
            config,
            local_run_args,
            vmm_tests_disk_cache_dir,
        } = self;

        let release = match config {
            PipelineConfig::Ci | PipelineConfig::PrRelease => true,
            PipelineConfig::Pr => false,
        };

        let mut pipeline = Pipeline::new();

        let mut vmgstools = BTreeMap::new();

        // configure pr/ci branch triggers and add gh pipeline name
        {
            let branches = vec!["main".into(), "release/*".into()];

            // Paths that don't affect the Rust build or tests. Changes
            // to only these paths will not trigger the CI pipeline on push.
            //
            // NOTE: The PR pipeline intentionally does NOT use paths-ignore,
            // because the "openvmm checkin gates" job is a required status
            // check. If the workflow is skipped due to path filters, the
            // gate is never reported and the PR is blocked. The CI pipeline
            // can still use paths-ignore since it has no required checks.
            let ci_paths_ignore = vec!["Guide/**".into(), "petri/logview/**".into()];

            match config {
                PipelineConfig::Ci => {
                    pipeline
                        .gh_set_ci_triggers(GhCiTriggers {
                            branches,
                            paths_ignore: ci_paths_ignore.clone(),
                            ..Default::default()
                        })
                        .gh_set_name("OpenVMM CI");
                }
                PipelineConfig::Pr => {
                    pipeline
                        .gh_set_pr_triggers(GhPrTriggers {
                            branches,
                            ..GhPrTriggers::new_draftable()
                        })
                        .gh_set_name("OpenVMM PR")
                        .ado_set_pr_triggers(AdoPrTriggers {
                            branches: vec!["main".into(), "release/*".into(), "embargo/*".into()],
                            exclude_paths: ci_paths_ignore.clone(),
                            ..Default::default()
                        });
                }
                PipelineConfig::PrRelease => {
                    // This workflow is triggered when a specific label is present on a PR.
                    let mut triggers = GhPrTriggers::new_draftable();
                    triggers.branches = branches;
                    triggers.types.push("labeled".into());
                    pipeline
                        .gh_set_pr_triggers(triggers)
                        .gh_set_name("[Optional] OpenVMM Release PR");
                }
            }
        }

        let openvmm_repo_source = match backend_hint {
            PipelineBackendHint::Local => {
                RepoSource::ExistingClone(ReadVar::from_static(crate::repo_root()))
            }
            PipelineBackendHint::Github => RepoSource::GithubSelf,
            PipelineBackendHint::Ado => {
                RepoSource::AdoResource(AdoResourcesRepositoryId::new_self())
            }
        };

        if let RepoSource::GithubSelf = &openvmm_repo_source {
            pipeline.gh_set_flowey_bootstrap_template(
                crate::pipelines_shared::gh_flowey_bootstrap_template::get_template(),
            );
        }

        if let RepoSource::AdoResource(source) = &openvmm_repo_source {
            pipeline.ado_set_flowey_bootstrap_template(
                crate::pipelines_shared::ado_flowey_bootstrap_template::get_template_ado(source),
            );
        }

        let cfg_common_params = crate::pipelines_shared::cfg_common_params::get_cfg_common_params(
            &mut pipeline,
            backend_hint,
            local_run_args,
        )?;

        pipeline.inject_all_jobs_with(move |job| {
            let mut job = job
                .dep_on(&cfg_common_params)
                .dep_on(|_| flowey_lib_hvlite::_jobs::cfg_versions::Request::Init)
                .dep_on(
                    |_| flowey_lib_hvlite::_jobs::cfg_hvlite_reposource::Params {
                        hvlite_repo_source: openvmm_repo_source.clone(),
                    },
                )
                .gh_grant_permissions::<flowey_lib_common::git_checkout::Node>([(
                    GhPermission::Contents,
                    GhPermissionValue::Read,
                )])
                .gh_grant_permissions::<flowey_lib_common::gh_task_azure_login::Node>([(
                    GhPermission::IdToken,
                    GhPermissionValue::Write,
                )]);

            // For the release pipeline, only run if the "release-ci-required" label is present and PR is not draft
            if matches!(config, PipelineConfig::PrRelease) {
                job = job.gh_dangerous_override_if(
                    "contains(github.event.pull_request.labels.*.name, 'release-ci-required') && github.event.pull_request.draft == false",
                );
            }

            job
        });

        let openhcl_musl_target = |arch: CommonArch| -> Triple {
            CommonTriple::Common {
                arch,
                platform: CommonPlatform::LinuxMusl,
            }
            .as_triple()
        };

        // initialize the various VMM tests nextest archive artifacts
        let (pub_vmm_tests_archive_linux_x86, use_vmm_tests_archive_linux_x86) =
            pipeline.new_typed_artifact("x64-linux-vmm-tests-archive");
        let (pub_vmm_tests_archive_linux_musl_x86, use_vmm_tests_archive_linux_musl_x86) =
            pipeline.new_typed_artifact("x64-linux-musl-vmm-tests-archive");
        let (pub_vmm_tests_archive_windows_x86, use_vmm_tests_archive_windows_x86) =
            pipeline.new_typed_artifact("x64-windows-vmm-tests-archive");
        let (pub_vmm_tests_archive_windows_aarch64, use_vmm_tests_archive_windows_aarch64) =
            pipeline.new_typed_artifact("aarch64-windows-vmm-tests-archive");

        // wrap each publish handle in an option, so downstream code can
        // `.take()` the handle when emitting the corresponding job
        let mut pub_vmm_tests_archive_linux_x86 = Some(pub_vmm_tests_archive_linux_x86);
        let mut pub_vmm_tests_archive_linux_musl_x86 = Some(pub_vmm_tests_archive_linux_musl_x86);
        let mut pub_vmm_tests_archive_windows_x86 = Some(pub_vmm_tests_archive_windows_x86);
        let mut pub_vmm_tests_archive_windows_aarch64 = Some(pub_vmm_tests_archive_windows_aarch64);

        // initialize the various "VmmTestsArtifactsBuilder" containers, which
        // are used to "skim off" various artifacts that the VMM test jobs
        // require.
        let mut vmm_tests_artifacts_linux_x86 =
            vmm_tests_artifact_builders::VmmTestsArtifactsBuilderLinuxX86::default();
        let mut vmm_tests_artifacts_linux_musl_x86 =
            vmm_tests_artifact_builders::VmmTestsArtifactsBuilderLinuxX86::default();
        let mut vmm_tests_artifacts_windows_x86 =
            vmm_tests_artifact_builders::VmmTestsArtifactsBuilderWindowsX86::default();
        let mut vmm_tests_artifacts_windows_aarch64 =
            vmm_tests_artifact_builders::VmmTestsArtifactsBuilderWindowsAarch64::default();

        // We need to maintain a list of all jobs, so we can hang the "all good"
        // job off of them. This is requires because github status checks only allow
        // specifying jobs, and not workflows.
        // There's more info in the following discussion:
        // <https://github.com/orgs/community/discussions/12395>
        let mut all_jobs = Vec::new();

        // Quick check gate
        //
        // Combined fmt + clippy on one self-hosted linux machine.
        // Catches the most common failures quickly before fanning out expensive jobs.
        let quick_check_job = if matches!(config, PipelineConfig::Pr | PipelineConfig::PrRelease) {
            let job = pipeline
                .new_job(
                    FlowPlatform::Linux(FlowPlatformLinuxDistro::Ubuntu),
                    FlowArch::X86_64,
                    "quick check [fmt, clippy x64-linux]",
                )
                .gh_set_pool(gh_pools::default_linux())
                .ado_set_pool(ado_pools::default_linux())
                // 1. xtask fmt (linux)
                .side_effect(|done| flowey_lib_hvlite::_jobs::check_xtask_fmt::Request {
                    target: CommonTriple::X86_64_LINUX_GNU,
                    done,
                })
                // 2. clippy for x64-linux-gnu
                .side_effect(|done| flowey_lib_hvlite::_jobs::check_clippy::Request {
                    target: target_lexicon::triple!("x86_64-unknown-linux-gnu"),
                    profile: CommonProfile::from_release(release),
                    done,
                    also_check_misc_nostd_crates: false,
                })
                .finish();

            Some(job)
        } else {
            // skip in CI
            None
        };

        // emit xtask fmt job
        {
            let windows_fmt_job = pipeline
                .new_job(
                    FlowPlatform::Windows,
                    FlowArch::X86_64,
                    "xtask fmt (windows)",
                )
                .gh_set_pool(gh_pools::windows_x64_gh())
                .ado_set_pool(ado_pools::default_windows())
                .side_effect(|done| flowey_lib_hvlite::_jobs::check_xtask_fmt::Request {
                    target: CommonTriple::X86_64_WINDOWS_MSVC,
                    done,
                })
                .finish();

            let linux_fmt_job = if let Some(ref qc) = quick_check_job {
                // PR/PrRelease: linux fmt is handled by the quick-check job
                qc.clone()
            } else {
                // CI mode: keep standalone linux fmt job
                let job = pipeline
                    .new_job(
                        FlowPlatform::Linux(FlowPlatformLinuxDistro::Ubuntu),
                        FlowArch::X86_64,
                        "xtask fmt (linux)",
                    )
                    .gh_set_pool(gh_pools::linux_x64_gh())
                    .ado_set_pool(ado_pools::default_linux())
                    .side_effect(|done| flowey_lib_hvlite::_jobs::check_xtask_fmt::Request {
                        target: CommonTriple::X86_64_LINUX_GNU,
                        done,
                    })
                    .finish();
                all_jobs.push(job.clone());
                job
            };

            // cut down on extra noise by having the linux check run first, and
            // then if it passes, run the windows checks just in case there is a
            // difference between the two.
            pipeline.non_artifact_dep(&windows_fmt_job, &linux_fmt_job);

            all_jobs.push(windows_fmt_job);
        }

        // emit shared dependencies jobs
        //
        // In order to ensure we start running VMM tests as soon as possible, we emit
        // a job for windows and linux building dependencies used by VMM tests on all platforms.
        // These jobs build dependencies for all architectures, as these dependencies are typically
        // small and fast to build.
        //
        // We have to create the per-arch artifacts up front so that we don't try
        // to mutably borrow `pipeline` while a job builder also holds a mutable borrow.
        let mut shared_win_pipette_artifacts = Vec::new();
        for arch in [CommonArch::Aarch64, CommonArch::X86_64] {
            let arch_tag = match arch {
                CommonArch::X86_64 => "x64",
                CommonArch::Aarch64 => "aarch64",
            };
            let (pub_pipette_windows, use_pipette_windows) =
                pipeline.new_typed_artifact(format!("{arch_tag}-windows-pipette"));
            // filter off artifacts required by the VMM tests job
            match arch {
                CommonArch::X86_64 => {
                    vmm_tests_artifacts_linux_x86.use_pipette_windows =
                        Some(use_pipette_windows.clone());
                    vmm_tests_artifacts_linux_musl_x86.use_pipette_windows =
                        Some(use_pipette_windows.clone());
                    vmm_tests_artifacts_windows_x86.use_pipette_windows =
                        Some(use_pipette_windows.clone());
                }
                CommonArch::Aarch64 => {
                    vmm_tests_artifacts_windows_aarch64.use_pipette_windows =
                        Some(use_pipette_windows.clone());
                }
            }
            shared_win_pipette_artifacts.push((arch, pub_pipette_windows));
        }
        let mut shared_win_job = pipeline
            .new_job(
                FlowPlatform::Windows,
                FlowArch::X86_64,
                "build artifacts (shared VMM tests) [windows]",
            )
            .gh_set_pool(gh_pools::default_windows())
            .ado_set_pool(ado_pools::default_windows());
        for (arch, pub_pipette_windows) in shared_win_pipette_artifacts {
            shared_win_job = shared_win_job.publish(pub_pipette_windows, |pipette| {
                flowey_lib_hvlite::build_pipette::Request {
                    target: CommonTriple::Common {
                        arch,
                        platform: CommonPlatform::WindowsMsvc,
                    },
                    profile: CommonProfile::from_release(release),
                    pipette,
                }
            });
        }
        all_jobs.push(shared_win_job.finish());

        // Now do linux
        //
        // Create the per-arch artifacts up front so that we don't try to
        // mutably borrow `pipeline` while the job builder also holds a
        // mutable borrow.
        let mut shared_linux_artifacts = Vec::new();
        for arch in [CommonArch::Aarch64, CommonArch::X86_64] {
            let arch_tag = match arch {
                CommonArch::X86_64 => "x64",
                CommonArch::Aarch64 => "aarch64",
            };

            let (pub_tpm_guest_tests, use_tpm_guest_tests) =
                pipeline.new_typed_artifact(format!("{arch_tag}-linux-tpm_guest_tests"));
            let (pub_guest_test_uefi, use_guest_test_uefi) =
                pipeline.new_typed_artifact(format!("{arch_tag}-guest_test_uefi"));
            let (pub_pipette_linux_musl, use_pipette_linux_musl) =
                pipeline.new_typed_artifact(format!("{arch_tag}-linux-musl-pipette"));
            let (pub_tmk_vmm, use_tmk_vmm) =
                pipeline.new_typed_artifact(format!("{arch_tag}-linux-musl-tmk_vmm"));
            let (pub_tmks, use_tmks) = pipeline.new_typed_artifact(format!("{arch_tag}-tmks"));

            match arch {
                CommonArch::X86_64 => {
                    vmm_tests_artifacts_linux_x86.use_guest_test_uefi =
                        Some(use_guest_test_uefi.clone());
                    vmm_tests_artifacts_windows_x86.use_guest_test_uefi =
                        Some(use_guest_test_uefi.clone());
                    vmm_tests_artifacts_windows_x86.use_tmks = Some(use_tmks.clone());
                    vmm_tests_artifacts_linux_x86.use_tmks = Some(use_tmks.clone());
                    vmm_tests_artifacts_windows_x86.use_tpm_guest_tests_linux =
                        Some(use_tpm_guest_tests.clone());
                    vmm_tests_artifacts_linux_musl_x86.use_guest_test_uefi =
                        Some(use_guest_test_uefi.clone());
                    vmm_tests_artifacts_linux_musl_x86.use_tmks = Some(use_tmks.clone());
                    vmm_tests_artifacts_windows_x86.use_pipette_linux_musl =
                        Some(use_pipette_linux_musl.clone());
                    vmm_tests_artifacts_linux_x86.use_pipette_linux_musl =
                        Some(use_pipette_linux_musl.clone());
                    vmm_tests_artifacts_linux_x86.use_tmk_vmm = Some(use_tmk_vmm.clone());
                    vmm_tests_artifacts_windows_x86.use_tmk_vmm_linux_musl =
                        Some(use_tmk_vmm.clone());
                    vmm_tests_artifacts_linux_musl_x86.use_pipette_linux_musl =
                        Some(use_pipette_linux_musl.clone());
                    vmm_tests_artifacts_linux_musl_x86.use_tmk_vmm = Some(use_tmk_vmm.clone());
                }
                CommonArch::Aarch64 => {
                    vmm_tests_artifacts_windows_aarch64.use_guest_test_uefi =
                        Some(use_guest_test_uefi.clone());
                    vmm_tests_artifacts_windows_aarch64.use_tmks = Some(use_tmks.clone());
                    vmm_tests_artifacts_windows_aarch64.use_pipette_linux_musl =
                        Some(use_pipette_linux_musl.clone());
                    vmm_tests_artifacts_windows_aarch64.use_tmk_vmm_linux_musl =
                        Some(use_tmk_vmm.clone());
                }
            }

            shared_linux_artifacts.push((
                arch,
                pub_tpm_guest_tests,
                pub_guest_test_uefi,
                pub_pipette_linux_musl,
                pub_tmk_vmm,
                pub_tmks,
            ));
        }

        let mut shared_linux_job = pipeline
            .new_job(
                FlowPlatform::Linux(FlowPlatformLinuxDistro::Ubuntu),
                FlowArch::X86_64,
                "build artifacts (shared VMM tests) [linux]",
            )
            .gh_set_pool(gh_pools::default_linux())
            .ado_set_pool(ado_pools::default_linux());
        for (
            arch,
            pub_tpm_guest_tests,
            pub_guest_test_uefi,
            pub_pipette_linux_musl,
            pub_tmk_vmm,
            pub_tmks,
        ) in shared_linux_artifacts
        {
            shared_linux_job = shared_linux_job
                .publish(pub_guest_test_uefi, |guest_test_uefi| {
                    flowey_lib_hvlite::build_guest_test_uefi::Request {
                        arch,
                        profile: CommonProfile::from_release(release),
                        guest_test_uefi,
                    }
                })
                .publish(pub_tmks, |tmks| flowey_lib_hvlite::build_tmks::Request {
                    arch,
                    profile: CommonProfile::from_release(release),
                    tmks,
                })
                .publish(pub_tpm_guest_tests, |tpm_guest_tests| {
                    flowey_lib_hvlite::build_tpm_guest_tests::Request {
                        target: CommonTriple::Common {
                            arch,
                            platform: CommonPlatform::LinuxGnu,
                        },
                        profile: CommonProfile::from_release(release),
                        tpm_guest_tests,
                    }
                })
                .publish(pub_pipette_linux_musl, |pipette| {
                    flowey_lib_hvlite::build_pipette::Request {
                        target: CommonTriple::Common {
                            arch,
                            platform: CommonPlatform::LinuxMusl,
                        },
                        profile: CommonProfile::from_release(release),
                        pipette,
                    }
                })
                .publish(pub_tmk_vmm, |tmk_vmm| {
                    flowey_lib_hvlite::build_tmk_vmm::Request {
                        target: CommonTriple::Common {
                            arch,
                            platform: CommonPlatform::LinuxMusl,
                        },
                        profile: CommonProfile::from_release(release),
                        tmk_vmm,
                    }
                });
        }

        all_jobs.push(shared_linux_job.finish());

        // emit windows build machine jobs
        //
        // In order to ensure we start running VMM tests as soon as possible, we emit
        // two separate windows job per arch - one for artifacts in the VMM tests
        // hotpath, and another for any auxiliary artifacts that aren't
        // required by VMM tests.
        for arch in [CommonArch::Aarch64, CommonArch::X86_64] {
            let arch_tag = match arch {
                CommonArch::X86_64 => "x64",
                CommonArch::Aarch64 => "aarch64",
            };

            // artifacts which _are_ in the VMM tests "hot path"
            let (pub_openvmm, use_openvmm) =
                pipeline.new_typed_artifact(format!("{arch_tag}-windows-openvmm"));

            let (pub_tmk_vmm, use_tmk_vmm) =
                pipeline.new_typed_artifact(format!("{arch_tag}-windows-tmk_vmm"));

            let (pub_prep_steps, use_prep_steps) =
                pipeline.new_typed_artifact(format!("{arch_tag}-windows-prep_steps"));

            let (pub_vmgstool, use_vmgstool) =
                pipeline.new_typed_artifact(format!("{arch_tag}-windows-vmgstool"));

            let (pub_tpm_guest_tests, use_tpm_guest_tests_windows) =
                pipeline.new_typed_artifact(format!("{arch_tag}-windows-tpm_guest_tests"));

            let (pub_test_igvm_agent_rpc_server, use_test_igvm_agent_rpc_server) = pipeline
                .new_typed_artifact(format!("{arch_tag}-windows-test_igvm_agent_rpc_server"));

            // filter off interesting artifacts required by the VMM tests job
            match arch {
                CommonArch::X86_64 => {
                    vmm_tests_artifacts_windows_x86.use_openvmm = Some(use_openvmm.clone());
                    vmm_tests_artifacts_windows_x86.use_tmk_vmm = Some(use_tmk_vmm.clone());
                    vmm_tests_artifacts_windows_x86.use_prep_steps = Some(use_prep_steps.clone());
                    vmm_tests_artifacts_windows_x86.use_vmgstool = Some(use_vmgstool.clone());
                    vmm_tests_artifacts_windows_x86.use_tpm_guest_tests_windows =
                        Some(use_tpm_guest_tests_windows.clone());
                    vmm_tests_artifacts_windows_x86.use_test_igvm_agent_rpc_server =
                        Some(use_test_igvm_agent_rpc_server.clone());
                }
                CommonArch::Aarch64 => {
                    vmm_tests_artifacts_windows_aarch64.use_openvmm = Some(use_openvmm.clone());
                    vmm_tests_artifacts_windows_aarch64.use_tmk_vmm = Some(use_tmk_vmm.clone());
                    vmm_tests_artifacts_windows_aarch64.use_vmgstool = Some(use_vmgstool.clone());
                }
            }
            // emit a job for artifacts which _are not_ in the VMM tests "hot
            // path"
            // artifacts which _are not_ in the VMM tests "hot path"
            let (pub_igvmfilegen, _use_igvmfilegen) =
                pipeline.new_typed_artifact(format!("{arch_tag}-windows-igvmfilegen"));
            let (pub_vmgs_lib, _use_vmgs_lib) =
                pipeline.new_typed_artifact(format!("{arch_tag}-windows-vmgs_lib"));
            let (pub_hypestv, _use_hypestv) =
                pipeline.new_typed_artifact(format!("{arch_tag}-windows-hypestv"));
            let (pub_ohcldiag_dev, _use_ohcldiag_dev) =
                pipeline.new_typed_artifact(format!("{arch_tag}-windows-ohcldiag-dev"));

            let job = pipeline
                .new_job(
                    FlowPlatform::Windows,
                    FlowArch::X86_64,
                    format!("build artifacts (not for VMM tests) [{arch_tag}-windows]"),
                )
                .gh_set_pool(gh_pools::default_windows())
                .ado_set_pool(ado_pools::default_windows())
                .publish(pub_hypestv, |hypestv| {
                    flowey_lib_hvlite::build_hypestv::Request {
                        target: CommonTriple::Common {
                            arch,
                            platform: CommonPlatform::WindowsMsvc,
                        },
                        profile: CommonProfile::from_release(release),
                        hypestv,
                    }
                })
                .publish(pub_vmgs_lib, |vmgs_lib| {
                    flowey_lib_hvlite::build_and_test_vmgs_lib::Request {
                        target: CommonTriple::Common {
                            arch,
                            platform: CommonPlatform::WindowsMsvc,
                        },
                        profile: CommonProfile::from_release(release),
                        vmgs_lib,
                    }
                })
                .publish(pub_igvmfilegen, |igvmfilegen| {
                    flowey_lib_hvlite::build_igvmfilegen::Request {
                        build_params:
                            flowey_lib_hvlite::build_igvmfilegen::IgvmfilegenBuildParams {
                                target: CommonTriple::Common {
                                    arch,
                                    platform: CommonPlatform::WindowsMsvc,
                                },
                                profile: CommonProfile::from_release(release).into(),
                            },
                        igvmfilegen,
                    }
                })
                .publish(pub_ohcldiag_dev, |ohcldiag_dev| {
                    flowey_lib_hvlite::build_ohcldiag_dev::Request {
                        target: CommonTriple::Common {
                            arch,
                            platform: CommonPlatform::WindowsMsvc,
                        },
                        profile: CommonProfile::from_release(release),
                        ohcldiag_dev,
                    }
                });

            all_jobs.push(job.finish());

            let vmgstool_target = CommonTriple::Common {
                arch,
                platform: CommonPlatform::WindowsMsvc,
            };
            if vmgstools
                .insert(vmgstool_target.to_string(), use_vmgstool.clone())
                .is_some()
            {
                anyhow::bail!("multiple vmgstools for the same target");
            }

            // emit a job for artifacts which _are_ in the VMM tests "hot path"
            let mut job = pipeline
                .new_job(
                    FlowPlatform::Windows,
                    FlowArch::X86_64,
                    format!("build artifacts (for VMM tests) [{arch_tag}-windows]"),
                )
                .gh_set_pool(gh_pools::default_windows())
                .ado_set_pool(ado_pools::default_windows())
                .publish(pub_openvmm, |openvmm| {
                    flowey_lib_hvlite::build_openvmm::Request {
                        params: flowey_lib_hvlite::build_openvmm::OpenvmmBuildParams {
                            target: CommonTriple::Common {
                                arch,
                                platform: CommonPlatform::WindowsMsvc,
                            },
                            profile: CommonProfile::from_release(release),
                            // FIXME: this relies on openvmm default features
                            features: [].into(),
                        },
                        version: None,
                        openvmm,
                    }
                })
                .publish(pub_tmk_vmm, |tmk_vmm| {
                    flowey_lib_hvlite::build_tmk_vmm::Request {
                        target: CommonTriple::Common {
                            arch,
                            platform: CommonPlatform::WindowsMsvc,
                        },
                        profile: CommonProfile::from_release(release),
                        tmk_vmm,
                    }
                })
                .publish(pub_prep_steps, |prep_steps| {
                    flowey_lib_hvlite::build_prep_steps::Request {
                        target: CommonTriple::Common {
                            arch,
                            platform: CommonPlatform::WindowsMsvc,
                        },
                        profile: CommonProfile::from_release(release),
                        prep_steps,
                    }
                })
                .publish(pub_vmgstool, |vmgstool| {
                    flowey_lib_hvlite::build_vmgstool::Request {
                        target: vmgstool_target,
                        profile: CommonProfile::from_release(release),
                        with_crypto: true,
                        with_test_helpers: true,
                        vmgstool,
                    }
                })
                .publish(pub_tpm_guest_tests, |tpm_guest_tests| {
                    flowey_lib_hvlite::build_tpm_guest_tests::Request {
                        target: CommonTriple::Common {
                            arch,
                            platform: CommonPlatform::WindowsMsvc,
                        },
                        profile: CommonProfile::from_release(release),
                        tpm_guest_tests,
                    }
                })
                .publish(
                    pub_test_igvm_agent_rpc_server,
                    |test_igvm_agent_rpc_server| {
                        flowey_lib_hvlite::build_test_igvm_agent_rpc_server::Request {
                            target: CommonTriple::Common {
                                arch,
                                platform: CommonPlatform::WindowsMsvc,
                            },
                            profile: CommonProfile::from_release(release),
                            test_igvm_agent_rpc_server,
                        }
                    },
                );

            // Hang building the windows VMM tests off this big windows job.
            match arch {
                CommonArch::X86_64 => {
                    let pub_vmm_tests_archive_windows_x86 =
                        pub_vmm_tests_archive_windows_x86.take().unwrap();
                    job = job.publish(pub_vmm_tests_archive_windows_x86, |archive|
                        flowey_lib_hvlite::build_nextest_vmm_tests::Request {
                        target: CommonTriple::X86_64_WINDOWS_MSVC.as_triple(),
                        profile: CommonProfile::from_release(release),
                        build_mode: flowey_lib_hvlite::build_nextest_vmm_tests::BuildNextestVmmTestsMode::Archive(
                            archive,
                        ),
                    });
                }
                CommonArch::Aarch64 => {
                    let pub_vmm_tests_archive_windows_aarch64 =
                        pub_vmm_tests_archive_windows_aarch64.take().unwrap();
                    job = job.publish(pub_vmm_tests_archive_windows_aarch64, |archive| flowey_lib_hvlite::build_nextest_vmm_tests::Request {
                        target: CommonTriple::AARCH64_WINDOWS_MSVC.as_triple(),
                        profile: CommonProfile::from_release(release),
                        build_mode: flowey_lib_hvlite::build_nextest_vmm_tests::BuildNextestVmmTestsMode::Archive(
                            archive,
                        ),
                    });
                }
            }

            all_jobs.push(job.finish());
        }

        // emit linux build machine jobs (without openhcl)
        for arch in [CommonArch::Aarch64, CommonArch::X86_64] {
            let arch_tag = match arch {
                CommonArch::X86_64 => "x64",
                CommonArch::Aarch64 => "aarch64",
            };

            let (pub_openvmm, use_openvmm) =
                pipeline.new_typed_artifact(format!("{arch_tag}-linux-openvmm"));
            let (pub_openvmm_vhost, use_openvmm_vhost) =
                pipeline.new_typed_artifact(format!("{arch_tag}-linux-openvmm_vhost"));
            let (pub_igvmfilegen, _) =
                pipeline.new_typed_artifact(format!("{arch_tag}-linux-igvmfilegen"));
            let (pub_vmgs_lib, _) =
                pipeline.new_typed_artifact(format!("{arch_tag}-linux-vmgs_lib"));
            let (pub_vmgstool, use_vmgstool) =
                pipeline.new_typed_artifact(format!("{arch_tag}-linux-vmgstool"));
            let (pub_ohcldiag_dev, _) =
                pipeline.new_typed_artifact(format!("{arch_tag}-linux-ohcldiag-dev"));
            // Also build openvmm and openvmm_vhost for musl on this job,
            // alongside pipette and tmk_vmm. This enables running VMM tests
            // on Azure Linux (MSHV) runners which have an older glibc.
            let (pub_openvmm_musl, use_openvmm_musl) =
                pipeline.new_typed_artifact(format!("{arch_tag}-linux-musl-openvmm"));
            let (pub_openvmm_vhost_musl, use_openvmm_vhost_musl) =
                pipeline.new_typed_artifact(format!("{arch_tag}-linux-musl-openvmm_vhost"));

            // skim off interesting artifacts required by the VMM tests job
            match arch {
                CommonArch::X86_64 => {
                    vmm_tests_artifacts_linux_x86.use_openvmm = Some(use_openvmm.clone());
                    vmm_tests_artifacts_linux_x86.use_openvmm_vhost =
                        Some(use_openvmm_vhost.clone());
                    vmm_tests_artifacts_linux_musl_x86.use_openvmm = Some(use_openvmm_musl.clone());
                    vmm_tests_artifacts_linux_musl_x86.use_openvmm_vhost =
                        Some(use_openvmm_vhost_musl.clone());
                }
                CommonArch::Aarch64 => {}
            }

            let vmgstool_target = CommonTriple::Common {
                arch,
                platform: CommonPlatform::LinuxGnu,
            };
            if vmgstools
                .insert(vmgstool_target.to_string(), use_vmgstool.clone())
                .is_some()
            {
                anyhow::bail!("multiple vmgstools for the same target");
            }

            // Emit a job for building dependencies used by just linux vmm tests
            let mut job = pipeline
                .new_job(
                    FlowPlatform::Linux(FlowPlatformLinuxDistro::Ubuntu),
                    FlowArch::X86_64,
                    format!("build artifacts (for VMM tests) [{arch_tag}-linux]"),
                )
                .gh_set_pool(gh_pools::default_linux())
                .ado_set_pool(ado_pools::default_linux())
                .publish(pub_openvmm, |openvmm| {
                    flowey_lib_hvlite::build_openvmm::Request {
                        params: flowey_lib_hvlite::build_openvmm::OpenvmmBuildParams {
                            target: CommonTriple::Common {
                                arch,
                                platform: CommonPlatform::LinuxGnu,
                            },
                            profile: CommonProfile::from_release(release),
                            // FIXME: this relies on openvmm default features
                            features: [flowey_lib_hvlite::build_openvmm::OpenvmmFeature::Tpm]
                                .into(),
                        },
                        version: None,
                        openvmm,
                    }
                })
                .publish(pub_openvmm_vhost, |openvmm_vhost| {
                    flowey_lib_hvlite::build_openvmm_vhost::Request {
                        params: flowey_lib_hvlite::build_openvmm_vhost::OpenvmmVhostBuildParams {
                            target: CommonTriple::Common {
                                arch,
                                platform: CommonPlatform::LinuxGnu,
                            },
                            profile: CommonProfile::from_release(release),
                        },
                        openvmm_vhost,
                    }
                })
                .publish(pub_vmgstool, |vmgstool| {
                    flowey_lib_hvlite::build_vmgstool::Request {
                        target: vmgstool_target,
                        profile: CommonProfile::from_release(release),
                        with_crypto: true,
                        with_test_helpers: true,
                        vmgstool,
                    }
                })
                .publish(pub_vmgs_lib, |vmgs_lib| {
                    flowey_lib_hvlite::build_and_test_vmgs_lib::Request {
                        target: CommonTriple::Common {
                            arch,
                            platform: CommonPlatform::LinuxGnu,
                        },
                        profile: CommonProfile::from_release(release),
                        vmgs_lib,
                    }
                })
                .publish(pub_igvmfilegen, |igvmfilegen| {
                    flowey_lib_hvlite::build_igvmfilegen::Request {
                        build_params:
                            flowey_lib_hvlite::build_igvmfilegen::IgvmfilegenBuildParams {
                                target: CommonTriple::Common {
                                    arch,
                                    platform: CommonPlatform::LinuxGnu,
                                },
                                profile: CommonProfile::from_release(release).into(),
                            },
                        igvmfilegen,
                    }
                })
                .publish(pub_ohcldiag_dev, |ohcldiag_dev| {
                    flowey_lib_hvlite::build_ohcldiag_dev::Request {
                        target: CommonTriple::Common {
                            arch,
                            platform: CommonPlatform::LinuxGnu,
                        },
                        profile: CommonProfile::from_release(release),
                        ohcldiag_dev,
                    }
                })
                .publish(pub_openvmm_musl, |openvmm| {
                    flowey_lib_hvlite::build_openvmm::Request {
                        params: flowey_lib_hvlite::build_openvmm::OpenvmmBuildParams {
                            target: CommonTriple::Common {
                                arch,
                                platform: CommonPlatform::LinuxMusl,
                            },
                            profile: CommonProfile::from_release(release),
                            features: [flowey_lib_hvlite::build_openvmm::OpenvmmFeature::Tpm]
                                .into(),
                        },
                        version: None,
                        openvmm,
                    }
                })
                .publish(pub_openvmm_vhost_musl, |openvmm_vhost| {
                    flowey_lib_hvlite::build_openvmm_vhost::Request {
                        params: flowey_lib_hvlite::build_openvmm_vhost::OpenvmmVhostBuildParams {
                            target: CommonTriple::Common {
                                arch,
                                platform: CommonPlatform::LinuxMusl,
                            },
                            profile: CommonProfile::from_release(release),
                        },
                        openvmm_vhost,
                    }
                });

            // Hang building the linux VMM tests off this big linux job.
            // No ARM64 VMM tests yet
            if matches!(arch, CommonArch::X86_64) {
                let pub_vmm_tests_archive_linux_x86 =
                    pub_vmm_tests_archive_linux_x86.take().unwrap();
                let pub_vmm_tests_archive_linux_musl_x86 =
                    pub_vmm_tests_archive_linux_musl_x86.take().unwrap();

                job = job.publish(pub_vmm_tests_archive_linux_x86, |archive| {
                        flowey_lib_hvlite::build_nextest_vmm_tests::Request {
                            target: CommonTriple::Common {
                                arch,
                                platform: CommonPlatform::LinuxGnu,
                            }.as_triple(),
                            profile: CommonProfile::from_release(release),
                            build_mode: flowey_lib_hvlite::build_nextest_vmm_tests::BuildNextestVmmTestsMode::Archive(
                                archive,
                            ),
                        }
                    }).publish(pub_vmm_tests_archive_linux_musl_x86, |archive| {
                    flowey_lib_hvlite::build_nextest_vmm_tests::Request {
                            target: CommonTriple::Common {
                                arch,
                                platform: CommonPlatform::LinuxMusl,
                            }.as_triple(),
                            profile: CommonProfile::from_release(release),
                            build_mode: flowey_lib_hvlite::build_nextest_vmm_tests::BuildNextestVmmTestsMode::Archive(
                                archive,
                            ),
                        }
                    });
            }

            all_jobs.push(job.finish());
        }

        let mut use_openhcl_igvm_files_mi_secure_x86 = None;

        // emit openhcl build job
        for (arch, mi_secure) in [
            (CommonArch::Aarch64, false),
            (CommonArch::X86_64, false),
            (CommonArch::X86_64, true),
        ] {
            let arch_tag = match arch {
                CommonArch::X86_64 => "x64",
                CommonArch::Aarch64 => "aarch64",
            };

            let tag = if mi_secure {
                format!("{arch_tag}-mi-secure")
            } else {
                arch_tag.to_string()
            };

            let openvmm_hcl_profile = if release {
                OpenvmmHclBuildProfile::OpenvmmHclShip
            } else {
                OpenvmmHclBuildProfile::Debug
            };

            let (pub_openhcl_igvm, use_openhcl_igvm) =
                pipeline.new_artifact(format!("{tag}-openhcl-igvm"));
            let (pub_openhcl_igvm_extras, _use_openhcl_igvm_extras) =
                pipeline.new_artifact(format!("{tag}-openhcl-igvm-extras"));

            let (pub_openhcl_baseline, _use_openhcl_baseline) =
                if matches!(config, PipelineConfig::Ci) && !mi_secure {
                    let (p, u) = pipeline.new_artifact(format!("{tag}-openhcl-baseline"));
                    (Some(p), Some(u))
                } else {
                    (None, None)
                };

            // skim off interesting artifacts required by the VMM tests job
            match (arch, mi_secure) {
                (CommonArch::X86_64, false) => {
                    vmm_tests_artifacts_windows_x86.use_openhcl_igvm_files = Some(use_openhcl_igvm);
                }
                (CommonArch::X86_64, true) => {
                    use_openhcl_igvm_files_mi_secure_x86 = Some(use_openhcl_igvm)
                }
                (CommonArch::Aarch64, false) => {
                    vmm_tests_artifacts_windows_aarch64.use_openhcl_igvm_files =
                        Some(use_openhcl_igvm);
                }
                _ => unreachable!(),
            }
            let igvm_recipes = match (arch, mi_secure) {
                (CommonArch::X86_64, false) => vec![
                    OpenhclIgvmRecipe::X64,
                    OpenhclIgvmRecipe::X64Devkern,
                    OpenhclIgvmRecipe::X64TestLinuxDirect,
                    OpenhclIgvmRecipe::X64TestLinuxDirectDevkern,
                    OpenhclIgvmRecipe::X64Cvm,
                ],
                (CommonArch::X86_64, true) => vec![
                    OpenhclIgvmRecipe::X64,
                    OpenhclIgvmRecipe::X64TestLinuxDirect,
                    OpenhclIgvmRecipe::X64Cvm,
                ],
                (CommonArch::Aarch64, false) => {
                    vec![
                        OpenhclIgvmRecipe::Aarch64,
                        OpenhclIgvmRecipe::Aarch64Devkern,
                    ]
                }
                _ => unreachable!(),
            };

            let build_openhcl_job_tag = |arch_tag, mi_secure| {
                format!(
                    "build openhcl {}[{arch_tag}-linux]",
                    if mi_secure { "(mi-secure) " } else { "" }
                )
            };
            let job = pipeline
                .new_job(
                    FlowPlatform::Linux(FlowPlatformLinuxDistro::Ubuntu),
                    FlowArch::X86_64,
                    build_openhcl_job_tag(arch_tag, mi_secure),
                )
                .gh_set_pool(gh_pools::default_linux())
                .ado_set_pool(ado_pools::default_linux())
                .dep_on(|ctx| {
                    let publish_baseline_artifact = pub_openhcl_baseline
                        .map(|baseline_artifact| ctx.publish_artifact(baseline_artifact));

                    flowey_lib_hvlite::_jobs::build_and_publish_openhcl_igvm_from_recipe::Params {
                        igvm_files: igvm_recipes
                            .clone()
                            .into_iter()
                            .map(|recipe| OpenhclIgvmBuildParams {
                                profile: openvmm_hcl_profile,
                                recipe,
                                custom_target: Some(CommonTriple::Custom(openhcl_musl_target(
                                    arch,
                                ))),
                                extra_features: if mi_secure {
                                    [OpenvmmHclFeature::MiSecure].into()
                                } else {
                                    BTreeSet::new()
                                },
                                // mi secure uses release_cfg=false to select dev manifests (with larger
                                // VTL2 memory) since mi-secure adds overhead that may not fit in
                                // the tighter release memory budget.
                                release_cfg: release && !mi_secure,
                            })
                            .collect(),
                        artifact_dir_openhcl_igvm: ctx.publish_artifact(pub_openhcl_igvm),
                        artifact_dir_openhcl_igvm_extras: ctx
                            .publish_artifact(pub_openhcl_igvm_extras),
                        artifact_openhcl_verify_size_baseline: publish_baseline_artifact,
                        done: ctx.new_done_handle(),
                    }
                });

            all_jobs.push(job.finish());

            // TODO: Once we have a few runs of the openvmm-mirror PR pipeline, this job can be re-worked to use ADO artifacts instead of GH artifacts.
            if matches!(config, PipelineConfig::Pr)
                && !matches!(backend_hint, PipelineBackendHint::Ado)
                && !mi_secure
            {
                let job = pipeline
                    .new_job(
                        FlowPlatform::Linux(FlowPlatformLinuxDistro::Ubuntu),
                        FlowArch::X86_64,
                        format!("verify openhcl binary size [{}]", arch_tag),
                    )
                    .gh_set_pool(gh_pools::linux_x64_gh())
                    .side_effect(|done| {
                        flowey_lib_hvlite::_jobs::check_openvmm_hcl_size::Request {
                            target: CommonTriple::Common {
                                arch,
                                platform: CommonPlatform::LinuxMusl,
                            },
                            done,
                            pipeline_name: "openvmm-ci.yaml".into(),
                            job_name: build_openhcl_job_tag(arch_tag, mi_secure),
                        }
                    });
                all_jobs.push(job.finish());
            }
        }

        // Emit clippy + unit-test jobs
        //
        // The only reason we bundle clippy and unit-tests together is to avoid
        // requiring another build agent.
        struct ClippyUnitTestJobParams<'a> {
            platform: FlowPlatform,
            arch: FlowArch,
            gh_pool: GhRunner,
            ado_pool: Option<AdoPool>,
            clippy_targets: Option<(&'a str, &'a [(Triple, bool)])>,
            unit_test_target: Option<(&'a str, Triple)>,
        }

        let macos_clippy_targets = [(target_lexicon::triple!("aarch64-apple-darwin"), false)];
        let x64_linux_macos_clippy_targets = [
            (target_lexicon::triple!("x86_64-unknown-linux-gnu"), false),
            (target_lexicon::triple!("aarch64-apple-darwin"), false),
        ];

        for ClippyUnitTestJobParams {
            platform,
            arch,
            gh_pool,
            ado_pool,
            clippy_targets,
            unit_test_target,
        } in [
            ClippyUnitTestJobParams {
                platform: FlowPlatform::Windows,
                arch: FlowArch::X86_64,
                gh_pool: gh_pools::windows_amd_1es(),
                ado_pool: Some(ado_pools::windows_amd_1es()),
                clippy_targets: Some((
                    "x64-windows",
                    &[(target_lexicon::triple!("x86_64-pc-windows-msvc"), false)],
                )),
                unit_test_target: Some((
                    "x64-windows",
                    target_lexicon::triple!("x86_64-pc-windows-msvc"),
                )),
            },
            ClippyUnitTestJobParams {
                platform: FlowPlatform::Linux(FlowPlatformLinuxDistro::Ubuntu),
                arch: FlowArch::X86_64,
                gh_pool: gh_pools::linux_amd_1es(),
                ado_pool: Some(ado_pools::linux_amd_1es()),
                clippy_targets: if quick_check_job.is_some() {
                    // quick check already ran clippy for x64-linux;
                    // still need macos cross-clippy here.
                    Some(("macos", macos_clippy_targets.as_slice()))
                } else {
                    Some((
                        "x64-linux, macos",
                        x64_linux_macos_clippy_targets.as_slice(),
                    ))
                },
                unit_test_target: Some((
                    "x64-linux",
                    target_lexicon::triple!("x86_64-unknown-linux-gnu"),
                )),
            },
            ClippyUnitTestJobParams {
                platform: FlowPlatform::Linux(FlowPlatformLinuxDistro::Ubuntu),
                arch: FlowArch::X86_64,
                gh_pool: gh_pools::linux_amd_1es(),
                ado_pool: Some(ado_pools::linux_amd_1es()),
                clippy_targets: Some((
                    "x64-linux-musl, misc nostd",
                    &[(openhcl_musl_target(CommonArch::X86_64), true)],
                )),
                unit_test_target: Some(("x64-linux-musl", openhcl_musl_target(CommonArch::X86_64))),
            },
            ClippyUnitTestJobParams {
                platform: FlowPlatform::Windows,
                arch: FlowArch::Aarch64,
                gh_pool: gh_pools::windows_arm_1es(),
                ado_pool: None,
                clippy_targets: Some((
                    "aarch64-windows",
                    &[(target_lexicon::triple!("aarch64-pc-windows-msvc"), false)],
                )),
                unit_test_target: Some((
                    "aarch64-windows",
                    target_lexicon::triple!("aarch64-pc-windows-msvc"),
                )),
            },
            ClippyUnitTestJobParams {
                platform: FlowPlatform::Linux(FlowPlatformLinuxDistro::Ubuntu),
                arch: FlowArch::Aarch64,
                gh_pool: gh_pools::linux_arm_1es(),
                ado_pool: None,
                clippy_targets: Some((
                    "aarch64-linux",
                    &[(target_lexicon::triple!("aarch64-unknown-linux-gnu"), false)],
                )),
                unit_test_target: Some((
                    "aarch64-linux",
                    target_lexicon::triple!("aarch64-unknown-linux-gnu"),
                )),
            },
            ClippyUnitTestJobParams {
                platform: FlowPlatform::Linux(FlowPlatformLinuxDistro::Ubuntu),
                arch: FlowArch::Aarch64,
                gh_pool: gh_pools::linux_arm_1es(),
                ado_pool: None,
                clippy_targets: Some((
                    "aarch64-linux-musl, misc nostd",
                    &[(openhcl_musl_target(CommonArch::Aarch64), true)],
                )),
                unit_test_target: Some((
                    "aarch64-linux-musl",
                    openhcl_musl_target(CommonArch::Aarch64),
                )),
            },
        ] {
            // Skip unsupported jobs on ADO backend
            if matches!(backend_hint, PipelineBackendHint::Ado) && ado_pool.is_none() {
                continue;
            }

            let mut job_name = Vec::new();
            if let Some((label, _)) = &clippy_targets {
                job_name.push(format!("clippy [{label}]"));
            }
            if let Some((label, _)) = &unit_test_target {
                job_name.push(format!("unit tests [{label}]"));
            }
            let job_name = job_name.join(", ");

            let unit_test_target = unit_test_target.map(|(label, target)| {
                let test_label = format!("{label}-unit-tests");
                let pub_unit_test_junit_xml = if matches!(backend_hint, PipelineBackendHint::Local)
                {
                    Some(pipeline.new_artifact(&test_label).0)
                } else {
                    None
                };
                (test_label, target, pub_unit_test_junit_xml)
            });

            let mut clippy_unit_test_job = pipeline
                .new_job(platform, arch, job_name)
                .gh_set_pool(gh_pool);

            if let Some(pool) = ado_pool {
                clippy_unit_test_job = clippy_unit_test_job.ado_set_pool(pool);
            }

            if let Some((_, targets)) = clippy_targets {
                for (target, also_check_misc_nostd_crates) in targets {
                    clippy_unit_test_job = clippy_unit_test_job.side_effect(|done| {
                        flowey_lib_hvlite::_jobs::check_clippy::Request {
                            target: target.clone(),
                            profile: CommonProfile::from_release(release),
                            done,
                            also_check_misc_nostd_crates: *also_check_misc_nostd_crates,
                        }
                    });
                }
            }

            if let Some((test_label, target, pub_unit_test_junit_xml)) = unit_test_target {
                clippy_unit_test_job = clippy_unit_test_job
                    .dep_on(|ctx| {
                        flowey_lib_hvlite::_jobs::build_and_run_nextest_unit_tests::Params {
                            junit_test_label: test_label,
                            nextest_profile:
                                flowey_lib_hvlite::run_cargo_nextest_run::NextestProfile::Ci,
                            fail_job_on_test_fail: true,
                            target: target.clone(),
                            profile: CommonProfile::from_release(release),
                            artifact_dir: pub_unit_test_junit_xml.map(|x| ctx.publish_artifact(x)),
                            done: ctx.new_done_handle(),
                        }
                    })
                    .side_effect(|done| {
                        flowey_lib_hvlite::_jobs::build_and_run_doc_tests::Params {
                            target,
                            profile: CommonProfile::from_release(release),
                            done,
                        }
                    });
            }

            all_jobs.push(clippy_unit_test_job.finish());
        }

        let vmm_tests_artifacts_windows_intel_x86 = vmm_tests_artifacts_windows_x86
            .clone()
            .finish()
            .map_err(|missing| {
                anyhow::anyhow!("missing required windows-intel vmm_tests artifact: {missing}")
            })?;
        let vmm_tests_artifacts_windows_intel_mi_secure_x86 = {
            let mut builder = vmm_tests_artifacts_windows_x86.clone();
            builder.use_openhcl_igvm_files = use_openhcl_igvm_files_mi_secure_x86;
            builder
        }
        .finish()
        .map_err(|missing| {
            anyhow::anyhow!(
                "missing required windows-intel-mi-secure vmm_tests artifact: {missing}"
            )
        })?;
        let vmm_tests_artifacts_windows_intel_tdx_x86 = vmm_tests_artifacts_windows_x86
            .clone()
            .finish()
            .map_err(|missing| {
                anyhow::anyhow!("missing required windows-intel-tdx vmm_tests artifact: {missing}")
            })?;
        let vmm_tests_artifacts_windows_amd_x86 = vmm_tests_artifacts_windows_x86
            .clone()
            .finish()
            .map_err(|missing| {
                anyhow::anyhow!("missing required windows-amd vmm_tests artifact: {missing}")
            })?;
        let vmm_tests_artifacts_windows_amd_snp_x86 = vmm_tests_artifacts_windows_x86
            .clone()
            .finish()
            .map_err(|missing| {
                anyhow::anyhow!("missing required windows-amd-snp vmm_tests artifact: {missing}")
            })?;
        let vmm_tests_artifacts_linux_mshv_x86 = vmm_tests_artifacts_linux_musl_x86
            .finish()
            .map_err(|missing| {
                anyhow::anyhow!("missing required linux-mshv (musl) vmm_tests artifact: {missing}")
            })?;
        let vmm_tests_artifacts_linux_x86 =
            vmm_tests_artifacts_linux_x86.finish().map_err(|missing| {
                anyhow::anyhow!("missing required linux vmm_tests artifact: {missing}")
            })?;
        let vmm_tests_artifacts_windows_aarch64 = vmm_tests_artifacts_windows_aarch64
            .finish()
            .map_err(|missing| {
                anyhow::anyhow!("missing required windows-aarch64 vmm_tests artifact: {missing}")
            })?;

        // Emit VMM tests runner jobs
        struct VmmTestJobParams<'a> {
            platform: FlowPlatform,
            arch: FlowArch,
            gh_pool: GhRunner,
            ado_pool: Option<AdoPool>,
            label: &'a str,
            target: CommonTriple,
            resolve_vmm_tests_artifacts: vmm_tests_artifact_builders::ResolveVmmTestsDepArtifacts,
            nextest_filter_expr: String,
            test_artifacts: Vec<KnownTestArtifacts>,
            needs_prep_run: bool,
            hugetlb_2mb_overcommit_pages: Option<u64>,
        }

        let standard_filter = {
            // Standard VM-based CI machines should be able to run all tests except
            // those that require special hardware features (tdx/snp) or need to be
            // run on a baremetal host (hyper-v vbs doesn't seem to work nested).
            //
            // Run "very_heavy" tests that require lots of VPs on the self-hosted
            // CVM runners that have more cores.
            //
            // Even though OpenVMM + VBS + Windows tests can run on standard CI
            // machines, we exclude them here to avoid needing to run prep_steps
            // on non-self-hosted runners. This saves several minutes of CI time
            // that would be used for very few tests. We need to run prep_steps
            // on CVM runners anyways, so we might as well run those tests there.
            //
            // Our standard runners need to be updated to support Hyper-V OpenHCL
            // PCAT, so run those tests on the CVM runners for now.
            let mut filter = "all() & !test(very_heavy) & !test(openvmm_openhcl_uefi_x64_windows_datacenter_core_2025_x64_prepped_vbs) & !test(hyperv_openhcl_pcat)".to_string();
            // Currently, we don't have a good way for ADO runners to authenticate in GitHub
            // (that don't involve PATs) which is a requirement to download GH Workflow Artifacts
            // required by the upgrade and downgrade servicing tests. For now,
            // we will exclude these tests from running in the internal mirror.
            // Our standard runners also need to be updated to run Hyper-V
            // servicing tests.
            match backend_hint {
                PipelineBackendHint::Ado => {
                    filter.push_str(
                        " & !(test(servicing) & (test(upgrade) + test(downgrade) + test(hyperv)))",
                    );
                }
                _ => {
                    filter.push_str(" & !(test(servicing) & test(hyperv))");
                }
            }
            filter
        };

        let standard_x64_test_artifacts = vec![
            KnownTestArtifacts::Alpine323X64Vhd,
            KnownTestArtifacts::FreeBsd13_2X64Vhd,
            KnownTestArtifacts::FreeBsd13_2X64Iso,
            KnownTestArtifacts::Gen1WindowsDataCenterCore2022X64Vhd,
            KnownTestArtifacts::Gen2WindowsDataCenterCore2022X64Vhd,
            KnownTestArtifacts::Gen2WindowsDataCenterCore2025X64Vhd,
            KnownTestArtifacts::Ubuntu2404ServerX64Vhd,
            KnownTestArtifacts::Ubuntu2504ServerX64Vhd,
            KnownTestArtifacts::VmgsWithBootEntry,
            KnownTestArtifacts::VmgsWith16kTpm,
        ];

        let cvm_filter = |isolation_type| {
            let mut filter = format!(
                "test({isolation_type}) + (test(vbs) & test(hyperv)) + test(very_heavy) + test(openvmm_openhcl_uefi_x64_windows_datacenter_core_2025_x64_prepped_vbs)"
            );
            // OpenHCL PCAT tests are flakey on AMD SNP runners, so only run on TDX for now
            if isolation_type == "tdx" {
                filter.push_str(" + test(hyperv_openhcl_pcat)");
            }

            // See comment for standard filter. Run hyper-v servicing tests on CVM runners.
            match backend_hint {
                PipelineBackendHint::Ado => {
                    filter.push_str(
                        " + (test(servicing) & !(test(upgrade) + test(downgrade)) & test(hyperv))",
                    );
                }
                _ => {
                    filter.push_str(" + (test(servicing) & test(hyperv))");
                }
            }

            // Exclude any PCAT tests that were picked up by other filters
            if isolation_type == "snp" {
                filter = format!("({filter}) & !test(pcat)")
            }
            filter
        };
        let cvm_x64_test_artifacts = vec![
            KnownTestArtifacts::Gen1WindowsDataCenterCore2022X64Vhd,
            KnownTestArtifacts::Gen2WindowsDataCenterCore2022X64Vhd,
            KnownTestArtifacts::Gen2WindowsDataCenterCore2025X64Vhd,
            KnownTestArtifacts::Ubuntu2504ServerX64Vhd,
            KnownTestArtifacts::VmgsWith16kTpm,
        ];

        for VmmTestJobParams {
            platform,
            arch,
            gh_pool,
            ado_pool,
            label,
            target,
            resolve_vmm_tests_artifacts,
            nextest_filter_expr,
            test_artifacts,
            needs_prep_run,
            hugetlb_2mb_overcommit_pages,
        } in [
            VmmTestJobParams {
                platform: FlowPlatform::Windows,
                arch: FlowArch::X86_64,
                gh_pool: gh_pools::windows_intel_1es(),
                ado_pool: Some(ado_pools::windows_intel_1es()),
                label: "x64-windows-intel",
                target: CommonTriple::X86_64_WINDOWS_MSVC,
                resolve_vmm_tests_artifacts: vmm_tests_artifacts_windows_intel_x86,
                nextest_filter_expr: standard_filter.clone(),
                test_artifacts: standard_x64_test_artifacts.clone(),
                needs_prep_run: false,
                hugetlb_2mb_overcommit_pages: None,
            },
            VmmTestJobParams {
                platform: FlowPlatform::Windows,
                arch: FlowArch::X86_64,
                gh_pool: gh_pools::windows_intel_1es(),
                ado_pool: Some(ado_pools::windows_intel_1es()),
                label: "x64-windows-intel-mi-secure",
                target: CommonTriple::X86_64_WINDOWS_MSVC,
                resolve_vmm_tests_artifacts: vmm_tests_artifacts_windows_intel_mi_secure_x86,
                nextest_filter_expr: "test(openhcl) & !test(servicing) & !test(cvm) & !test(memory_validation) & !test(very_heavy) & !test(hyperv_openhcl_pcat) & !test(prepped_vbs) & !test(256mb)"
                    .to_string(),
                test_artifacts: standard_x64_test_artifacts.clone(),
                needs_prep_run: false,
                hugetlb_2mb_overcommit_pages: None,
            },
            VmmTestJobParams {
                platform: FlowPlatform::Windows,
                arch: FlowArch::X86_64,
                gh_pool: gh_pools::windows_tdx_self_hosted_baremetal(),
                ado_pool: None,
                label: "x64-windows-intel-tdx",
                target: CommonTriple::X86_64_WINDOWS_MSVC,
                resolve_vmm_tests_artifacts: vmm_tests_artifacts_windows_intel_tdx_x86,
                nextest_filter_expr: cvm_filter("tdx"),
                test_artifacts: cvm_x64_test_artifacts.clone(),
                needs_prep_run: true,
                hugetlb_2mb_overcommit_pages: None,
            },
            VmmTestJobParams {
                platform: FlowPlatform::Windows,
                arch: FlowArch::X86_64,
                gh_pool: gh_pools::windows_amd_1es(),
                ado_pool: Some(ado_pools::windows_amd_1es()),
                label: "x64-windows-amd",
                target: CommonTriple::X86_64_WINDOWS_MSVC,
                resolve_vmm_tests_artifacts: vmm_tests_artifacts_windows_amd_x86,
                nextest_filter_expr: standard_filter.clone(),
                test_artifacts: standard_x64_test_artifacts.clone(),
                needs_prep_run: false,
                hugetlb_2mb_overcommit_pages: None,
            },
            VmmTestJobParams {
                platform: FlowPlatform::Windows,
                arch: FlowArch::X86_64,
                gh_pool: gh_pools::windows_snp_self_hosted_baremetal(),
                ado_pool: None,
                label: "x64-windows-amd-snp",
                target: CommonTriple::X86_64_WINDOWS_MSVC,
                resolve_vmm_tests_artifacts: vmm_tests_artifacts_windows_amd_snp_x86,
                nextest_filter_expr: cvm_filter("snp"),
                test_artifacts: cvm_x64_test_artifacts,
                needs_prep_run: true,
                hugetlb_2mb_overcommit_pages: None,
            },
            VmmTestJobParams {
                platform: FlowPlatform::Linux(FlowPlatformLinuxDistro::Ubuntu),
                arch: FlowArch::X86_64,
                gh_pool: gh_pools::linux_amd_1es(),
                ado_pool: Some(ado_pools::linux_amd_1es()),
                label: "x64-linux-amd-kvm",
                target: CommonTriple::X86_64_LINUX_GNU,
                resolve_vmm_tests_artifacts: vmm_tests_artifacts_linux_x86,
                // - No legal way to obtain gen1 pcat blobs on non-msft linux machines
                nextest_filter_expr: format!("{standard_filter} & !test(pcat_x64)"),
                test_artifacts: standard_x64_test_artifacts.clone(),
                needs_prep_run: false,
                hugetlb_2mb_overcommit_pages: Some(HUGETLB_2MB_OVERCOMMIT_PAGES),
            },
            VmmTestJobParams {
                platform: FlowPlatform::Linux(FlowPlatformLinuxDistro::AzureLinux),
                arch: FlowArch::X86_64,
                gh_pool: gh_pools::linux_mshv_1es(),
                ado_pool: None,
                label: "x64-linux-intel-mshv",
                target: CommonTriple::X86_64_LINUX_MUSL,
                resolve_vmm_tests_artifacts: vmm_tests_artifacts_linux_mshv_x86,
                // - No legal way to obtain gen1 pcat blobs on non-msft linux machines
                nextest_filter_expr: format!("{standard_filter} & !test(pcat_x64)"),
                test_artifacts: standard_x64_test_artifacts.clone(),
                needs_prep_run: false,
                hugetlb_2mb_overcommit_pages: None,
            },
            VmmTestJobParams {
                platform: FlowPlatform::Windows,
                arch: FlowArch::Aarch64,
                gh_pool: gh_pools::windows_arm_self_hosted_baremetal(),
                ado_pool: None,
                label: "aarch64-windows",
                target: CommonTriple::AARCH64_WINDOWS_MSVC,
                resolve_vmm_tests_artifacts: vmm_tests_artifacts_windows_aarch64,
                nextest_filter_expr: "all()".to_string(),
                test_artifacts: vec![
                    KnownTestArtifacts::Alpine323Aarch64Vhd,
                    KnownTestArtifacts::Ubuntu2404ServerAarch64Vhd,
                    KnownTestArtifacts::Windows11EnterpriseAarch64Vhdx,
                    KnownTestArtifacts::VmgsWithBootEntry,
                    KnownTestArtifacts::VmgsWith16kTpm,
                ],
                needs_prep_run: false,
                hugetlb_2mb_overcommit_pages: None,
            },
        ] {
            // Skip unsupported jobs on ADO backend
            if matches!(backend_hint, PipelineBackendHint::Ado) && ado_pool.is_none() {
                continue;
            }

            let test_label = format!("{label}-vmm-tests");

            let pub_vmm_tests_results = if matches!(backend_hint, PipelineBackendHint::Local) {
                Some(pipeline.new_artifact(&test_label).0)
            } else {
                None
            };

            let use_vmm_tests_archive = match target {
                CommonTriple::X86_64_WINDOWS_MSVC => &use_vmm_tests_archive_windows_x86,
                CommonTriple::X86_64_LINUX_GNU => &use_vmm_tests_archive_linux_x86,
                CommonTriple::X86_64_LINUX_MUSL => &use_vmm_tests_archive_linux_musl_x86,
                CommonTriple::AARCH64_WINDOWS_MSVC => &use_vmm_tests_archive_windows_aarch64,
                _ => unreachable!(),
            };

            let mut vmm_tests_run_job = pipeline
                .new_job(platform, arch, format!("run vmm-tests [{label}]"))
                .gh_set_pool(gh_pool);

            if let Some(pool) = ado_pool {
                vmm_tests_run_job = vmm_tests_run_job.ado_set_pool(pool);
            }

            vmm_tests_run_job = vmm_tests_run_job.dep_on(|ctx| {
                flowey_lib_hvlite::_jobs::consume_and_test_nextest_vmm_tests_archive::Params {
                    junit_test_label: test_label,
                    nextest_vmm_tests_archive: ctx.use_typed_artifact(use_vmm_tests_archive),
                    target: target.as_triple(),
                    nextest_profile: flowey_lib_hvlite::run_cargo_nextest_run::NextestProfile::Ci,
                    nextest_filter_expr: Some(nextest_filter_expr),
                    dep_artifact_dirs: resolve_vmm_tests_artifacts(ctx),
                    test_artifacts,
                    fail_job_on_test_fail: true,
                    artifact_dir: pub_vmm_tests_results.map(|x| ctx.publish_artifact(x)),
                    needs_prep_run,
                    hugetlb_2mb_overcommit_pages,
                    done: ctx.new_done_handle(),
                }
            });

            if let Some(vmm_tests_disk_cache_dir) = vmm_tests_disk_cache_dir.clone() {
                vmm_tests_run_job = vmm_tests_run_job.config(
                    flowey_lib_hvlite::download_openvmm_vmm_tests_artifacts::Config {
                        custom_cache_dir: Some(vmm_tests_disk_cache_dir),
                        ..Default::default()
                    },
                );
            }

            all_jobs.push(vmm_tests_run_job.finish());
        }

        // test the flowey local backend by running cargo xflowey build-igvm on x64
        {
            if matches!(backend_hint, PipelineBackendHint::Github) {
                let job = pipeline
                    .new_job(
                        FlowPlatform::Linux(FlowPlatformLinuxDistro::Ubuntu),
                        FlowArch::X86_64,
                        "test flowey local backend",
                    )
                    .gh_set_pool(gh_pools::linux_x64_gh())
                    .side_effect(|done| {
                        flowey_lib_hvlite::_jobs::test_local_flowey_build_igvm::Request {
                            base_recipe: OpenhclIgvmRecipe::X64,
                            done,
                        }
                    });
                all_jobs.push(job.finish());
            }
        }

        // all jobs depend on the quick-check gate
        if let Some(ref quick_check) = quick_check_job {
            for job in all_jobs.iter() {
                pipeline.non_artifact_dep(job, quick_check);
            }
            all_jobs.push(quick_check.clone());
        }

        if matches!(config, PipelineConfig::Pr)
            && matches!(backend_hint, PipelineBackendHint::Github)
        {
            // Add a job that depends on all others as a workaround for
            // https://github.com/orgs/community/discussions/12395.
            //
            // This workaround then itself requires _another_ workaround, requiring
            // the use of `gh_dangerous_override_if`, and some additional custom job
            // logic, to deal with https://github.com/actions/runner/issues/2566.
            //
            // TODO: Add a way for this job to skip flowey setup and become a true
            // no-op.
            let all_good_job = pipeline
                .new_job(
                    FlowPlatform::Linux(FlowPlatformLinuxDistro::Ubuntu),
                    FlowArch::X86_64,
                    "openvmm checkin gates",
                )
                .gh_set_pool(gh_pools::linux_x64_gh())
                // always run this job, regardless whether or not any previous jobs failed
                .gh_dangerous_override_if("always() && github.event.pull_request.draft == false")
                .gh_dangerous_global_env_var("ANY_JOBS_FAILED", "${{ contains(needs.*.result, 'cancelled') || contains(needs.*.result, 'failure') }}")
                .side_effect(|done| flowey_lib_hvlite::_jobs::all_good_job::Params {
                    did_fail_env_var: "ANY_JOBS_FAILED".into(),
                    done,
                })
                .finish();

            for job in all_jobs.iter() {
                pipeline.non_artifact_dep(&all_good_job, job);
            }
        }

        if matches!(config, PipelineConfig::Ci)
            && matches!(backend_hint, PipelineBackendHint::Github)
        {
            let publish_vmgstool_job = pipeline
                .new_job(
                    FlowPlatform::Linux(FlowPlatformLinuxDistro::Ubuntu),
                    FlowArch::X86_64,
                    "publish vmgstool",
                )
                .gh_grant_permissions::<flowey_lib_common::publish_gh_release::Node>([(
                    GhPermission::Contents,
                    GhPermissionValue::Write,
                )])
                .gh_set_pool(gh_pools::linux_x64_gh())
                .dep_on(
                    |ctx| flowey_lib_hvlite::_jobs::publish_vmgstool_gh_release::Request {
                        vmgstools: vmgstools
                            .into_iter()
                            .map(|(t, v)| (t, ctx.use_typed_artifact(&v)))
                            .collect(),
                        done: ctx.new_done_handle(),
                    },
                )
                .finish();

            // All other jobs must succeed in order to publish
            for job in all_jobs.iter() {
                pipeline.non_artifact_dep(&publish_vmgstool_job, job);
            }
        }

        Ok(pipeline)
    }
}

/// Utility builders which make it easy to "skim off" artifacts required by VMM
/// test execution from other pipeline jobs.
//
// FUTURE: if we end up having a _lot_ of VMM test jobs, this would be the sort
// of thing that would really benefit from a derive macro.
mod vmm_tests_artifact_builders {
    use flowey::pipeline::prelude::*;
    use flowey_lib_hvlite::_jobs::consume_and_test_nextest_vmm_tests_archive::VmmTestsDepArtifacts;
    use flowey_lib_hvlite::build_guest_test_uefi::GuestTestUefiOutput;
    use flowey_lib_hvlite::build_openvmm::OpenvmmOutput;
    use flowey_lib_hvlite::build_openvmm_vhost::OpenvmmVhostOutput;
    use flowey_lib_hvlite::build_pipette::PipetteOutput;
    use flowey_lib_hvlite::build_prep_steps::PrepStepsOutput;
    use flowey_lib_hvlite::build_test_igvm_agent_rpc_server::TestIgvmAgentRpcServerOutput;
    use flowey_lib_hvlite::build_tmk_vmm::TmkVmmOutput;
    use flowey_lib_hvlite::build_tmks::TmksOutput;
    use flowey_lib_hvlite::build_tpm_guest_tests::TpmGuestTestsOutput;
    use flowey_lib_hvlite::build_vmgstool::VmgstoolOutput;

    pub type ResolveVmmTestsDepArtifacts =
        Box<dyn Fn(&mut PipelineJobCtx<'_>) -> VmmTestsDepArtifacts>;

    #[derive(Default, Clone)]
    pub struct VmmTestsArtifactsBuilderLinuxX86 {
        // windows build machine
        pub use_pipette_windows: Option<UseTypedArtifact<PipetteOutput>>,
        pub use_tmk_vmm: Option<UseTypedArtifact<TmkVmmOutput>>,
        // linux build machine
        pub use_openvmm: Option<UseTypedArtifact<OpenvmmOutput>>,
        pub use_openvmm_vhost: Option<UseTypedArtifact<OpenvmmVhostOutput>>,
        pub use_pipette_linux_musl: Option<UseTypedArtifact<PipetteOutput>>,
        // any machine
        pub use_guest_test_uefi: Option<UseTypedArtifact<GuestTestUefiOutput>>,
        pub use_tmks: Option<UseTypedArtifact<TmksOutput>>,
    }

    impl VmmTestsArtifactsBuilderLinuxX86 {
        pub fn finish(self) -> Result<ResolveVmmTestsDepArtifacts, &'static str> {
            let VmmTestsArtifactsBuilderLinuxX86 {
                use_openvmm,
                use_openvmm_vhost,
                use_guest_test_uefi,
                use_pipette_windows,
                use_pipette_linux_musl,
                use_tmk_vmm,
                use_tmks,
            } = self;

            let use_guest_test_uefi = use_guest_test_uefi.ok_or("guest_test_uefi")?;
            let use_openvmm = use_openvmm.ok_or("openvmm")?;
            let use_pipette_linux_musl = use_pipette_linux_musl.ok_or("pipette_linux_musl")?;
            let use_pipette_windows = use_pipette_windows.ok_or("pipette_windows")?;
            let use_tmk_vmm = use_tmk_vmm.ok_or("tmk_vmm")?;
            let use_tmks = use_tmks.ok_or("tmks")?;

            Ok(Box::new(move |ctx| VmmTestsDepArtifacts {
                openvmm: Some(ctx.use_typed_artifact(&use_openvmm)),
                openvmm_vhost: use_openvmm_vhost
                    .as_ref()
                    .map(|a| ctx.use_typed_artifact(a)),
                pipette_windows: Some(ctx.use_typed_artifact(&use_pipette_windows)),
                pipette_linux_musl: Some(ctx.use_typed_artifact(&use_pipette_linux_musl)),
                guest_test_uefi: Some(ctx.use_typed_artifact(&use_guest_test_uefi)),
                tmk_vmm: Some(ctx.use_typed_artifact(&use_tmk_vmm)),
                tmks: Some(ctx.use_typed_artifact(&use_tmks)),
                // not currently required, since OpenHCL tests cannot be run on OpenVMM on linux
                artifact_dir_openhcl_igvm_files: None,
                tmk_vmm_linux_musl: None,
                prep_steps: None,
                vmgstool: None,
                tpm_guest_tests_windows: None,
                tpm_guest_tests_linux: None,
                test_igvm_agent_rpc_server: None,
            }))
        }
    }

    #[derive(Default, Clone)]
    pub struct VmmTestsArtifactsBuilderWindowsX86 {
        // windows build machine
        pub use_openvmm: Option<UseTypedArtifact<OpenvmmOutput>>,
        pub use_pipette_windows: Option<UseTypedArtifact<PipetteOutput>>,
        pub use_tmk_vmm: Option<UseTypedArtifact<TmkVmmOutput>>,
        pub use_prep_steps: Option<UseTypedArtifact<PrepStepsOutput>>,
        pub use_vmgstool: Option<UseTypedArtifact<VmgstoolOutput>>,
        pub use_tpm_guest_tests_windows: Option<UseTypedArtifact<TpmGuestTestsOutput>>,
        pub use_tpm_guest_tests_linux: Option<UseTypedArtifact<TpmGuestTestsOutput>>,
        pub use_test_igvm_agent_rpc_server: Option<UseTypedArtifact<TestIgvmAgentRpcServerOutput>>,
        // linux build machine
        pub use_openhcl_igvm_files: Option<UseArtifact>,
        pub use_pipette_linux_musl: Option<UseTypedArtifact<PipetteOutput>>,
        pub use_tmk_vmm_linux_musl: Option<UseTypedArtifact<TmkVmmOutput>>,
        // any machine
        pub use_guest_test_uefi: Option<UseTypedArtifact<GuestTestUefiOutput>>,
        pub use_tmks: Option<UseTypedArtifact<TmksOutput>>,
    }

    impl VmmTestsArtifactsBuilderWindowsX86 {
        pub fn finish(self) -> Result<ResolveVmmTestsDepArtifacts, &'static str> {
            let VmmTestsArtifactsBuilderWindowsX86 {
                use_openvmm,
                use_pipette_windows,
                use_pipette_linux_musl,
                use_guest_test_uefi,
                use_openhcl_igvm_files,
                use_tmk_vmm,
                use_tmk_vmm_linux_musl,
                use_tmks,
                use_prep_steps,
                use_vmgstool,
                use_tpm_guest_tests_windows,
                use_tpm_guest_tests_linux,
                use_test_igvm_agent_rpc_server,
            } = self;

            let use_openvmm = use_openvmm.ok_or("openvmm")?;
            let use_pipette_windows = use_pipette_windows.ok_or("pipette_windows")?;
            let use_pipette_linux_musl = use_pipette_linux_musl.ok_or("pipette_linux_musl")?;
            let use_guest_test_uefi = use_guest_test_uefi.ok_or("guest_test_uefi")?;
            let use_openhcl_igvm_files = use_openhcl_igvm_files.ok_or("openhcl_igvm_files")?;
            let use_tmk_vmm = use_tmk_vmm.ok_or("tmk_vmm")?;
            let use_tmk_vmm_linux_musl = use_tmk_vmm_linux_musl.ok_or("tmk_vmm_linux_musl")?;
            let use_tmks = use_tmks.ok_or("tmks")?;
            let use_prep_steps = use_prep_steps.ok_or("prep_steps")?;
            let use_vmgstool = use_vmgstool.ok_or("vmgstool")?;
            let use_tpm_guest_tests_windows =
                use_tpm_guest_tests_windows.ok_or("tpm_guest_tests_windows")?;
            let use_tpm_guest_tests_linux =
                use_tpm_guest_tests_linux.ok_or("tpm_guest_tests_linux")?;
            let use_test_igvm_agent_rpc_server =
                use_test_igvm_agent_rpc_server.ok_or("test_igvm_agent_rpc_server")?;

            Ok(Box::new(move |ctx| VmmTestsDepArtifacts {
                openvmm: Some(ctx.use_typed_artifact(&use_openvmm)),
                openvmm_vhost: None,
                pipette_windows: Some(ctx.use_typed_artifact(&use_pipette_windows)),
                pipette_linux_musl: Some(ctx.use_typed_artifact(&use_pipette_linux_musl)),
                guest_test_uefi: Some(ctx.use_typed_artifact(&use_guest_test_uefi)),
                artifact_dir_openhcl_igvm_files: Some(ctx.use_artifact(&use_openhcl_igvm_files)),
                tmk_vmm: Some(ctx.use_typed_artifact(&use_tmk_vmm)),
                tmk_vmm_linux_musl: Some(ctx.use_typed_artifact(&use_tmk_vmm_linux_musl)),
                tmks: Some(ctx.use_typed_artifact(&use_tmks)),
                prep_steps: Some(ctx.use_typed_artifact(&use_prep_steps)),
                vmgstool: Some(ctx.use_typed_artifact(&use_vmgstool)),
                tpm_guest_tests_windows: Some(ctx.use_typed_artifact(&use_tpm_guest_tests_windows)),
                tpm_guest_tests_linux: Some(ctx.use_typed_artifact(&use_tpm_guest_tests_linux)),
                test_igvm_agent_rpc_server: Some(
                    ctx.use_typed_artifact(&use_test_igvm_agent_rpc_server),
                ),
            }))
        }
    }

    #[derive(Default, Clone)]
    pub struct VmmTestsArtifactsBuilderWindowsAarch64 {
        // windows build machine
        pub use_openvmm: Option<UseTypedArtifact<OpenvmmOutput>>,
        pub use_pipette_windows: Option<UseTypedArtifact<PipetteOutput>>,
        pub use_tmk_vmm: Option<UseTypedArtifact<TmkVmmOutput>>,
        pub use_vmgstool: Option<UseTypedArtifact<VmgstoolOutput>>,
        // linux build machine
        pub use_openhcl_igvm_files: Option<UseArtifact>,
        pub use_pipette_linux_musl: Option<UseTypedArtifact<PipetteOutput>>,
        pub use_tmk_vmm_linux_musl: Option<UseTypedArtifact<TmkVmmOutput>>,
        // any machine
        pub use_guest_test_uefi: Option<UseTypedArtifact<GuestTestUefiOutput>>,
        pub use_tmks: Option<UseTypedArtifact<TmksOutput>>,
    }

    impl VmmTestsArtifactsBuilderWindowsAarch64 {
        pub fn finish(self) -> Result<ResolveVmmTestsDepArtifacts, &'static str> {
            let VmmTestsArtifactsBuilderWindowsAarch64 {
                use_openvmm,
                use_pipette_windows,
                use_pipette_linux_musl,
                use_guest_test_uefi,
                use_openhcl_igvm_files,
                use_tmk_vmm,
                use_tmk_vmm_linux_musl,
                use_tmks,
                use_vmgstool,
            } = self;

            let use_openvmm = use_openvmm.ok_or("openvmm")?;
            let use_pipette_windows = use_pipette_windows.ok_or("pipette_windows")?;
            let use_pipette_linux_musl = use_pipette_linux_musl.ok_or("pipette_linux_musl")?;
            let use_guest_test_uefi = use_guest_test_uefi.ok_or("guest_test_uefi")?;
            let use_openhcl_igvm_files = use_openhcl_igvm_files.ok_or("openhcl_igvm_files")?;
            let use_tmk_vmm = use_tmk_vmm.ok_or("tmk_vmm")?;
            let use_tmk_vmm_linux_musl = use_tmk_vmm_linux_musl.ok_or("tmk_vmm_linux_musl")?;
            let use_tmks = use_tmks.ok_or("tmks")?;
            let use_vmgstool = use_vmgstool.ok_or("vmgstool")?;

            Ok(Box::new(move |ctx| VmmTestsDepArtifacts {
                openvmm: Some(ctx.use_typed_artifact(&use_openvmm)),
                openvmm_vhost: None,
                pipette_windows: Some(ctx.use_typed_artifact(&use_pipette_windows)),
                pipette_linux_musl: Some(ctx.use_typed_artifact(&use_pipette_linux_musl)),
                guest_test_uefi: Some(ctx.use_typed_artifact(&use_guest_test_uefi)),
                artifact_dir_openhcl_igvm_files: Some(ctx.use_artifact(&use_openhcl_igvm_files)),
                tmk_vmm: Some(ctx.use_typed_artifact(&use_tmk_vmm)),
                tmk_vmm_linux_musl: Some(ctx.use_typed_artifact(&use_tmk_vmm_linux_musl)),
                tmks: Some(ctx.use_typed_artifact(&use_tmks)),
                prep_steps: None,
                vmgstool: Some(ctx.use_typed_artifact(&use_vmgstool)),
                tpm_guest_tests_windows: None,
                tpm_guest_tests_linux: None,
                test_igvm_agent_rpc_server: None,
            }))
        }
    }
}