petri/vm/openvmm/

construct.rs

// Copyright (c) Microsoft Corporation.
// Licensed under the MIT License.

//! Contains [`PetriVmConfigOpenVmm::new`], which builds a [`PetriVmConfigOpenVmm`] with all
//! default settings for a given [`Firmware`] and [`MachineArch`].

use super::PetriVmConfigOpenVmm;
use super::PetriVmResourcesOpenVmm;
use crate::Drive;
use crate::Firmware;
use crate::IsolationType;
use crate::MemoryConfig;
use crate::OpenHclConfig;
use crate::PetriLogSource;
use crate::PetriVmConfig;
use crate::PetriVmResources;
use crate::PetriVmgsResource;
use crate::ProcessorTopology;
use crate::SecureBootTemplate;
use crate::TpmConfig;
use crate::UefiConfig;
use crate::VmbusStorageType;
use crate::linux_direct_serial_agent::LinuxDirectSerialAgent;

use crate::MmioConfig;
use crate::SIZE_1_MB;
use crate::VmbusStorageController;
use crate::openvmm::memdiff_vmgs;
use crate::openvmm::petri_disk_to_openvmm;
use crate::vm::PetriVmProperties;
use crate::vm::append_cmdline;
use anyhow::Context;
use framebuffer::FRAMEBUFFER_SIZE;
use framebuffer::Framebuffer;
use framebuffer::FramebufferAccess;
use fs_err::File;
use futures::StreamExt;
use get_resources::crash::GuestCrashDeviceHandle;
use get_resources::ged::FirmwareEvent;
use guid::Guid;
use hyperv_ic_resources::shutdown::ShutdownIcHandle;
use ide_resources::GuestMedia;
use ide_resources::IdeDeviceConfig;
use mesh_process::Mesh;
use nvme_resources::NamespaceDefinition;
use nvme_resources::NvmeControllerHandle;
use openvmm_defs::config::Config;
use openvmm_defs::config::DEFAULT_MMIO_GAPS_AARCH64;
use openvmm_defs::config::DEFAULT_MMIO_GAPS_AARCH64_WITH_VTL2;
use openvmm_defs::config::DEFAULT_MMIO_GAPS_X86;
use openvmm_defs::config::DEFAULT_MMIO_GAPS_X86_WITH_VTL2;
use openvmm_defs::config::DEFAULT_PCAT_BOOT_ORDER;
use openvmm_defs::config::DeviceVtl;
use openvmm_defs::config::HypervisorConfig;
use openvmm_defs::config::LateMapVtl0MemoryPolicy;
use openvmm_defs::config::LoadMode;
use openvmm_defs::config::ProcessorTopologyConfig;
use openvmm_defs::config::SerialInformation;
use openvmm_defs::config::VmbusConfig;
use openvmm_defs::config::VpciDeviceConfig;
use openvmm_defs::config::Vtl2BaseAddressType;
use openvmm_defs::config::Vtl2Config;
use openvmm_pcat_locator::RomFileLocation;
use pal_async::DefaultDriver;
use pal_async::socket::PolledSocket;
use pal_async::task::Spawn;
use pal_async::task::Task;
use petri_artifacts_common::tags::MachineArch;
use petri_artifacts_core::ResolvedArtifact;
use pipette_client::PIPETTE_VSOCK_PORT;
use scsidisk_resources::SimpleScsiDiskHandle;
use scsidisk_resources::SimpleScsiDvdHandle;
use serial_16550_resources::ComPort;
use serial_core::resources::DisconnectedSerialBackendHandle;
use serial_socket::net::OpenSocketSerialConfig;
use sparse_mmap::alloc_shared_memory;
use std::collections::HashMap;
use storvsp_resources::ScsiControllerHandle;
use storvsp_resources::ScsiDeviceAndPath;
use storvsp_resources::ScsiPath;
use tempfile::TempPath;
use tpm_resources::TpmDeviceHandle;
use tpm_resources::TpmRegisterLayout;
use uidevices_resources::SynthVideoHandle;
use unix_socket::UnixListener;
use unix_socket::UnixStream;
use video_core::SharedFramebufferHandle;
use virtio_resources::VirtioPciDeviceHandle;
use virtio_resources::blk::VirtioBlkHandle;
use vm_manifest_builder::VmChipsetResult;
use vm_manifest_builder::VmManifestBuilder;
use vm_resource::IntoResource;
use vm_resource::Resource;
use vm_resource::kind::SerialBackendHandle;
use vm_resource::kind::VmbusDeviceHandleKind;
use vmbus_serial_resources::VmbusSerialDeviceHandle;
use vmbus_serial_resources::VmbusSerialPort;
use vmcore::non_volatile_store::resources::EphemeralNonVolatileStoreHandle;
use vmgs_resources::GuestStateEncryptionPolicy;
use vmgs_resources::VmgsFileHandle;
use vmotherboard::ChipsetDeviceHandle;

impl PetriVmConfigOpenVmm {
    /// Create a new VM configuration.
    pub async fn new(
        openvmm_path: &ResolvedArtifact,
        petri_vm_config: PetriVmConfig,
        resources: &PetriVmResources,
        properties: PetriVmProperties,
    ) -> anyhow::Result<Self> {
        let PetriVmConfig {
            name: _,
            arch,
            host_log_levels,
            firmware,
            memory,
            proc_topology,
            vmgs,
            tpm: tpm_config,
            vmbus_storage_controllers,
        } = petri_vm_config;

        tracing::debug!(?firmware, ?arch, "Petri VM firmware configuration");

        let PetriVmResources { driver, log_source } = resources;

        let mesh = Mesh::new("petri_mesh".to_string())?;

        let setup = PetriVmConfigSetupCore {
            arch,
            firmware: &firmware,
            driver,
            logger: log_source,
            vmgs: &vmgs,
            tpm_config: tpm_config.as_ref(),
            mesh: &mesh,
            openvmm_path,
            uses_pipette_as_init: properties.uses_pipette_as_init,
            enable_serial: properties.enable_serial,
        };

        let mut chipset = VmManifestBuilder::new(
            match firmware {
                Firmware::LinuxDirect { .. } => {
                    vm_manifest_builder::BaseChipsetType::HyperVGen2LinuxDirect
                }
                Firmware::OpenhclLinuxDirect { .. } => {
                    vm_manifest_builder::BaseChipsetType::HclHost
                }
                Firmware::OpenhclUefi { .. } => vm_manifest_builder::BaseChipsetType::HclHost,
                Firmware::Pcat { .. } => vm_manifest_builder::BaseChipsetType::HypervGen1,
                Firmware::Uefi { .. } => vm_manifest_builder::BaseChipsetType::HypervGen2Uefi,
                Firmware::OpenhclPcat { .. } => todo!("OpenVMM OpenHCL PCAT"),
            },
            match arch {
                MachineArch::X86_64 => vm_manifest_builder::MachineArch::X86_64,
                MachineArch::Aarch64 => vm_manifest_builder::MachineArch::Aarch64,
            },
        );

        let mut load_mode = setup.load_firmware()?;

        // If using pipette-as-init, replace the initrd with the pre-built
        // one that has pipette injected. run_core() guarantees that
        // prebuilt_initrd is set when uses_pipette_as_init is true.
        if properties.uses_pipette_as_init {
            if let LoadMode::Linux { initrd, .. } = &mut load_mode {
                let prebuilt = properties
                    .prebuilt_initrd
                    .as_ref()
                    .expect("uses_pipette_as_init requires prebuilt_initrd");
                let file = std::fs::File::open(prebuilt).with_context(|| {
                    format!("failed to open prebuilt initrd at {}", prebuilt.display())
                })?;
                *initrd = Some(file);
            }
        }

        let (emulated_serial_config, log_stream_tasks, linux_direct_serial_agent) =
            if !properties.enable_serial {
                // No emulated serial backends (OpenHCL VMBus serial stubs may still exist)
                ([None, None, None, None], Vec::new(), None)
            } else {
                let SerialData {
                    emulated_serial_config,
                    serial_tasks,
                    linux_direct_serial_agent,
                } = setup.configure_serial(log_source)?;
                (
                    emulated_serial_config,
                    serial_tasks,
                    linux_direct_serial_agent,
                )
            };
        let mut emulated_serial_config = emulated_serial_config;

        let (video_dev, framebuffer, framebuffer_view) = match setup.config_video()? {
            Some((v, fb, fba)) => {
                chipset = chipset.with_framebuffer();
                (Some(v), Some(fb), Some(fba.view()?))
            }
            None => (None, None, None),
        };

        let ide_disks = ide_controllers_to_openvmm(firmware.ide_controllers())?;
        let (mut vmbus_devices, vpci_devices) =
            vmbus_storage_controllers_to_openvmm(&vmbus_storage_controllers)?;

        let (firmware_event_send, firmware_event_recv) = mesh::mpsc_channel();

        let make_vsock_listener = || -> anyhow::Result<(UnixListener, TempPath)> {
            Ok(tempfile::Builder::new()
                .make(|path| UnixListener::bind(path))?
                .into_parts())
        };

        let (with_vtl2, vtl2_vmbus, ged, ged_send, vtl2_vsock_path) = if firmware.is_openhcl() {
            let (ged, ged_send) = setup.config_openhcl_vmbus_devices(
                &mut emulated_serial_config,
                &mut vmbus_devices,
                &firmware_event_send,
                framebuffer.is_some(),
            )?;

            let late_map_vtl0_memory = match load_mode {
                LoadMode::Igvm {
                    vtl2_base_address: Vtl2BaseAddressType::Vtl2Allocate { .. },
                    ..
                } => {
                    // Late Map VTL0 memory not supported when test supplies Vtl2Allocate
                    None
                }
                _ => Some(LateMapVtl0MemoryPolicy::InjectException),
            };

            let (vtl2_vsock_listener, vtl2_vsock_path) = make_vsock_listener()?;
            (
                Some(Vtl2Config {
                    vtl0_alias_map: false, // TODO: enable when OpenVMM supports it for DMA
                    late_map_vtl0_memory,
                }),
                Some(VmbusConfig {
                    vsock_listener: Some(vtl2_vsock_listener),
                    vsock_path: Some(vtl2_vsock_path.to_string_lossy().into_owned()),
                    vmbus_max_version: None,
                    vtl2_redirect: false,
                    #[cfg(windows)]
                    vmbusproxy_handle: None,
                }),
                Some(ged),
                Some(ged_send),
                Some(vtl2_vsock_path),
            )
        } else {
            (None, None, None, None, None)
        };

        // Configure the serial ports now that they have been updated by the
        // OpenHCL configuration.
        if properties.enable_serial {
            chipset = chipset.with_serial(emulated_serial_config);
            // Set so that we don't pull serial data until the guest is
            // ready. Otherwise, Linux will drop the input serial data
            // on the floor during boot.
            if matches!(firmware, Firmware::LinuxDirect { .. }) && !properties.uses_pipette_as_init
            {
                chipset = chipset.with_serial_wait_for_rts();
            }
        }

        // Extract video configuration
        let vga_firmware = match video_dev {
            Some(VideoDevice::Vga(firmware)) => Some(firmware),
            Some(VideoDevice::Synth(vtl, resource)) => {
                vmbus_devices.push((vtl, resource));
                None
            }
            None => None,
        };

        // Add default VMBus devices (skipped in minimal mode).
        let (shutdown_ic_send, kvp_ic_send) = if !properties.minimal_mode {
            let (shutdown_ic_send, shutdown_ic_recv) = mesh::channel();
            vmbus_devices.push((
                DeviceVtl::Vtl0,
                ShutdownIcHandle {
                    recv: shutdown_ic_recv,
                }
                .into_resource(),
            ));

            let (kvp_ic_send, kvp_ic_recv) = mesh::channel();
            vmbus_devices.push((
                DeviceVtl::Vtl0,
                hyperv_ic_resources::kvp::KvpIcHandle { recv: kvp_ic_recv }.into_resource(),
            ));

            vmbus_devices.push((
                DeviceVtl::Vtl0,
                hyperv_ic_resources::timesync::TimesyncIcHandle.into_resource(),
            ));

            (shutdown_ic_send, kvp_ic_send)
        } else {
            // Minimal mode: no ICs. Create dummy senders so the fields
            // are populated (calls to send_enlightened_shutdown will fail
            // with a channel error, which is fine — minimal VMs shut down
            // via reboot(2) directly).
            let (shutdown_ic_send, _) = mesh::channel();
            let (kvp_ic_send, _) = mesh::channel();
            (shutdown_ic_send, kvp_ic_send)
        };

        // Make a vmbus vsock path for pipette connections
        let (vmbus_vsock_listener, vmbus_vsock_path) = make_vsock_listener()?;

        let chipset = chipset
            .build()
            .context("failed to build chipset configuration")?;

        let memory = {
            let MemoryConfig {
                startup_bytes,
                dynamic_memory_range,
                mmio_gaps,
            } = memory;

            if dynamic_memory_range.is_some() {
                anyhow::bail!("dynamic memory not supported in OpenVMM");
            }

            openvmm_defs::config::MemoryConfig {
                mem_size: startup_bytes,
                mmio_gaps: match mmio_gaps {
                    MmioConfig::Platform => {
                        if firmware.is_openhcl() {
                            match arch {
                                MachineArch::X86_64 => DEFAULT_MMIO_GAPS_X86_WITH_VTL2.into(),
                                MachineArch::Aarch64 => DEFAULT_MMIO_GAPS_AARCH64_WITH_VTL2.into(),
                            }
                        } else {
                            match arch {
                                MachineArch::X86_64 => DEFAULT_MMIO_GAPS_X86.into(),
                                MachineArch::Aarch64 => DEFAULT_MMIO_GAPS_AARCH64.into(),
                            }
                        }
                    }
                    MmioConfig::Custom(ranges) => ranges,
                },
                pci_ecam_gaps: vec![],
                pci_mmio_gaps: vec![],
                prefetch_memory: false,
                private_memory: false,
                transparent_hugepages: false,
            }
        };

        let processor_topology = {
            let ProcessorTopology {
                vp_count,
                enable_smt,
                vps_per_socket,
                apic_mode,
            } = proc_topology;

            ProcessorTopologyConfig {
                proc_count: vp_count,
                vps_per_socket,
                enable_smt,
                arch: Some(match arch {
                    MachineArch::X86_64 => openvmm_defs::config::ArchTopologyConfig::X86(
                        openvmm_defs::config::X86TopologyConfig {
                            x2apic: match apic_mode {
                                None => openvmm_defs::config::X2ApicConfig::Auto,
                                Some(x) => match x {
                                    crate::ApicMode::Xapic => {
                                        openvmm_defs::config::X2ApicConfig::Unsupported
                                    }
                                    crate::ApicMode::X2apicSupported => {
                                        openvmm_defs::config::X2ApicConfig::Supported
                                    }
                                    crate::ApicMode::X2apicEnabled => {
                                        openvmm_defs::config::X2ApicConfig::Enabled
                                    }
                                },
                            },
                            ..Default::default()
                        },
                    ),
                    MachineArch::Aarch64 => openvmm_defs::config::ArchTopologyConfig::Aarch64(
                        openvmm_defs::config::Aarch64TopologyConfig::default(),
                    ),
                }),
            }
        };

        let (secure_boot_enabled, custom_uefi_vars) = firmware.uefi_config().map_or_else(
            || (false, Default::default()),
            |c| {
                (
                    c.secure_boot_enabled,
                    match (arch, c.secure_boot_template) {
                        (MachineArch::X86_64, Some(SecureBootTemplate::MicrosoftWindows)) => {
                            hyperv_secure_boot_templates::x64::microsoft_windows()
                        }
                        (
                            MachineArch::X86_64,
                            Some(SecureBootTemplate::MicrosoftUefiCertificateAuthority),
                        ) => hyperv_secure_boot_templates::x64::microsoft_uefi_ca(),
                        (MachineArch::Aarch64, Some(SecureBootTemplate::MicrosoftWindows)) => {
                            hyperv_secure_boot_templates::aarch64::microsoft_windows()
                        }
                        (
                            MachineArch::Aarch64,
                            Some(SecureBootTemplate::MicrosoftUefiCertificateAuthority),
                        ) => hyperv_secure_boot_templates::aarch64::microsoft_uefi_ca(),
                        (_, None) => Default::default(),
                    },
                )
            },
        );

        let vmgs = if firmware.is_openhcl() {
            None
        } else {
            Some(memdiff_vmgs(&vmgs)?)
        };

        let VmChipsetResult {
            chipset,
            mut chipset_devices,
        } = chipset;

        // Add the TPM
        if let Some(tpm) = setup.config_tpm().await? {
            chipset_devices.push(tpm);
        }

        let config = Config {
            // Firmware
            load_mode,
            firmware_event_send: Some(firmware_event_send),

            // CPU and RAM
            memory,
            processor_topology,

            // Base chipset
            chipset,
            chipset_devices,

            // Basic virtualization device support
            hypervisor: HypervisorConfig {
                with_hv: true,
                user_mode_hv_enlightenments: false,
                user_mode_apic: false,
                with_vtl2,
                with_isolation: match firmware.isolation() {
                    Some(IsolationType::Vbs) => Some(openvmm_defs::config::IsolationType::Vbs),
                    None => None,
                    _ => anyhow::bail!("unsupported isolation type"),
                },
            },
            vmbus: Some(VmbusConfig {
                vsock_listener: Some(vmbus_vsock_listener),
                vsock_path: Some(vmbus_vsock_path.to_string_lossy().into_owned()),
                vmbus_max_version: None,
                vtl2_redirect: firmware.openhcl_config().is_some_and(|c| c.vmbus_redirect),
                #[cfg(windows)]
                vmbusproxy_handle: None,
            }),
            vtl2_vmbus,

            // Devices
            floppy_disks: vec![],
            ide_disks,
            pcie_root_complexes: vec![],
            pcie_devices: vec![],
            pcie_switches: vec![],
            vpci_devices,
            vmbus_devices,

            // Video support
            framebuffer,
            vga_firmware,

            secure_boot_enabled,
            custom_uefi_vars,
            vmgs,

            // Don't automatically reset the guest by default
            automatic_guest_reset: false,

            // Disabled for VMM tests by default
            #[cfg(windows)]
            kernel_vmnics: vec![],
            input: mesh::Receiver::new(),
            vtl2_gfx: false,
            virtio_devices: vec![],
            #[cfg(windows)]
            vpci_resources: vec![],
            debugger_rpc: None,
            generation_id_recv: None,
            rtc_delta_milliseconds: 0,
            efi_diagnostics_log_level: Default::default(), // TODO: Add config for tests
        };

        // Make the pipette connection listener.
        let path = config.vmbus.as_ref().unwrap().vsock_path.as_ref().unwrap();
        let path = format!("{path}_{PIPETTE_VSOCK_PORT}");
        let pipette_listener = PolledSocket::new(
            driver,
            UnixListener::bind(path).context("failed to bind to pipette listener")?,
        )?;

        // Make the vtl2 pipette connection listener.
        let vtl2_pipette_listener = if let Some(vtl2_vmbus) = &config.vtl2_vmbus {
            let path = vtl2_vmbus.vsock_path.as_ref().unwrap();
            let path = format!("{path}_{PIPETTE_VSOCK_PORT}");
            Some(PolledSocket::new(
                driver,
                UnixListener::bind(path).context("failed to bind to vtl2 pipette listener")?,
            )?)
        } else {
            None
        };

        Ok(Self {
            runtime_config: firmware.into_runtime_config(vmbus_storage_controllers),
            arch,
            host_log_levels,
            config,
            mesh,

            resources: PetriVmResourcesOpenVmm {
                log_stream_tasks,
                firmware_event_recv,
                shutdown_ic_send,
                kvp_ic_send,
                ged_send,
                pipette_listener,
                vtl2_pipette_listener,
                linux_direct_serial_agent,
                driver: driver.clone(),
                output_dir: log_source.output_dir().to_owned(),
                openvmm_path: openvmm_path.clone(),
                vtl2_vsock_path,
                _vmbus_vsock_path: vmbus_vsock_path,
                properties,
            },

            openvmm_log_file: log_source.log_file("openvmm")?,

            memory_backing_file: None,

            ged,
            framebuffer_view,
        })
    }
}

struct PetriVmConfigSetupCore<'a> {
    arch: MachineArch,
    firmware: &'a Firmware,
    driver: &'a DefaultDriver,
    logger: &'a PetriLogSource,
    vmgs: &'a PetriVmgsResource,
    tpm_config: Option<&'a TpmConfig>,
    mesh: &'a Mesh,
    openvmm_path: &'a ResolvedArtifact,
    uses_pipette_as_init: bool,
    enable_serial: bool,
}

struct SerialData {
    emulated_serial_config: [Option<Resource<SerialBackendHandle>>; 4],
    serial_tasks: Vec<Task<anyhow::Result<()>>>,
    linux_direct_serial_agent: Option<LinuxDirectSerialAgent>,
}

enum VideoDevice {
    Vga(RomFileLocation),
    Synth(DeviceVtl, Resource<VmbusDeviceHandleKind>),
}

impl PetriVmConfigSetupCore<'_> {
    fn configure_serial(&self, logger: &PetriLogSource) -> anyhow::Result<SerialData> {
        let mut serial_tasks = Vec::new();

        let serial0_log_file = logger.log_file(match self.firmware {
            Firmware::LinuxDirect { .. } | Firmware::OpenhclLinuxDirect { .. } => "linux",
            Firmware::Pcat { .. } | Firmware::OpenhclPcat { .. } => "pcat",
            Firmware::Uefi { .. } | Firmware::OpenhclUefi { .. } => "uefi",
        })?;

        let (serial0_host, serial0) = self
            .create_serial_stream()
            .context("failed to create serial0 stream")?;
        let (serial0_read, serial0_write) = serial0_host.split();
        let serial0_task = self.driver.spawn(
            "serial0-console",
            crate::log_task(serial0_log_file, serial0_read, "serial0-console"),
        );
        serial_tasks.push(serial0_task);

        let serial2 = if self.firmware.is_openhcl() {
            let (serial2_host, serial2) = self
                .create_serial_stream()
                .context("failed to create serial2 stream")?;
            let serial2_task = self.driver.spawn(
                "serial2-openhcl",
                crate::log_task(logger.log_file("openhcl")?, serial2_host, "serial2-openhcl"),
            );
            serial_tasks.push(serial2_task);
            serial2
        } else {
            None
        };

        if self.firmware.is_linux_direct() && !self.uses_pipette_as_init {
            // Non-pipette-as-init Linux direct: create serial1 and a serial
            // agent so we can send shell commands to launch pipette.
            let (serial1_host, serial1) = self.create_serial_stream()?;
            let (serial1_read, _serial1_write) = serial1_host.split();
            let linux_direct_serial_agent =
                LinuxDirectSerialAgent::new(serial1_read, serial0_write);
            Ok(SerialData {
                emulated_serial_config: [serial0, serial1, serial2, None],
                serial_tasks,
                linux_direct_serial_agent: Some(linux_direct_serial_agent),
            })
        } else {
            Ok(SerialData {
                emulated_serial_config: [serial0, None, serial2, None],
                serial_tasks,
                linux_direct_serial_agent: None,
            })
        }
    }

    fn create_serial_stream(
        &self,
    ) -> anyhow::Result<(
        PolledSocket<UnixStream>,
        Option<Resource<SerialBackendHandle>>,
    )> {
        let (host_side, guest_side) = UnixStream::pair()?;
        let host_side = PolledSocket::new(self.driver, host_side)?;
        let serial = OpenSocketSerialConfig::from(guest_side).into_resource();
        Ok((host_side, Some(serial)))
    }

    fn load_firmware(&self) -> anyhow::Result<LoadMode> {
        // The test kernel has both CONFIG_VIRTIO_VSOCK=y and
        // CONFIG_HYPERV_VSOCKETS=y built in. The kernel only allows one G2H
        // vsock transport, and virtio_vsock_init runs first, claiming the
        // slot. This causes hv_sock registration to fail with -EBUSY,
        // breaking pipette's AF_VSOCK connection. Blacklist virtio_vsock_init
        // so that hv_sock can register as the G2H transport.
        const VIRTIO_VSOCK_BLACKLIST: &str = "initcall_blacklist=virtio_vsock_init";

        Ok(match (self.arch, &self.firmware) {
            (arch, Firmware::LinuxDirect { kernel, initrd }) => {
                let console = match arch {
                    MachineArch::X86_64 => "console=ttyS0",
                    MachineArch::Aarch64 => "console=ttyAMA0 earlycon",
                };
                let kernel = File::open(kernel.clone())
                    .context("Failed to open kernel")?
                    .into();
                let initrd = File::open(initrd.clone())
                    .context("Failed to open initrd")?
                    .into();

                let init = if self.uses_pipette_as_init {
                    "/pipette"
                } else {
                    "/bin/sh"
                };

                let serial_args = if self.enable_serial {
                    format!("{console} debug ")
                } else {
                    String::new()
                };

                let cmdline =
                    format!("{serial_args}panic=-1 rdinit={init} {VIRTIO_VSOCK_BLACKLIST}");

                LoadMode::Linux {
                    kernel,
                    initrd: Some(initrd),
                    cmdline,
                    custom_dsdt: None,
                    enable_serial: self.enable_serial,
                    boot_mode: openvmm_defs::config::LinuxDirectBootMode::Acpi,
                }
            }
            (
                MachineArch::X86_64,
                Firmware::Pcat {
                    bios_firmware: firmware,
                    guest: _,         // load_boot_disk
                    svga_firmware: _, // config_video
                    ide_controllers: _,
                },
            ) => {
                let firmware = openvmm_pcat_locator::find_pcat_bios(firmware.get())
                    .context("Failed to load packaged PCAT binary")?;
                LoadMode::Pcat {
                    firmware,
                    boot_order: DEFAULT_PCAT_BOOT_ORDER,
                }
            }
            (
                _,
                Firmware::Uefi {
                    uefi_firmware: firmware,
                    guest: _, // load_boot_disk
                    uefi_config:
                        UefiConfig {
                            secure_boot_enabled: _,  // new
                            secure_boot_template: _, // new
                            disable_frontpage,
                            default_boot_always_attempt,
                            enable_vpci_boot,
                        },
                },
            ) => {
                let firmware = File::open(firmware.clone())
                    .context("Failed to open uefi firmware file")?
                    .into();
                LoadMode::Uefi {
                    firmware,
                    enable_debugging: false,
                    enable_memory_protections: false,
                    disable_frontpage: *disable_frontpage,
                    enable_tpm: self.tpm_config.is_some(),
                    enable_battery: false,
                    enable_serial: true,
                    enable_vpci_boot: *enable_vpci_boot,
                    uefi_console_mode: Some(openvmm_defs::config::UefiConsoleMode::Com1),
                    default_boot_always_attempt: *default_boot_always_attempt,
                    bios_guid: Guid::new_random(),
                }
            }
            (
                MachineArch::X86_64,
                Firmware::OpenhclLinuxDirect {
                    igvm_path,
                    openhcl_config,
                }
                | Firmware::OpenhclUefi {
                    igvm_path,
                    guest: _,       // load_boot_disk
                    isolation: _,   // new via Firmware::isolation
                    uefi_config: _, // config_openhcl_vmbus_devices
                    openhcl_config,
                },
            ) => {
                let OpenHclConfig {
                    vmbus_redirect: _, // config_openhcl_vmbus_devices
                    custom_command_line: _,
                    log_levels: _,
                    vtl2_base_address_type,
                    vtl2_settings: _, // run_core
                } = openhcl_config;

                let mut cmdline = Some(openhcl_config.command_line());

                append_cmdline(&mut cmdline, "panic=-1 reboot=triple");

                let isolated = match self.firmware {
                    Firmware::OpenhclLinuxDirect { .. } => {
                        // Set UNDERHILL_SERIAL_WAIT_FOR_RTS=1 so that we don't pull serial data
                        // until the guest is ready. Otherwise, Linux will drop the input serial
                        // data on the floor during boot.
                        append_cmdline(
                            &mut cmdline,
                            format!(
                                "UNDERHILL_SERIAL_WAIT_FOR_RTS=1 UNDERHILL_CMDLINE_APPEND=\"rdinit=/bin/sh {VIRTIO_VSOCK_BLACKLIST}\""
                            ),
                        );
                        false
                    }
                    Firmware::OpenhclUefi { isolation, .. } if isolation.is_some() => true,
                    _ => false,
                };

                // For certain configurations, we need to override the override
                // in new_underhill_vm.
                //
                // TODO: remove this (and OpenHCL override) once host changes
                // are saturated.
                if let Firmware::OpenhclUefi {
                    uefi_config:
                        UefiConfig {
                            default_boot_always_attempt,
                            secure_boot_enabled,
                            ..
                        },
                    ..
                } = self.firmware
                {
                    if !isolated
                        && !secure_boot_enabled
                        && self.tpm_config.is_none()
                        && !default_boot_always_attempt
                    {
                        append_cmdline(&mut cmdline, "HCL_DEFAULT_BOOT_ALWAYS_ATTEMPT=0");
                    }
                }

                let vtl2_base_address = vtl2_base_address_type.unwrap_or_else(|| {
                    if isolated {
                        // Isolated VMs must load at the location specified by
                        // the file, as they do not support relocation.
                        Vtl2BaseAddressType::File
                    } else {
                        // By default, utilize IGVM relocation and tell OpenVMM
                        // to place VTL2 at 512MB. This tests both relocation
                        // support in OpenVMM, and relocation support within
                        // OpenHCL.
                        Vtl2BaseAddressType::Absolute(512 * SIZE_1_MB)
                    }
                });

                let file = File::open(igvm_path.clone())
                    .context("failed to open openhcl firmware file")?
                    .into();
                LoadMode::Igvm {
                    file,
                    cmdline: cmdline.unwrap_or_default(),
                    vtl2_base_address,
                    com_serial: Some(SerialInformation {
                        io_port: ComPort::Com3.io_port(),
                        irq: ComPort::Com3.irq().into(),
                    }),
                }
            }
            (a, f) => anyhow::bail!("Unsupported firmware {f:?} for arch {a:?}"),
        })
    }

    fn config_openhcl_vmbus_devices(
        &self,
        serial: &mut [Option<Resource<SerialBackendHandle>>],
        devices: &mut impl Extend<(DeviceVtl, Resource<VmbusDeviceHandleKind>)>,
        firmware_event_send: &mesh::Sender<FirmwareEvent>,
        framebuffer: bool,
    ) -> anyhow::Result<(
        get_resources::ged::GuestEmulationDeviceHandle,
        mesh::Sender<get_resources::ged::GuestEmulationRequest>,
    )> {
        let serial0 = serial[0].take();
        devices.extend([(
            DeviceVtl::Vtl2,
            VmbusSerialDeviceHandle {
                port: VmbusSerialPort::Com1,
                backend: serial0.unwrap_or_else(|| DisconnectedSerialBackendHandle.into_resource()),
            }
            .into_resource(),
        )]);
        let serial1 = serial[1].take();
        devices.extend([(
            DeviceVtl::Vtl2,
            VmbusSerialDeviceHandle {
                port: VmbusSerialPort::Com2,
                backend: serial1.unwrap_or_else(|| DisconnectedSerialBackendHandle.into_resource()),
            }
            .into_resource(),
        )]);

        let crash = spawn_dump_handler(self.driver, self.logger).into_resource();
        devices.extend([(DeviceVtl::Vtl2, crash)]);

        let (guest_request_send, guest_request_recv) = mesh::channel();

        let (
            UefiConfig {
                secure_boot_enabled,
                secure_boot_template,
                disable_frontpage,
                default_boot_always_attempt,
                enable_vpci_boot,
            },
            OpenHclConfig { vmbus_redirect, .. },
        ) = match self.firmware {
            Firmware::OpenhclUefi {
                uefi_config,
                openhcl_config,
                ..
            } => (uefi_config, openhcl_config),
            Firmware::OpenhclLinuxDirect { openhcl_config, .. } => {
                (&UefiConfig::default(), openhcl_config)
            }
            _ => anyhow::bail!("not a supported openhcl firmware config"),
        };

        let test_gsp_by_id = self
            .vmgs
            .disk()
            .is_some_and(|x| matches!(x.encryption_policy, GuestStateEncryptionPolicy::GspById(_)));

        // Save the GED handle to add later after configuration is complete.
        let ged = get_resources::ged::GuestEmulationDeviceHandle {
            firmware: get_resources::ged::GuestFirmwareConfig::Uefi {
                firmware_debug: false,
                disable_frontpage: *disable_frontpage,
                enable_vpci_boot: *enable_vpci_boot,
                console_mode: get_resources::ged::UefiConsoleMode::COM1,
                default_boot_always_attempt: *default_boot_always_attempt,
            },
            com1: true,
            com2: true,
            serial_tx_only: false,
            vmbus_redirection: *vmbus_redirect,
            vtl2_settings: None, // Will be added at startup to allow tests to modify
            vmgs: memdiff_vmgs(self.vmgs)?,
            framebuffer: framebuffer.then(|| SharedFramebufferHandle.into_resource()),
            guest_request_recv,
            enable_tpm: self.tpm_config.is_some(),
            firmware_event_send: Some(firmware_event_send.clone()),
            secure_boot_enabled: *secure_boot_enabled,
            secure_boot_template: match secure_boot_template {
                Some(SecureBootTemplate::MicrosoftWindows) => {
                    get_resources::ged::GuestSecureBootTemplateType::MicrosoftWindows
                }
                Some(SecureBootTemplate::MicrosoftUefiCertificateAuthority) => {
                    get_resources::ged::GuestSecureBootTemplateType::MicrosoftUefiCertificateAuthority
                }
                None => get_resources::ged::GuestSecureBootTemplateType::None,
            },
            enable_battery: false,
            no_persistent_secrets: self.tpm_config.as_ref().is_some_and(|c| c.no_persistent_secrets),
            igvm_attest_test_config: None,
            test_gsp_by_id,
            efi_diagnostics_log_level: Default::default(), // TODO: make configurable
            hv_sint_enabled: false,
        };

        Ok((ged, guest_request_send))
    }

    fn config_video(
        &self,
    ) -> anyhow::Result<Option<(VideoDevice, Framebuffer, FramebufferAccess)>> {
        if self.firmware.isolation().is_some() {
            return Ok(None);
        }

        let video_dev = match self.firmware {
            Firmware::Pcat { svga_firmware, .. } | Firmware::OpenhclPcat { svga_firmware, .. } => {
                Some(VideoDevice::Vga(
                    openvmm_pcat_locator::find_svga_bios(svga_firmware.get())
                        .context("Failed to load VGA BIOS")?,
                ))
            }
            Firmware::Uefi { .. } | Firmware::OpenhclUefi { .. } => Some(VideoDevice::Synth(
                DeviceVtl::Vtl0,
                SynthVideoHandle {
                    framebuffer: SharedFramebufferHandle.into_resource(),
                }
                .into_resource(),
            )),
            Firmware::OpenhclLinuxDirect { .. } | Firmware::LinuxDirect { .. } => None,
        };

        Ok(if let Some(vdev) = video_dev {
            let vram =
                alloc_shared_memory(FRAMEBUFFER_SIZE, "vram").context("allocating framebuffer")?;
            let (fb, fba) = framebuffer::framebuffer(vram, FRAMEBUFFER_SIZE, 0)
                .context("creating framebuffer")?;
            Some((vdev, fb, fba))
        } else {
            None
        })
    }

    async fn config_tpm(&self) -> anyhow::Result<Option<ChipsetDeviceHandle>> {
        if !self.firmware.is_openhcl()
            && let Some(TpmConfig {
                no_persistent_secrets,
            }) = self.tpm_config
        {
            let register_layout = match self.arch {
                MachineArch::X86_64 => TpmRegisterLayout::IoPort,
                MachineArch::Aarch64 => TpmRegisterLayout::Mmio,
            };

            let (ppi_store, nvram_store) = if self.vmgs.disk().is_none() || *no_persistent_secrets {
                (
                    EphemeralNonVolatileStoreHandle.into_resource(),
                    EphemeralNonVolatileStoreHandle.into_resource(),
                )
            } else {
                (
                    VmgsFileHandle::new(vmgs_format::FileId::TPM_PPI, true).into_resource(),
                    VmgsFileHandle::new(vmgs_format::FileId::TPM_NVRAM, true).into_resource(),
                )
            };

            Ok(Some(ChipsetDeviceHandle {
                name: "tpm".to_string(),
                resource: chipset_device_worker_defs::RemoteChipsetDeviceHandle {
                    device: TpmDeviceHandle {
                        ppi_store,
                        nvram_store,
                        refresh_tpm_seeds: false,
                        ak_cert_type: tpm_resources::TpmAkCertTypeResource::None,
                        register_layout,
                        guest_secret_key: None,
                        logger: None,
                        is_confidential_vm: self.firmware.isolation().is_some(),
                        // TODO: generate an actual BIOS GUID and put it here
                        bios_guid: Guid::ZERO,
                        nvram_size: None,
                    }
                    .into_resource(),
                    worker_host: self.make_device_worker("tpm").await?,
                }
                .into_resource(),
            }))
        } else {
            Ok(None)
        }
    }

    async fn make_device_worker(&self, name: &str) -> anyhow::Result<mesh_worker::WorkerHost> {
        let (host, runner) = mesh_worker::worker_host();
        self.mesh
            .launch_host(
                mesh_process::ProcessConfig::new(name).process_name(self.openvmm_path),
                openvmm_defs::entrypoint::MeshHostParams { runner },
            )
            .await?;
        Ok(host)
    }
}

fn spawn_dump_handler(driver: &DefaultDriver, logger: &PetriLogSource) -> GuestCrashDeviceHandle {
    let (send, mut recv) = mesh::channel();
    let handle = GuestCrashDeviceHandle {
        request_dump: send,
        max_dump_size: 256 * 1024 * 1024,
    };
    driver
        .spawn("openhcl-dump-handler", {
            let logger = logger.clone();
            let driver = driver.clone();
            async move {
                while let Some(rpc) = recv.next().await {
                    rpc.handle_failable_sync(|done| {
                        let (file, path) = logger.create_attachment("openhcl.core")?.into_parts();
                        driver
                            .spawn("crash-waiter", async move {
                                let filename = path.file_name().unwrap().to_str().unwrap();
                                if done.await.is_ok() {
                                    tracing::warn!(filename, "openhcl crash dump complete");
                                } else {
                                    tracing::error!(
                                        filename,
                                        "openhcl crash dump incomplete, may be corrupted"
                                    );
                                }
                            })
                            .detach();
                        anyhow::Ok(file)
                    })
                }
            }
        })
        .detach();
    handle
}

/// Convert the generic IDE configuration to OpenVMM IDE disks.
fn ide_controllers_to_openvmm(
    ide_controllers: Option<&[[Option<Drive>; 2]; 2]>,
) -> anyhow::Result<Vec<IdeDeviceConfig>> {
    let mut ide_disks = Vec::new();

    if let Some(ide_controllers) = ide_controllers {
        for (controller_number, controller) in ide_controllers.iter().enumerate() {
            for (controller_location, drive) in controller.iter().enumerate() {
                if let Some(drive) = drive {
                    if let Some(disk) = &drive.disk {
                        let disk = petri_disk_to_openvmm(disk)?;
                        let guest_media = if drive.is_dvd {
                            GuestMedia::Dvd(
                                SimpleScsiDvdHandle {
                                    media: Some(disk),
                                    requests: None,
                                }
                                .into_resource(),
                            )
                        } else {
                            GuestMedia::Disk {
                                disk_type: disk,
                                read_only: false,
                                disk_parameters: None,
                            }
                        };

                        ide_disks.push(IdeDeviceConfig {
                            path: ide_resources::IdePath {
                                channel: controller_number as u8,
                                drive: controller_location as u8,
                            },
                            guest_media,
                        });
                    }
                }
            }
        }
    }

    Ok(ide_disks)
}

/// Convert the generic VMBUS storage configuration to OpenVMM VMBUS and VPCI devices.
fn vmbus_storage_controllers_to_openvmm(
    vmbus_storage_controllers: &HashMap<Guid, VmbusStorageController>,
) -> anyhow::Result<(
    Vec<(DeviceVtl, Resource<VmbusDeviceHandleKind>)>,
    Vec<VpciDeviceConfig>,
)> {
    let mut vmbus_devices = Vec::new();
    let mut vpci_devices = Vec::new();

    // Add VMBus storage
    for (instance_id, controller) in vmbus_storage_controllers {
        let vtl = match controller.target_vtl {
            crate::Vtl::Vtl0 => DeviceVtl::Vtl0,
            crate::Vtl::Vtl1 => DeviceVtl::Vtl1,
            crate::Vtl::Vtl2 => DeviceVtl::Vtl2,
        };
        match controller.controller_type {
            VmbusStorageType::Scsi => {
                let mut devices = Vec::new();
                for (lun, Drive { disk, is_dvd }) in &controller.drives {
                    if !*is_dvd && let Some(disk) = disk {
                        devices.push(ScsiDeviceAndPath {
                            path: ScsiPath {
                                path: 0,
                                target: 0,
                                lun: (*lun).try_into().expect("invalid scsi lun"),
                            },
                            device: SimpleScsiDiskHandle {
                                disk: petri_disk_to_openvmm(disk)?,
                                read_only: false,
                                parameters: Default::default(),
                            }
                            .into_resource(),
                        });
                    } else {
                        todo!("dvd ({}) or empty ({})", *is_dvd, disk.is_none())
                    }
                }

                vmbus_devices.push((
                    vtl,
                    ScsiControllerHandle {
                        instance_id: *instance_id,
                        max_sub_channel_count: 1,
                        io_queue_depth: None,
                        devices,
                        requests: None,
                        poll_mode_queue_depth: None,
                    }
                    .into_resource(),
                ));
            }
            VmbusStorageType::Nvme => {
                let mut namespaces = Vec::new();
                for (nsid, Drive { disk, is_dvd }) in &controller.drives {
                    if !*is_dvd && let Some(disk) = disk {
                        namespaces.push(NamespaceDefinition {
                            nsid: *nsid,
                            read_only: false,
                            disk: petri_disk_to_openvmm(disk)?,
                        });
                    } else {
                        todo!("dvd ({}) or empty ({})", *is_dvd, disk.is_none())
                    }
                }

                vpci_devices.push(VpciDeviceConfig {
                    vtl,
                    instance_id: *instance_id,
                    resource: NvmeControllerHandle {
                        subsystem_id: *instance_id,
                        max_io_queues: 64,
                        msix_count: 64,
                        namespaces,
                        requests: None,
                    }
                    .into_resource(),
                });
            }
            VmbusStorageType::VirtioBlk => {
                // Each virtio-blk drive needs a unique VPCI instance ID.
                // Use a fixed template GUID with data1 set to the LUN.
                const VIRTIO_BLK_INSTANCE_ID_TEMPLATE: Guid = Guid {
                    data1: 0,
                    data2: 0x1234,
                    data3: 0x5678,
                    data4: [0xab, 0xcd, 0xef, 0x01, 0x23, 0x45, 0x67, 0x89],
                };
                for (lun, Drive { disk, is_dvd }) in &controller.drives {
                    if *is_dvd {
                        anyhow::bail!("dvd not supported with virtio-blk");
                    }
                    let Some(disk) = disk else {
                        anyhow::bail!("empty drive not supported with virtio-blk");
                    };
                    let mut drive_id = VIRTIO_BLK_INSTANCE_ID_TEMPLATE;
                    drive_id.data1 = *lun;
                    vpci_devices.push(VpciDeviceConfig {
                        vtl,
                        instance_id: drive_id,
                        resource: VirtioPciDeviceHandle(
                            VirtioBlkHandle {
                                disk: petri_disk_to_openvmm(disk)?,
                                read_only: false,
                            }
                            .into_resource(),
                        )
                        .into_resource(),
                    });
                }
            }
        }
    }

    Ok((vmbus_devices, vpci_devices))
}