openvmm_entry/

lib.rs

// Copyright (c) Microsoft Corporation.
// Licensed under the MIT License.

//! This module implements the interactive control process and the entry point
//! for the worker process.

#![expect(missing_docs)]
#![forbid(unsafe_code)]

mod cli_args;
mod crash_dump;
mod kvp;
mod meshworker;
mod pidfile;
mod repl;
mod serial_io;
mod storage_builder;
mod tracing_init;
mod ttrpc;
mod vm_controller;

// `pub` so that the missing_docs warning fires for options without
// documentation.
pub use cli_args::Options;
use console_relay::ConsoleLaunchOptions;

use crate::cli_args::SecureBootTemplateCli;
use anyhow::Context;
use anyhow::bail;
use chipset_resources::battery::HostBatteryUpdate;
use cli_args::DiskCliKind;
use cli_args::EfiDiagnosticsLogLevelCli;
use cli_args::EndpointConfigCli;
use cli_args::IgvmPersonalityCli;
use cli_args::NicConfigCli;
use cli_args::ProvisionVmgs;
use cli_args::SerialConfigCli;
use cli_args::TpmVersionCli;
use cli_args::UefiConsoleModeCli;
use cli_args::VirtioBusCli;
use cli_args::VmgsCli;
use crash_dump::spawn_dump_handler;
use cxl::test::CxlTestDeviceHandle;
use disk_backend_resources::DelayDiskHandle;
use disk_backend_resources::DiskLayerDescription;
use disk_backend_resources::layer::DiskLayerHandle;
use disk_backend_resources::layer::RamDiskLayerHandle;
use disk_backend_resources::layer::SqliteAutoCacheDiskLayerHandle;
use disk_backend_resources::layer::SqliteDiskLayerHandle;
use floppy_resources::FloppyDiskConfig;
use framebuffer::FRAMEBUFFER_SIZE;
use framebuffer::FramebufferAccess;
use futures::AsyncReadExt;
use futures::AsyncWrite;
use futures::StreamExt;
use futures::executor::block_on;
use futures::io::AllowStdIo;
use gdma_resources::GdmaDeviceHandle;
use gdma_resources::VportDefinition;
use guid::Guid;
use input_core::MultiplexedInputHandle;
use inspect::InspectMut;
use mesh::CancelContext;
use mesh::CellUpdater;
use mesh::rpc::RpcSend;
use meshworker::VmmMesh;
use net_backend_resources::mac_address::MacAddress;
use nvme_resources::NvmeControllerRequest;
use openvmm_defs::config::Config;
use openvmm_defs::config::DEFAULT_PCAT_BOOT_ORDER;
use openvmm_defs::config::DeviceVtl;
use openvmm_defs::config::HypervisorConfig;
use openvmm_defs::config::IgvmPersonality;
use openvmm_defs::config::LateMapVtl0MemoryPolicy;
use openvmm_defs::config::LoadMode;
use openvmm_defs::config::MemoryConfig;
use openvmm_defs::config::NumaDistance;
use openvmm_defs::config::NumaNode;
use openvmm_defs::config::NumaTopology;
use openvmm_defs::config::PcieDeviceConfig;
use openvmm_defs::config::PcieMmioRangeConfig;
use openvmm_defs::config::PciePortConfig;
use openvmm_defs::config::PcieRootComplexConfig;
use openvmm_defs::config::PcieSwitchConfig;
use openvmm_defs::config::ProcessorTopologyConfig;
use openvmm_defs::config::RootComplexCxlConfig;
use openvmm_defs::config::UartId;
use openvmm_defs::config::VirtioBus;
use openvmm_defs::config::VmbusConfig;
use openvmm_defs::config::VpAssignment;
use openvmm_defs::config::VpciDeviceConfig;
use openvmm_defs::config::Vtl2BaseAddressType;
use openvmm_defs::config::Vtl2Config;
use openvmm_defs::rpc::VmRpc;
use openvmm_defs::worker::VM_WORKER;
use openvmm_defs::worker::VmWorkerParameters;
use openvmm_helpers::disk::OpenDiskOptions;
use openvmm_helpers::disk::create_disk_type;
use openvmm_helpers::disk::open_disk_type;
use pal_async::DefaultDriver;
use pal_async::DefaultPool;
use pal_async::socket::PolledSocket;
use pal_async::task::Spawn;
use pal_async::task::Task;
use serial_16550_resources::ComPort;
use serial_core::resources::DisconnectedSerialBackendHandle;
use serial_core::resources::NullSerialBackendHandle;
use sparse_mmap::alloc_shared_memory;
use std::cell::RefCell;
use std::collections::BTreeMap;
use std::collections::HashSet;
use std::fmt::Write as _;
use std::io;
#[cfg(unix)]
use std::io::IsTerminal;
use std::io::Write;
use std::net::TcpListener;
use std::path::Path;
use std::path::PathBuf;
use std::sync::Arc;
use std::thread;
use std::time::Duration;
use storvsp_resources::ScsiControllerRequest;
use tpm_resources::TpmDeviceHandle;
use tpm_resources::TpmRegisterLayout;
use tpm_resources::TpmVersion;
use uidevices_resources::SynthKeyboardHandle;
use uidevices_resources::SynthMouseHandle;
use uidevices_resources::SynthVideoHandle;
use video_core::SharedFramebufferHandle;
use virtio_resources::VirtioPciDeviceHandle;
use vm_manifest_builder::BaseChipsetType;
use vm_manifest_builder::MachineArch;
use vm_manifest_builder::VmChipsetResult;
use vm_manifest_builder::VmManifestBuilder;
use vm_resource::IntoResource;
use vm_resource::Resource;
use vm_resource::kind::DiskHandleKind;
use vm_resource::kind::DiskLayerHandleKind;
use vm_resource::kind::NetEndpointHandleKind;
use vm_resource::kind::VirtioDeviceHandle;
use vm_resource::kind::VmbusDeviceHandleKind;
use vmbus_serial_resources::VmbusSerialDeviceHandle;
use vmbus_serial_resources::VmbusSerialPort;
use vmcore::non_volatile_store::resources::EphemeralNonVolatileStoreHandle;
use vmgs_resources::GuestStateEncryptionPolicy;
use vmgs_resources::VmgsDisk;
use vmgs_resources::VmgsFileHandle;
use vmgs_resources::VmgsResource;
use vmotherboard::ChipsetDeviceHandle;
use vnc_worker_defs::VncParameters;

pub fn openvmm_main() {
    // Save the current state of the terminal so we can restore it back to
    // normal before exiting.
    #[cfg(unix)]
    let orig_termios = io::stderr().is_terminal().then(term::get_termios);

    let mut pidfile_guard: Option<pidfile::Pidfile> = None;
    let exit_code = match do_main(&mut pidfile_guard) {
        Ok(code) => code,
        Err(err) => {
            eprintln!("fatal error: {:?}", err);
            1
        }
    };

    // Restore the terminal to its initial state.
    #[cfg(unix)]
    if let Some(orig_termios) = orig_termios {
        term::set_termios(orig_termios);
    }

    // Clean up the pidfile before terminating, since
    // pal::process::terminate skips destructors.
    drop(pidfile_guard);

    // Terminate the process immediately without graceful shutdown of DLLs or
    // C++ destructors or anything like that. This is all unnecessary and saves
    // time on Windows.
    //
    // Do flush stdout, though, since there may be buffered data.
    let _ = io::stdout().flush();
    pal::process::terminate(exit_code);
}

#[derive(Default)]
struct VmResources {
    console_in: Option<Box<dyn AsyncWrite + Send + Unpin>>,
    /// Keeps the dedicated serial reactor alive while serial I/O objects exist.
    serial_driver: Option<DefaultDriver>,
    framebuffer_access: Option<FramebufferAccess>,
    shutdown_ic: Option<mesh::Sender<hyperv_ic_resources::shutdown::ShutdownRpc>>,
    kvp_ic: Option<mesh::Sender<hyperv_ic_resources::kvp::KvpConnectRpc>>,
    scsi_rpc: Option<mesh::Sender<ScsiControllerRequest>>,
    nvme_vtl2_rpc: Option<mesh::Sender<NvmeControllerRequest>>,
    consomme_rpc: Option<mesh::Sender<net_backend_resources::consomme::ConsommeRequest>>,
    ged_rpc: Option<mesh::Sender<get_resources::ged::GuestEmulationRequest>>,
    vtl2_settings: Option<vtl2_settings_proto::Vtl2Settings>,
    /// Receives dirty rectangles from the synthetic video device for the VNC worker.
    dirty_rect_recv: Option<mesh::Receiver<Vec<video_core::DirtyRect>>>,
    #[cfg(windows)]
    switch_ports: Vec<vmswitch::kernel::SwitchPort>,
}

struct ConsoleState<'a> {
    device: &'a str,
    input: Box<dyn AsyncWrite + Unpin + Send>,
}

/// Build a flat list of switches with their parent port assignments.
///
/// This function converts hierarchical CLI switch definitions into a flat list
/// where each switch specifies its parent port directly.
fn build_switch_list(all_switches: &[cli_args::GenericPcieSwitchCli]) -> Vec<PcieSwitchConfig> {
    all_switches
        .iter()
        .map(|switch_cli| PcieSwitchConfig {
            name: switch_cli.name.clone(),
            parent_port: switch_cli.port_name.clone(),
            ports: (0..switch_cli.num_downstream_ports)
                .map(|i| PciePortConfig {
                    name: format!("{}-downstream-{}", switch_cli.name, i),
                    devfn: None,
                    hotplug: switch_cli.hotplug,
                    acs_capabilities_supported: switch_cli.acs_capabilities_supported,
                    cxl: false,
                    pasid: switch_cli.pasid,
                })
                .collect(),
        })
        .collect()
}

fn base_chipset_type(opt: &Options) -> BaseChipsetType {
    if let Some(igvm) = &opt.igvm {
        match igvm.personality {
            IgvmPersonalityCli::Openhcl => BaseChipsetType::HclHost,
            IgvmPersonalityCli::Uefi => BaseChipsetType::HypervGen2Uefi,
            IgvmPersonalityCli::LinuxDirect if opt.isolation.is_some() => {
                BaseChipsetType::EnlightenedLinuxDirect
            }
            IgvmPersonalityCli::LinuxDirect if opt.hv => BaseChipsetType::HyperVGen2LinuxDirect,
            IgvmPersonalityCli::LinuxDirect => BaseChipsetType::UnenlightenedLinuxDirect,
        }
    } else if matches!(opt.isolation, Some(cli_args::IsolationCli::Snp)) {
        BaseChipsetType::EnlightenedLinuxDirect
    } else if opt.pcat {
        BaseChipsetType::HypervGen1
    } else if opt.uefi.is_some() {
        BaseChipsetType::HypervGen2Uefi
    } else if opt.hv {
        BaseChipsetType::HyperVGen2LinuxDirect
    } else {
        BaseChipsetType::UnenlightenedLinuxDirect
    }
}

/// Build the loader's [`SmbiosConfig`](openvmm_defs::config::SmbiosConfig) from
/// the parsed `--smbios` arguments.
///
/// Multiple `--smbios` arguments are merged (erroring on a field set twice).
/// String overrides left unset fall through to the loader's default identity.
/// The system UUID defaults to the all-zero GUID unless overridden with
/// `uuid=GUID`; `uuid=random` requests a freshly generated per-VM GUID.
fn smbios_config_from_cli(
    args: &[cli_args::SmbiosCli],
) -> anyhow::Result<openvmm_defs::config::SmbiosConfig> {
    let mut merged = cli_args::SmbiosCli::default();
    for arg in args {
        merged.merge(arg.clone())?;
    }
    let cli_args::SmbiosCli {
        bios:
            cli_args::SmbiosBiosCli {
                vendor: bios_vendor,
                version: bios_version,
                release_date: bios_release_date,
                release: bios_release,
            },
        system:
            cli_args::SmbiosSystemCli {
                manufacturer: system_manufacturer,
                product_name: system_product,
                version: system_version,
                serial_number: system_serial,
                sku_number: system_sku,
                family: system_family,
                uuid: system_uuid,
            },
    } = merged;
    Ok(openvmm_defs::config::SmbiosConfig {
        bios: openvmm_defs::config::SmbiosBiosOverrides {
            vendor: bios_vendor,
            version: bios_version,
            release_date: bios_release_date,
            release: bios_release.map(|r| (r.0, r.1)),
        },
        system: openvmm_defs::config::SmbiosSystemOverrides {
            manufacturer: system_manufacturer,
            product_name: system_product,
            version: system_version,
            serial_number: system_serial,
            sku_number: system_sku,
            family: system_family,
            uuid: match system_uuid {
                None => Guid::ZERO,
                Some(cli_args::SmbiosUuid::Random) => Guid::new_random(),
                Some(cli_args::SmbiosUuid::Fixed(guid)) => guid,
            },
        },
    })
}

async fn vm_config_from_command_line(
    spawner: impl Spawn,
    mesh: &VmmMesh,
    opt: &Options,
) -> anyhow::Result<(Config, VmResources)> {
    opt.validate_isolation_options()?;
    opt.validate_igvm_options()?;

    let (_, serial_driver) = DefaultPool::spawn_on_thread("serial");
    let uefi = opt.effective_uefi()?;
    let default_uefi = cli_args::UefiCli::default();
    let uefi_options = uefi.as_ref().unwrap_or(&default_uefi);

    let openhcl_vtl = if opt.vtl2 {
        DeviceVtl::Vtl2
    } else {
        DeviceVtl::Vtl0
    };

    let console_state: RefCell<Option<ConsoleState<'_>>> = RefCell::new(None);
    let setup_serial = |name: &str, cli_cfg, device| -> anyhow::Result<_> {
        Ok(match cli_cfg {
            SerialConfigCli::Console => {
                if let Some(console_state) = console_state.borrow().as_ref() {
                    bail!("console already set by {}", console_state.device);
                }
                let (config, serial) = serial_io::anonymous_serial_pair(&serial_driver)?;
                let (serial_read, serial_write) = AsyncReadExt::split(serial);
                *console_state.borrow_mut() = Some(ConsoleState {
                    device,
                    input: Box::new(serial_write),
                });
                thread::Builder::new()
                    .name(name.to_owned())
                    .spawn(move || {
                        let _ = block_on(futures::io::copy(
                            serial_read,
                            &mut AllowStdIo::new(term::raw_stdout()),
                        ));
                    })
                    .unwrap();
                Some(config)
            }
            SerialConfigCli::Stderr => {
                let (config, serial) = serial_io::anonymous_serial_pair(&serial_driver)?;
                thread::Builder::new()
                    .name(name.to_owned())
                    .spawn(move || {
                        let _ = block_on(futures::io::copy(
                            serial,
                            &mut AllowStdIo::new(term::raw_stderr()),
                        ));
                    })
                    .unwrap();
                Some(config)
            }
            SerialConfigCli::File(path) => Some(serial_io::file_serial(&path, false)?),
            SerialConfigCli::None => None,
            SerialConfigCli::Null => Some(NullSerialBackendHandle.into_resource()),
            SerialConfigCli::Pipe(path) => {
                Some(serial_io::bind_serial(&path).context("failed to bind serial")?)
            }
            SerialConfigCli::Tcp(addr) => {
                Some(serial_io::bind_tcp_serial(&addr).context("failed to bind serial")?)
            }
            SerialConfigCli::NewConsole(app, window_title) => {
                let path = console_relay::random_console_path();
                let config =
                    serial_io::bind_serial(&path).context("failed to bind console serial")?;
                let window_title =
                    window_title.unwrap_or_else(|| name.to_uppercase() + " [OpenVMM]");

                console_relay::launch_console(
                    app.or_else(openvmm_terminal_app).as_deref(),
                    &path,
                    ConsoleLaunchOptions {
                        window_title: Some(window_title),
                    },
                )
                .context("failed to launch console")?;

                Some(config)
            }
        })
    };

    let mut vmbus_devices = Vec::new();

    let com_debugger_mode = [
        opt.com1.as_ref().is_some_and(|c| c.debugger_mode),
        opt.com2.as_ref().is_some_and(|c| c.debugger_mode),
        opt.com3.as_ref().is_some_and(|c| c.debugger_mode),
        opt.com4.as_ref().is_some_and(|c| c.debugger_mode),
    ];

    let serial0_cfg = setup_serial(
        "com1",
        opt.com1
            .clone()
            .map_or(SerialConfigCli::Console, |c| c.backend),
        if cfg!(guest_arch = "x86_64") {
            "ttyS0"
        } else {
            "ttyAMA0"
        },
    )?;
    let serial1_cfg = setup_serial(
        "com2",
        opt.com2
            .clone()
            .map_or(SerialConfigCli::None, |c| c.backend),
        if cfg!(guest_arch = "x86_64") {
            "ttyS1"
        } else {
            "ttyAMA1"
        },
    )?;
    let serial2_cfg = setup_serial(
        "com3",
        opt.com3
            .clone()
            .map_or(SerialConfigCli::None, |c| c.backend),
        if cfg!(guest_arch = "x86_64") {
            "ttyS2"
        } else {
            "ttyAMA2"
        },
    )?;
    let serial3_cfg = setup_serial(
        "com4",
        opt.com4
            .clone()
            .map_or(SerialConfigCli::None, |c| c.backend),
        if cfg!(guest_arch = "x86_64") {
            "ttyS3"
        } else {
            "ttyAMA3"
        },
    )?;
    let with_vmbus_com1_serial = if let Some(vmbus_com1_cfg) = setup_serial(
        "vmbus_com1",
        opt.vmbus_com1_serial
            .clone()
            .unwrap_or(SerialConfigCli::None),
        "vmbus_com1",
    )? {
        vmbus_devices.push((
            openhcl_vtl,
            VmbusSerialDeviceHandle {
                port: VmbusSerialPort::Com1,
                backend: vmbus_com1_cfg,
            }
            .into_resource(),
        ));
        true
    } else {
        false
    };
    let with_vmbus_com2_serial = if let Some(vmbus_com2_cfg) = setup_serial(
        "vmbus_com2",
        opt.vmbus_com2_serial
            .clone()
            .unwrap_or(SerialConfigCli::None),
        "vmbus_com2",
    )? {
        vmbus_devices.push((
            openhcl_vtl,
            VmbusSerialDeviceHandle {
                port: VmbusSerialPort::Com2,
                backend: vmbus_com2_cfg,
            }
            .into_resource(),
        ));
        true
    } else {
        false
    };
    let debugcon_cfg = setup_serial(
        "debugcon",
        opt.debugcon
            .clone()
            .map(|cfg| cfg.serial)
            .unwrap_or(SerialConfigCli::None),
        "debugcon",
    )?;

    let virtio_console_backend = if let Some(serial_cfg) = opt.virtio_console.clone() {
        setup_serial("virtio-console", serial_cfg, "hvc0")?
    } else {
        None
    };

    let mut resources = VmResources::default();
    let mut console_str = "";
    if let Some(ConsoleState { device, input }) = console_state.into_inner() {
        resources.console_in = Some(input);
        console_str = device;
    }

    if opt.shared_memory {
        tracing::warn!("--shared-memory/-M flag has no effect and will be removed");
    }
    if opt.deprecated_prefetch {
        tracing::warn!("--prefetch is deprecated; use --memory prefetch=on");
    }
    if opt.deprecated_private_memory {
        tracing::warn!("--private-memory is deprecated; use --memory shared=off");
    }
    if opt.deprecated_thp {
        tracing::warn!("--thp is deprecated; use --memory shared=off,thp=on");
    }
    if opt.deprecated_memory_backing_file.is_some() {
        tracing::warn!("--memory-backing-file is deprecated; use --memory file=<path>");
    }

    opt.validate_memory_options()?;

    const MAX_PROCESSOR_COUNT: u32 = 1024;

    if opt.processors == 0 || opt.processors > MAX_PROCESSOR_COUNT {
        bail!("invalid proc count: {}", opt.processors);
    }

    // Total SCSI channel count should not exceed the processor count
    // (at most, one channel per VP).
    if opt.scsi_sub_channels > (MAX_PROCESSOR_COUNT - 1) as u16 {
        bail!(
            "invalid SCSI sub-channel count: requested {}, max {}",
            opt.scsi_sub_channels,
            MAX_PROCESSOR_COUNT - 1
        );
    }

    let with_get = opt.get || (opt.vtl2 && !opt.no_get);

    let mut storage = storage_builder::StorageBuilder::new(with_get.then_some(openhcl_vtl));

    // Register named controllers first, so that --disk on=<name>
    // references can be resolved.
    for ctrl in &opt.nvme_pci {
        let transport = match &ctrl.transport {
            cli_args::NvmeControllerTransport::Pcie(port) => {
                storage_builder::NvmeControllerTransport::Pcie(port.clone())
            }
            cli_args::NvmeControllerTransport::Vpci(guid) => {
                let guid = guid.unwrap_or_else(|| storage_builder::deterministic_guid(&ctrl.id));
                storage_builder::NvmeControllerTransport::Vpci(guid)
            }
        };
        storage.add_nvme_controller(ctrl.id.clone(), ctrl.vtl, transport, None)?;
    }

    for ctrl in &opt.vmbus_scsi {
        let instance_id = storage_builder::deterministic_guid(&ctrl.id);
        storage.add_scsi_controller(ctrl.id.clone(), ctrl.vtl, instance_id, ctrl.sub_channels)?;
    }

    for ctrl in &opt.openhcl_controller {
        let controller_type = match ctrl.controller_type {
            cli_args::OpenhclControllerType::Scsi => storage_builder::OpenhclControllerType::Scsi,
            cli_args::OpenhclControllerType::Nvme => storage_builder::OpenhclControllerType::Nvme,
        };
        let instance_id = ctrl
            .guid
            .unwrap_or_else(|| storage_builder::deterministic_guid(&ctrl.id));
        storage.add_openhcl_controller(ctrl.id.clone(), controller_type, instance_id)?;
    }

    for &cli_args::DiskCli {
        vtl,
        ref kind,
        read_only,
        is_dvd,
        underhill,
        ref pcie_port,
        ref serial,
        ref controller,
        nsid,
        lun,
        ref relay,
    } in &opt.disk
    {
        if serial.is_some() {
            anyhow::bail!("`serial` is only supported by `--virtio-blk`");
        }
        if controller.is_none() && underhill.is_none() && relay.is_none() {
            tracing::warn!(
                "--disk without `on` is deprecated; \
                 use --vmbus-scsi and --disk on=<name> instead"
            );
        }

        let relay_target = relay
            .as_ref()
            .map(|(name, loc)| storage_builder::RelayTarget {
                controller: name.clone(),
                location: *loc,
            });

        let target = if let Some(name) = controller {
            if pcie_port.is_some() {
                anyhow::bail!("`on` is incompatible with `pcie_port` on `--disk`");
            }
            storage_builder::DiskLocation::Named {
                controller: name.clone(),
                nsid,
                lun,
            }
        } else if pcie_port.is_some() {
            anyhow::bail!("`--disk` is incompatible with `pcie_port` without `controller`");
        } else {
            if opt.no_vmbus {
                anyhow::bail!(
                    "`--disk` without `on=` attaches to the default VMBus SCSI controller and \
                     cannot be used with `--no-vmbus`; use `on=<name>` to attach to a named controller"
                );
            }
            storage_builder::DiskLocation::Scsi(None)
        };

        storage
            .add(
                vtl,
                underhill,
                relay_target,
                target,
                kind,
                is_dvd,
                read_only,
            )
            .await?;
    }

    for &cli_args::IdeDiskCli {
        ref kind,
        read_only,
        channel,
        device,
        is_dvd,
    } in &opt.ide
    {
        storage
            .add(
                DeviceVtl::Vtl0,
                None,
                None,
                storage_builder::DiskLocation::Ide(channel, device),
                kind,
                is_dvd,
                read_only,
            )
            .await?;
    }

    if !opt.nvme.is_empty() {
        tracing::warn!("--nvme is deprecated; use --nvme-pci and --disk on=<name> instead");

        // Pre-register implicit PCIe controllers for unique port names.
        let mut registered_ports = std::collections::BTreeSet::new();
        for disk in &opt.nvme {
            if let Some(port) = &disk.pcie_port {
                if registered_ports.insert(port.clone()) {
                    storage.add_nvme_controller(
                        port.clone(),
                        DeviceVtl::Vtl0,
                        storage_builder::NvmeControllerTransport::Pcie(port.clone()),
                        None,
                    ).with_context(|| format!(
                        "legacy --nvme flag conflicts with an explicit controller named '{port}'; \
                         use --nvme-pci and --disk on=<name> instead"
                    ))?;
                }
            }
        }
    }

    for &cli_args::DiskCli {
        vtl,
        ref kind,
        read_only,
        is_dvd,
        underhill,
        ref pcie_port,
        ref serial,
        controller: _,
        nsid: _,
        lun: _,
        relay: _,
    } in &opt.nvme
    {
        if serial.is_some() {
            anyhow::bail!("`serial` is only supported by `--virtio-blk`");
        }
        let target = if let Some(port) = pcie_port {
            storage_builder::DiskLocation::Named {
                controller: port.clone(),
                nsid: None,
                lun: None,
            }
        } else {
            storage_builder::DiskLocation::Nvme(None)
        };
        storage
            .add(vtl, underhill, None, target, kind, is_dvd, read_only)
            .await?;
    }

    for &cli_args::DiskCli {
        vtl,
        ref kind,
        read_only,
        is_dvd,
        ref underhill,
        ref pcie_port,
        ref serial,
        controller: _,
        nsid: _,
        lun: _,
        relay: _,
    } in &opt.virtio_blk
    {
        if underhill.is_some() {
            anyhow::bail!("underhill not supported with virtio-blk");
        }
        storage
            .add(
                vtl,
                None,
                None,
                storage_builder::DiskLocation::VirtioBlk {
                    pcie_port: pcie_port.clone(),
                    serial: serial.clone(),
                },
                kind,
                is_dvd,
                read_only,
            )
            .await?;
    }

    let mut floppy_disks = Vec::new();
    for disk in &opt.floppy {
        let &cli_args::FloppyDiskCli {
            ref kind,
            read_only,
        } = disk;
        floppy_disks.push(FloppyDiskConfig {
            disk_type: disk_open(kind, read_only).await?,
            read_only,
        });
    }

    let mut vpci_mana_nics = [(); 3].map(|()| None);
    let mut pcie_mana_nics = BTreeMap::<String, GdmaDeviceHandle>::new();
    let mut underhill_nics = Vec::new();
    let mut vpci_devices = Vec::new();

    let mut nic_index = 0;
    for cli_cfg in &opt.net {
        if cli_cfg.pcie_port.is_some() {
            anyhow::bail!("`--net` does not support PCIe");
        }
        let vport = parse_endpoint(cli_cfg, &mut nic_index, &mut resources)?;
        if cli_cfg.underhill {
            if !opt.no_alias_map {
                anyhow::bail!("must specify --no-alias-map to offer NICs to VTL2");
            }
            let mana = vpci_mana_nics[openhcl_vtl as usize].get_or_insert_with(|| {
                let vpci_instance_id = Guid::new_random();
                underhill_nics.push(vtl2_settings_proto::NicDeviceLegacy {
                    instance_id: vpci_instance_id.to_string(),
                    subordinate_instance_id: None,
                    max_sub_channels: None,
                });
                (vpci_instance_id, GdmaDeviceHandle { vports: Vec::new() })
            });
            mana.1.vports.push(VportDefinition {
                mac_address: vport.mac_address,
                endpoint: vport.endpoint,
            });
        } else {
            vmbus_devices.push(vport.into_netvsp_handle());
        }
    }

    if opt.nic {
        let nic_config = parse_endpoint(
            &NicConfigCli {
                vtl: DeviceVtl::Vtl0,
                endpoint: EndpointConfigCli::Consomme {
                    cidr: None,
                    host_fwd: Vec::new(),
                },
                max_queues: None,
                underhill: false,
                pcie_port: None,
            },
            &mut nic_index,
            &mut resources,
        )?;
        vmbus_devices.push(nic_config.into_netvsp_handle());
    }

    // Build initial PCIe devices list from CLI options. Storage devices
    // (e.g., NVMe controllers on PCIe ports) are added later by storage_builder.
    let mut pcie_devices = Vec::new();
    for (index, cli_cfg) in opt.pcie_remote.iter().enumerate() {
        tracing::info!(
            port_name = %cli_cfg.port_name,
            socket_addr = ?cli_cfg.socket_addr,
            "instantiating PCIe remote device"
        );

        // Generate a deterministic instance ID based on index
        const PCIE_REMOTE_BASE_INSTANCE_ID: Guid =
            guid::guid!("28ed784d-c059-429f-9d9a-46bea02562c0");
        let instance_id = Guid {
            data1: index as u32,
            ..PCIE_REMOTE_BASE_INSTANCE_ID
        };

        pcie_devices.push(PcieDeviceConfig {
            port_name: cli_cfg.port_name.clone(),
            resource: pcie_remote_resources::PcieRemoteHandle {
                instance_id,
                socket_addr: cli_cfg.socket_addr.clone(),
                hu: cli_cfg.hu,
                controller: cli_cfg.controller,
            }
            .into_resource(),
        });
    }

    #[cfg(windows)]
    let mut kernel_vmnics = Vec::new();
    #[cfg(windows)]
    for (index, switch_id) in opt.kernel_vmnic.iter().enumerate() {
        // Pick a random MAC address.
        let mut mac_address = [0x00, 0x15, 0x5D, 0, 0, 0];
        getrandom::fill(&mut mac_address[3..]).expect("rng failure");

        // Pick a fixed instance ID based on the index.
        const BASE_INSTANCE_ID: Guid = guid::guid!("00000000-435d-11ee-9f59-00155d5016fc");
        let instance_id = Guid {
            data1: index as u32,
            ..BASE_INSTANCE_ID
        };

        let switch_id = if switch_id == "default" {
            None
        } else {
            Some(switch_id.as_str())
        };
        let (port_id, port) = new_switch_port(switch_id)?;
        resources.switch_ports.push(port);

        kernel_vmnics.push(openvmm_defs::config::KernelVmNicConfig {
            instance_id,
            mac_address: mac_address.into(),
            switch_port_id: port_id,
        });
    }

    for vport in &opt.mana {
        let vport = parse_endpoint(vport, &mut nic_index, &mut resources)?;
        let vport_array = match (vport.vtl as usize, vport.pcie_port) {
            (vtl, None) => {
                &mut vpci_mana_nics[vtl]
                    .get_or_insert_with(|| {
                        (Guid::new_random(), GdmaDeviceHandle { vports: Vec::new() })
                    })
                    .1
                    .vports
            }
            (0, Some(pcie_port)) => {
                &mut pcie_mana_nics
                    .entry(pcie_port)
                    .or_insert(GdmaDeviceHandle { vports: Vec::new() })
                    .vports
            }
            _ => anyhow::bail!("PCIe NICs only supported to VTL0"),
        };
        vport_array.push(VportDefinition {
            mac_address: vport.mac_address,
            endpoint: vport.endpoint,
        });
    }

    vpci_devices.extend(
        vpci_mana_nics
            .into_iter()
            .enumerate()
            .filter_map(|(vtl, nic)| {
                nic.map(|(instance_id, handle)| VpciDeviceConfig {
                    vtl: match vtl {
                        0 => DeviceVtl::Vtl0,
                        1 => DeviceVtl::Vtl1,
                        2 => DeviceVtl::Vtl2,
                        _ => unreachable!(),
                    },
                    instance_id,
                    resource: handle.into_resource(),
                    vnode: None,
                })
            }),
    );

    pcie_devices.extend(
        pcie_mana_nics
            .into_iter()
            .map(|(pcie_port, handle)| PcieDeviceConfig {
                port_name: pcie_port,
                resource: handle.into_resource(),
            }),
    );

    for cxl_test in &opt.cxl_test {
        pcie_devices.push(PcieDeviceConfig {
            port_name: cxl_test.pcie_port.clone(),
            resource: CxlTestDeviceHandle {
                hdm_size_bytes: cxl_test.hdm_size,
            }
            .into_resource(),
        });
    }

    #[cfg(guest_arch = "aarch64")]
    let arch = MachineArch::Aarch64;
    #[cfg(guest_arch = "x86_64")]
    let arch = MachineArch::X86_64;

    #[cfg(guest_arch = "x86_64")]
    anyhow::ensure!(
        opt.amd_iommu.is_empty() || opt.intel_vtd.is_empty(),
        "--amd-iommu and --intel-vtd cannot both be used in the same VM"
    );

    #[cfg(guest_arch = "x86_64")]
    let mut amd_iommu_names: HashSet<&str> = opt.amd_iommu.iter().map(|s| s.as_str()).collect();
    #[cfg(guest_arch = "x86_64")]
    let mut vtd_names: HashSet<&str> = opt.intel_vtd.iter().map(|s| s.as_str()).collect();

    // Map each `--smmu` entry to its root complex, rejecting duplicate `rc=`
    // entries up front. Entries are removed as they are matched to a root
    // complex below; any left over refer to unknown root complexes.
    #[cfg(guest_arch = "aarch64")]
    let mut smmu_names: std::collections::HashMap<&str, &cli_args::SmmuCli> = {
        let mut map = std::collections::HashMap::new();
        for s in &opt.smmu {
            if map.insert(s.rc_name.as_str(), s).is_some() {
                anyhow::bail!(
                    "--smmu specified multiple times for root complex '{}'",
                    s.rc_name
                );
            }
        }
        map
    };

    let mut pcie_root_complexes = Vec::new();
    for (i, rc_cli) in opt.pcie_root_complex.iter().enumerate() {
        let ports: Vec<PciePortConfig> = opt
            .pcie_root_port
            .iter()
            .filter(|port_cli| port_cli.root_complex_name == rc_cli.name)
            .map(|port_cli| PciePortConfig {
                name: port_cli.name.clone(),
                devfn: port_cli.devfn,
                hotplug: port_cli.hotplug,
                acs_capabilities_supported: port_cli.acs_capabilities_supported,
                cxl: port_cli.cxl,
                pasid: port_cli.pasid,
            })
            .collect();

        const ONE_MB: u64 = 1024 * 1024;
        // Keep all PCI windows 1MB-granular to match layout and downstream placement rules.
        let low_mmio_size = (rc_cli.low_mmio as u64).next_multiple_of(ONE_MB);
        let high_mmio_size = rc_cli
            .high_mmio
            .checked_next_multiple_of(ONE_MB)
            .context("high mmio rounding error")?;

        // Count CXL-capable ports under the root bus. If the root bus has CXL root ports, it needs CHBCR.
        let cxl_port_count = ports.iter().filter(|port| port.cxl).count() as u64;

        let cxl = if cxl_port_count != 0 {
            Some(RootComplexCxlConfig {
                hdm_size: rc_cli.hdm,
                hdm_window_restrictions: rc_cli.hdm_window_restrictions.bits(),
            })
        } else {
            None
        };
        pcie_root_complexes.push(PcieRootComplexConfig {
            index: i as u32,
            name: rc_cli.name.clone(),
            segment: rc_cli.segment,
            start_bus: rc_cli.start_bus,
            end_bus: rc_cli.end_bus,
            low_mmio: if let Some(base) = rc_cli.low_mmio_base {
                PcieMmioRangeConfig::Fixed(
                    memory_range::MemoryRange::try_new(base..base.wrapping_add(low_mmio_size))
                        .context("invalid low MMIO range")?,
                )
            } else {
                PcieMmioRangeConfig::Dynamic {
                    size: low_mmio_size,
                }
            },
            high_mmio: if let Some(base) = rc_cli.high_mmio_base {
                PcieMmioRangeConfig::Fixed(
                    memory_range::MemoryRange::try_new(base..base.wrapping_add(high_mmio_size))
                        .context("invalid high MMIO range")?,
                )
            } else {
                PcieMmioRangeConfig::Dynamic {
                    size: high_mmio_size,
                }
            },
            cxl,
            ports,
            #[cfg(guest_arch = "aarch64")]
            iommu: smmu_names.remove(rc_cli.name.as_str()).map(|s| {
                openvmm_defs::config::PcieIommuConfig::Smmu {
                    accel: s.accel,
                    oas: match s.oas {
                        cli_args::SmmuOasCli::Auto => openvmm_defs::config::SmmuOas::Auto,
                        cli_args::SmmuOasCli::Fixed(bits) => {
                            openvmm_defs::config::SmmuOas::Fixed(bits)
                        }
                    },
                }
            }),
            #[cfg(guest_arch = "x86_64")]
            iommu: if amd_iommu_names.remove(rc_cli.name.as_str()) {
                Some(openvmm_defs::config::PcieIommuConfig::AmdVi)
            } else if vtd_names.remove(rc_cli.name.as_str()) {
                Some(openvmm_defs::config::PcieIommuConfig::IntelVtd)
            } else {
                None
            },
            vnode: rc_cli.vnode,
            preserve_bars: rc_cli.preserve_bars,
        });
    }

    #[cfg(guest_arch = "aarch64")]
    if let Some(name) = smmu_names.into_keys().next() {
        anyhow::bail!("--smmu refers to unknown root complex '{name}'");
    }
    #[cfg(guest_arch = "x86_64")]
    if let Some(name) = amd_iommu_names.into_iter().next() {
        anyhow::bail!("--amd-iommu refers to unknown root complex '{name}'");
    }
    #[cfg(guest_arch = "x86_64")]
    if let Some(name) = vtd_names.into_iter().next() {
        anyhow::bail!("--intel-vtd refers to unknown root complex '{name}'");
    }

    let pcie_switches = build_switch_list(&opt.pcie_switch);
    let pcie_generic_initiators = opt
        .pcie_generic_initiator
        .iter()
        .map(|gi| openvmm_defs::config::PcieGenericInitiatorConfig {
            port_name: gi.port_name.clone(),
            node: gi.node,
        })
        .collect();
    #[cfg(target_os = "linux")]
    let vfio_pcie_devices: Vec<PcieDeviceConfig> = {
        use std::collections::HashMap;
        use vm_resource::IntoResource;

        // Process --iommu flags: open /dev/iommu for each declared context.
        let mut iommu_map: HashMap<String, std::fs::File> = HashMap::new();
        for iommu_cli in &opt.iommu {
            anyhow::ensure!(
                !iommu_map.contains_key(&iommu_cli.id),
                "duplicate --iommu id={}",
                iommu_cli.id
            );
            let file = std::fs::OpenOptions::new()
                .read(true)
                .write(true)
                .open("/dev/iommu")
                .context("failed to open /dev/iommu (is iommufd available?)")?;
            iommu_map.insert(iommu_cli.id.clone(), file);
        }

        opt.vfio
            .iter()
            .map(|cli_cfg| {
                let sysfs_path = Path::new("/sys/bus/pci/devices").join(&cli_cfg.pci_id);

                if let Some(iommu_id) = &cli_cfg.iommu {
                    // cdev + iommufd path
                    let iommufd = iommu_map.get(iommu_id).with_context(|| {
                        format!(
                            "--vfio device {} references iommu={iommu_id}, \
                             but no --iommu id={iommu_id} was specified",
                            cli_cfg.pci_id
                        )
                    })?;
                    // Clone the iommufd fd so the per-iommu manager can own it.
                    // The first device for a given iommu ID uses the cloned fd
                    // to create the IoasManager; subsequent devices reuse the
                    // existing manager and the cloned fd is dropped.
                    let iommufd = iommufd.try_clone().with_context(|| {
                        format!("failed to dup iommufd fd for iommu={iommu_id}")
                    })?;

                    // Open the cdev device node.
                    let vfio_dev_dir = sysfs_path.join("vfio-dev");
                    let entry = std::fs::read_dir(&vfio_dev_dir)
                        .with_context(|| {
                            format!(
                                "failed to read {}: is {} bound to vfio-pci?",
                                vfio_dev_dir.display(),
                                cli_cfg.pci_id
                            )
                        })?
                        .next()
                        .context("no vfio-dev entry found")?
                        .context("failed to read vfio-dev entry")?;
                    let dev_path = Path::new("/dev/vfio/devices").join(entry.file_name());
                    let cdev = std::fs::OpenOptions::new()
                        .read(true)
                        .write(true)
                        .open(&dev_path)
                        .with_context(|| format!("failed to open {}", dev_path.display()))?;

                    Ok(PcieDeviceConfig {
                        port_name: cli_cfg.port_name.clone(),
                        resource: vfio_assigned_device_resources::VfioCdevDeviceHandle {
                            pci_id: cli_cfg.pci_id.clone(),
                            cdev,
                            iommufd,
                            iommu_id: iommu_id.clone(),
                            bar_addresses: cli_cfg.bar_addresses,
                        }
                        .into_resource(),
                    })
                } else {
                    // Legacy group/container path
                    let iommu_group_link = std::fs::read_link(sysfs_path.join("iommu_group"))
                        .with_context(|| {
                            format!("failed to read IOMMU group for {}", cli_cfg.pci_id)
                        })?;
                    let group_id: u64 = iommu_group_link
                        .file_name()
                        .and_then(|s| s.to_str())
                        .context("invalid iommu_group symlink")?
                        .parse()
                        .context("failed to parse IOMMU group ID")?;
                    let group = std::fs::OpenOptions::new()
                        .read(true)
                        .write(true)
                        .open(format!("/dev/vfio/{group_id}"))
                        .with_context(|| format!("failed to open /dev/vfio/{group_id}"))?;

                    Ok(PcieDeviceConfig {
                        port_name: cli_cfg.port_name.clone(),
                        resource: vfio_assigned_device_resources::VfioDeviceHandle {
                            pci_id: cli_cfg.pci_id.clone(),
                            group,
                            bar_addresses: cli_cfg.bar_addresses,
                        }
                        .into_resource(),
                    })
                }
            })
            .collect::<anyhow::Result<Vec<_>>>()?
    };

    #[cfg(windows)]
    let vpci_resources: Vec<_> = opt
        .device
        .iter()
        .map(|path| -> anyhow::Result<_> {
            Ok(virt_whp::device::DeviceHandle(
                whp::VpciResource::new(
                    None,
                    Default::default(),
                    &whp::VpciResourceDescriptor::Sriov(path, 0, 0),
                )
                .with_context(|| format!("opening PCI device {}", path))?,
            ))
        })
        .collect::<Result<_, _>>()?;

    // Create a vmbusproxy handle if needed by any devices.
    #[cfg(windows)]
    let vmbusproxy_handle = if !kernel_vmnics.is_empty() {
        Some(vmbus_proxy::ProxyHandle::new().context("failed to open vmbusproxy handle")?)
    } else {
        None
    };

    let framebuffer = if opt.gfx || opt.vtl2_gfx || opt.vnc.vnc || opt.pcat {
        let vram = alloc_shared_memory(FRAMEBUFFER_SIZE, "vram")?;
        let (fb, fba) =
            framebuffer::framebuffer(vram, FRAMEBUFFER_SIZE, 0).context("creating framebuffer")?;
        resources.framebuffer_access = Some(fba);
        Some(fb)
    } else {
        None
    };

    let load_mode;
    let with_hv;

    let any_serial_configured = serial0_cfg.is_some()
        || serial1_cfg.is_some()
        || serial2_cfg.is_some()
        || serial3_cfg.is_some();

    let has_com3 = serial2_cfg.is_some();

    let mut chipset = VmManifestBuilder::new(base_chipset_type(opt), arch);

    if framebuffer.is_some() {
        chipset = chipset.with_framebuffer();
    }
    if opt.guest_watchdog {
        chipset = chipset.with_guest_watchdog();
    }
    if any_serial_configured {
        chipset = chipset.with_serial([serial0_cfg, serial1_cfg, serial2_cfg, serial3_cfg]);
    }
    chipset = chipset.with_serial_debugger_mode(com_debugger_mode);
    if opt.battery {
        let (tx, rx) = mesh::channel();
        tx.send(HostBatteryUpdate::default_present());
        chipset = chipset.with_battery(rx);
    }
    if opt.no_vmbus {
        chipset = chipset.without_vmbus();
    }
    if let Some(cfg) = &opt.debugcon {
        chipset = chipset.with_debugcon(
            debugcon_cfg.unwrap_or_else(|| DisconnectedSerialBackendHandle.into_resource()),
            cfg.port,
        );
    }

    let (base_template, custom_uefi_json) = {
        #[cfg(guest_arch = "aarch64")]
        use firmware_uefi_resources::aarch64_secure_boot_templates as secure_boot_templates;
        #[cfg(guest_arch = "x86_64")]
        use firmware_uefi_resources::x64_secure_boot_templates as secure_boot_templates;
        let base_template = opt.secure_boot_template.map(|template| match template {
            SecureBootTemplateCli::Windows => secure_boot_templates::microsoft_windows(),
            SecureBootTemplateCli::UefiCa => secure_boot_templates::microsoft_uefi_ca(),
        });

        // TODO: fallback to VMGS read if no command line flag was given

        let custom_uefi_json = match &opt.custom_uefi_json {
            Some(file) => Some(
                fs_err::read(file)
                    .context("opening custom uefi json file")?
                    .into(),
            ),
            None => None,
        };

        (base_template, custom_uefi_json)
    };

    if (uefi.is_some() && opt.igvm.is_none())
        || opt
            .igvm
            .as_ref()
            .is_some_and(|igvm| igvm.personality == IgvmPersonalityCli::Uefi)
    {
        let log_level = match uefi_options.diagnostics.unwrap_or_default() {
            EfiDiagnosticsLogLevelCli::Default => firmware_uefi_resources::LogLevel::make_default(),
            EfiDiagnosticsLogLevelCli::Info => firmware_uefi_resources::LogLevel::make_info(),
            EfiDiagnosticsLogLevelCli::Full => firmware_uefi_resources::LogLevel::make_full(),
        };
        let nvram_storage = if opt.vmgs.is_some() {
            VmgsFileHandle::new(vmgs_format::FileId::BIOS_NVRAM, true).into_resource()
        } else {
            EphemeralNonVolatileStoreHandle.into_resource()
        };
        chipset = chipset.with_uefi(vm_manifest_builder::UefiManifest::new(
            arch,
            base_template,
            custom_uefi_json,
            opt.secure_boot,
            log_level,
            None,
            nvram_storage,
            None,
        ));
    }

    // Build the SMBIOS config once, up front, so that UEFI and Linux direct
    // boot share a single source for the VM's BIOS GUID / system UUID. The TPM
    // also keys off this GUID.
    let smbios = Box::new(smbios_config_from_cli(&opt.smbios)?);
    let bios_guid = smbios.system.uuid;

    // Capture the SMBIOS config for the OpenHCL/GED path before `smbios` is
    // potentially moved into a non-VTL2 LoadMode below. The GED forwards only
    // the system identity to the paravisor and fails closed on BIOS overrides
    // it cannot honor, so it is delivered as the shared `SmbiosConfig`.
    let ged_smbios = (*smbios).clone();

    let layout_config = chipset.layout_config();
    let VmChipsetResult {
        chipset,
        mut chipset_devices,
        pci_chipset_devices,
        isa_dma_controller,
        capabilities,
    } = chipset
        .build()
        .context("failed to build chipset configuration")?;

    let tpm_version = opt.tpm.map(|cli_ver| match cli_ver {
        TpmVersionCli::V138 => TpmVersion::V138,
        TpmVersionCli::V185 => TpmVersion::V185,
    });

    if opt.restore_snapshot.is_some() {
        // Snapshot restore: skip firmware loading entirely. Device state and
        // memory come from the snapshot directory.
        load_mode = LoadMode::None;
        with_hv = true;
    } else if let Some(igvm) = &opt.igvm {
        let cli_args::UefiCli {
            firmware,
            debug: _,
            enable_memory_protections: _,
            force_dma_bounce: _,
            force_firmware_version,
            disable_frontpage: _,
            console: _,
            diagnostics: _,
            default_boot_always_attempt: _,
        } = uefi_options;

        anyhow::ensure!(
            firmware.is_none(),
            "--uefi firmware is not supported with --igvm"
        );
        anyhow::ensure!(
            !force_firmware_version,
            "--uefi force_firmware_version is not supported with --igvm"
        );
        let file = fs_err::File::open(&igvm.firmware)
            .context("failed to open igvm file")?
            .into();
        let cmdline = opt.cmdline.join(" ");
        with_hv = match igvm.personality {
            IgvmPersonalityCli::Openhcl | IgvmPersonalityCli::Uefi => true,
            IgvmPersonalityCli::LinuxDirect => opt.hv,
        };

        load_mode = LoadMode::Igvm {
            file,
            cmdline,
            vtl2_base_address: if opt.vtl2 {
                opt.igvm_vtl2_relocation_type
            } else {
                Vtl2BaseAddressType::File
            },
            com_serial: has_com3.then_some(UartId::Com(ComPort::Com3)),
            personality: match igvm.personality {
                IgvmPersonalityCli::Openhcl => IgvmPersonality::Openhcl,
                IgvmPersonalityCli::Uefi => IgvmPersonality::Uefi,
                IgvmPersonalityCli::LinuxDirect => IgvmPersonality::LinuxDirect,
            },
        };

        // An IGVM launch carries no SMBIOS field of its own; the identity is
        // only delivered over the GET/GED channel, which is absent here. Reject
        // overrides that would otherwise be silently dropped.
        let smbios_requested = !opt.smbios.is_empty();
        let smbios_delivered_via_get = with_get && with_hv;
        if smbios_requested && !smbios_delivered_via_get {
            anyhow::bail!(
                "--smbios is not supported for IGVM launches without an OpenHCL GET channel"
            );
        }
    } else if opt.pcat {
        // Emit a nice error early instead of complaining about missing firmware.
        if arch != MachineArch::X86_64 {
            anyhow::bail!("pcat not supported on this architecture");
        }
        with_hv = true;

        let firmware = openvmm_pcat_locator::find_pcat_bios(opt.pcat_firmware.as_deref())?;
        load_mode = LoadMode::Pcat {
            firmware,
            boot_order: opt
                .pcat_boot_order
                .map(|x| x.0)
                .unwrap_or(DEFAULT_PCAT_BOOT_ORDER),
            hibernation_enabled: opt.hibernation,
            smbios,
        };
    } else if let Some(uefi_options) = &uefi {
        use openvmm_defs::config::UefiConsoleMode;

        let cli_args::UefiCli {
            firmware,
            debug,
            enable_memory_protections,
            force_dma_bounce,
            force_firmware_version,
            disable_frontpage,
            console,
            diagnostics: _,
            default_boot_always_attempt,
        } = uefi_options;

        if opt.no_hv && cfg!(guest_arch = "x86_64") {
            anyhow::bail!("--no-hv is not supported on x86_64");
        }

        with_hv = !opt.no_hv;

        let default_firmware = cli_args::default_uefi_firmware();
        let firmware = fs_err::File::open(
            firmware
                .as_ref()
                .or(default_firmware.as_ref())
                .context("must provide uefi firmware when booting with uefi")?,
        )
        .context("failed to open uefi firmware")?;

        // TODO: It would be better to default memory protections to on, but currently Linux does not boot via UEFI due to what
        //       appears to be a GRUB memory protection fault. Memory protections are therefore only enabled if configured.
        load_mode = LoadMode::Uefi {
            firmware: firmware.into(),
            enable_debugging: *debug,
            enable_memory_protections: *enable_memory_protections,
            disable_frontpage: *disable_frontpage,
            tpm_version,
            enable_battery: opt.battery,
            enable_serial: any_serial_configured,
            enable_vpci_boot: false,
            uefi_console_mode: console.map(|m| match m {
                UefiConsoleModeCli::Default => UefiConsoleMode::Default,
                UefiConsoleModeCli::Com1 => UefiConsoleMode::Com1,
                UefiConsoleModeCli::Com2 => UefiConsoleMode::Com2,
                UefiConsoleModeCli::None => UefiConsoleMode::None,
            }),
            default_boot_always_attempt: *default_boot_always_attempt,
            smbios,
            enable_vmbus: !opt.no_vmbus,
            force_dma_bounce: *force_dma_bounce,
            enable_hv: !opt.no_hv,
            hibernation_enabled: opt.hibernation,
            force_firmware_version: *force_firmware_version,
        };
    } else {
        // Linux Direct
        let mut cmdline = "panic=-1 debug".to_string();

        with_hv = opt.hv;
        if with_hv && opt.pcie_root_complex.is_empty() {
            cmdline += " pci=off";
        }

        if !console_str.is_empty() {
            let _ = write!(&mut cmdline, " console={}", console_str);
        }

        if opt.gfx {
            cmdline += " console=tty";
        }
        for extra in &opt.cmdline {
            let _ = write!(&mut cmdline, " {}", extra);
        }

        let kernel = fs_err::File::open(
            (opt.kernel.0)
                .as_ref()
                .context("must provide kernel when booting with linux direct")?,
        )
        .context("failed to open kernel")?;
        let initrd = (opt.initrd.0)
            .as_ref()
            .map(fs_err::File::open)
            .transpose()
            .context("failed to open initrd")?;

        load_mode = LoadMode::Linux {
            kernel: kernel.into(),
            initrd: initrd.map(Into::into),
            cmdline,
            enable_serial: any_serial_configured,
            isolation: if matches!(opt.isolation, Some(cli_args::IsolationCli::Snp)) {
                openvmm_defs::config::LinuxIsolationConfig::Snp {
                    restricted_injection: opt.snp_restricted_injection,
                }
            } else {
                openvmm_defs::config::LinuxIsolationConfig::None
            },
            boot_mode: if opt.device_tree {
                openvmm_defs::config::LinuxDirectBootMode::DeviceTree
            } else {
                openvmm_defs::config::LinuxDirectBootMode::Acpi
            },
            smbios,
        };
    }

    let mut vmgs = Some(if let Some(VmgsCli { kind, provision }) = &opt.vmgs {
        let disk = VmgsDisk {
            disk: disk_open(kind, false)
                .await
                .context("failed to open vmgs disk")?,
            encryption_policy: if opt.test_gsp_by_id {
                GuestStateEncryptionPolicy::GspById(true)
            } else {
                GuestStateEncryptionPolicy::None(true)
            },
        };
        match provision {
            ProvisionVmgs::OnEmpty => VmgsResource::Disk(disk),
            ProvisionVmgs::OnFailure => VmgsResource::ReprovisionOnFailure(disk),
            ProvisionVmgs::True => VmgsResource::Reprovision(disk),
        }
    } else {
        VmgsResource::Ephemeral
    });

    if with_get && with_hv {
        let has_vtl0_nvme = storage.has_vtl0_nvme();
        let vtl2_settings = vtl2_settings_proto::Vtl2Settings {
            version: vtl2_settings_proto::vtl2_settings_base::Version::V1.into(),
            fixed: Some(Default::default()),
            dynamic: Some(vtl2_settings_proto::Vtl2SettingsDynamic {
                storage_controllers: storage.build_openhcl_settings(opt.vmbus_redirect),
                nic_devices: underhill_nics,
            }),
            namespace_settings: Vec::default(),
        };

        // Cache the VTL2 settings for later modification via the interactive console.
        resources.vtl2_settings = Some(vtl2_settings.clone());

        let (send, guest_request_recv) = mesh::channel();
        resources.ged_rpc = Some(send);

        let vmgs = vmgs.take().unwrap();

        vmbus_devices.extend([
            (
                openhcl_vtl,
                get_resources::gel::GuestEmulationLogHandle.into_resource(),
            ),
            (
                openhcl_vtl,
                get_resources::ged::GuestEmulationDeviceHandle {
                    firmware: if opt.pcat {
                        get_resources::ged::GuestFirmwareConfig::Pcat {
                            boot_order: opt
                                .pcat_boot_order
                                .map_or(DEFAULT_PCAT_BOOT_ORDER, |x| x.0)
                                .map(|x| match x {
                                    openvmm_defs::config::PcatBootDevice::Floppy => {
                                        get_resources::ged::PcatBootDevice::Floppy
                                    }
                                    openvmm_defs::config::PcatBootDevice::HardDrive => {
                                        get_resources::ged::PcatBootDevice::HardDrive
                                    }
                                    openvmm_defs::config::PcatBootDevice::Optical => {
                                        get_resources::ged::PcatBootDevice::Optical
                                    }
                                    openvmm_defs::config::PcatBootDevice::Network => {
                                        get_resources::ged::PcatBootDevice::Network
                                    }
                                }),
                        }
                    } else {
                        use get_resources::ged::UefiConsoleMode;

                        get_resources::ged::GuestFirmwareConfig::Uefi {
                            enable_vpci_boot: has_vtl0_nvme,
                            firmware_debug: uefi_options.debug,
                            enable_memory_protections: uefi_options.enable_memory_protections,
                            disable_frontpage: uefi_options.disable_frontpage,
                            console_mode: match uefi_options.console.unwrap_or(UefiConsoleModeCli::Default) {
                                UefiConsoleModeCli::Default => UefiConsoleMode::Default,
                                UefiConsoleModeCli::Com1 => UefiConsoleMode::COM1,
                                UefiConsoleModeCli::Com2 => UefiConsoleMode::COM2,
                                UefiConsoleModeCli::None => UefiConsoleMode::None,
                            },
                            default_boot_always_attempt: uefi_options.default_boot_always_attempt,
                        }
                    },
                    com1: with_vmbus_com1_serial,
                    com2: with_vmbus_com2_serial,
                    serial_tx_only: opt.serial_tx_only,
                    vtl2_settings: Some(prost::Message::encode_to_vec(&vtl2_settings)),
                    vmbus_redirection: opt.vmbus_redirect,
                    vmgs,
                    framebuffer: opt
                        .vtl2_gfx
                        .then(|| SharedFramebufferHandle.into_resource()),
                    guest_request_recv,
                    tpm_version: tpm_version.map(|v| match v {
                        TpmVersion::V138 => get_resources::ged::GedTpmVersion::V138,
                        TpmVersion::V185 => get_resources::ged::GedTpmVersion::V185,
                    }),
                    firmware_event_send: None,
                    ipmi_sel_event_send: None,
                    secure_boot_enabled: opt.secure_boot,
                    secure_boot_template: match opt.secure_boot_template {
                        Some(SecureBootTemplateCli::Windows) => {
                            get_resources::ged::GuestSecureBootTemplateType::MicrosoftWindows
                        },
                        Some(SecureBootTemplateCli::UefiCa) => {
                            get_resources::ged::GuestSecureBootTemplateType::MicrosoftUefiCertificateAuthority
                        }
                        None => {
                            get_resources::ged::GuestSecureBootTemplateType::None
                        },
                    },
                    enable_battery: opt.battery,
                    enable_ipmi: false,
                    enable_hibernation: opt.hibernation,
                    no_persistent_secrets: true,
                    igvm_attest_test_config: None,
                    test_gsp_by_id: opt.test_gsp_by_id,
                    efi_diagnostics_log_level: {
                        match uefi_options.diagnostics.unwrap_or_default() {
                            EfiDiagnosticsLogLevelCli::Default => get_resources::ged::EfiDiagnosticsLogLevelType::Default,
                            EfiDiagnosticsLogLevelCli::Info => get_resources::ged::EfiDiagnosticsLogLevelType::Info,
                            EfiDiagnosticsLogLevelCli::Full => get_resources::ged::EfiDiagnosticsLogLevelType::Full,
                        }
                    },
                    force_dma_bounce_enabled: uefi_options.force_dma_bounce,
                    smbios: ged_smbios,
                }
                .into_resource(),
            ),
        ]);
    }

    if let Some(tpm_version) = tpm_version
        && !opt.vtl2
    {
        let register_layout = if cfg!(guest_arch = "x86_64") {
            TpmRegisterLayout::IoPort
        } else {
            TpmRegisterLayout::Mmio
        };

        let (ppi_store, nvram_store) = if opt.vmgs.is_some() {
            (
                VmgsFileHandle::new(vmgs_format::FileId::TPM_PPI, true).into_resource(),
                VmgsFileHandle::new(tpm_vmgs::tpm_nvram_file_id(tpm_version), true).into_resource(),
            )
        } else {
            (
                EphemeralNonVolatileStoreHandle.into_resource(),
                EphemeralNonVolatileStoreHandle.into_resource(),
            )
        };

        chipset_devices.push(ChipsetDeviceHandle {
            name: "tpm".to_string(),
            resource: chipset_device_worker_defs::RemoteChipsetDeviceHandle {
                device: TpmDeviceHandle {
                    version: tpm_version,
                    ppi_store,
                    nvram_store,
                    nvram_size: None,
                    refresh_tpm_seeds: false,
                    ak_cert_type: tpm_resources::TpmAkCertTypeResource::None,
                    register_layout,
                    guest_secret_key: None,
                    logger: None,
                    is_confidential_vm: false,
                    bios_guid,
                }
                .into_resource(),
                worker_host: mesh.make_host("tpm", None).await?,
            }
            .into_resource(),
        });
    }

    let vga_firmware = if opt.pcat {
        Some(openvmm_pcat_locator::find_svga_bios(
            opt.vga_firmware.as_deref(),
        )?)
    } else {
        None
    };

    if opt.gfx {
        // Channel for the video device to report dirty rectangles to the VNC worker.
        let (dirt_send, dirt_recv) = mesh::channel();
        resources.dirty_rect_recv = Some(dirt_recv);

        vmbus_devices.extend([
            (
                DeviceVtl::Vtl0,
                SynthVideoHandle {
                    framebuffer: SharedFramebufferHandle.into_resource(),
                    dirt_send: Some(dirt_send),
                }
                .into_resource(),
            ),
            (
                DeviceVtl::Vtl0,
                SynthKeyboardHandle {
                    source: MultiplexedInputHandle {
                        // Save 0 for PS/2
                        elevation: 1,
                    }
                    .into_resource(),
                }
                .into_resource(),
            ),
            (
                DeviceVtl::Vtl0,
                SynthMouseHandle {
                    source: MultiplexedInputHandle {
                        // Save 0 for PS/2
                        elevation: 1,
                    }
                    .into_resource(),
                }
                .into_resource(),
            ),
        ]);
    }

    let vsock_listener = |path: Option<&str>| -> anyhow::Result<_> {
        if let Some(path) = path {
            cleanup_socket(path.as_ref());
            let listener = unix_socket::UnixListener::bind(path)
                .with_context(|| format!("failed to bind to hybrid vsock path: {}", path))?;
            Ok(Some(listener))
        } else {
            Ok(None)
        }
    };

    let vtl0_vsock_listener = vsock_listener(opt.vmbus_vsock_path.as_deref())?;
    let vtl2_vsock_listener = vsock_listener(opt.vmbus_vtl2_vsock_path.as_deref())?;

    if let Some(path) = &opt.openhcl_dump_path {
        let (resource, task) = spawn_dump_handler(&spawner, path.clone(), None);
        task.detach();
        vmbus_devices.push((openhcl_vtl, resource));
    }

    #[cfg(guest_arch = "aarch64")]
    let topology_arch = openvmm_defs::config::ArchTopologyConfig::Aarch64(
        openvmm_defs::config::Aarch64TopologyConfig {
            // TODO: allow this to be configured from the command line
            gic_config: None,
            pmu_gsiv: openvmm_defs::config::PmuGsivConfig::Platform,
            gic_msi: match opt.gic_msi {
                cli_args::GicMsiCli::Auto => openvmm_defs::config::GicMsiConfig::Auto,
                cli_args::GicMsiCli::Its => openvmm_defs::config::GicMsiConfig::Its,
                cli_args::GicMsiCli::V2m => {
                    openvmm_defs::config::GicMsiConfig::V2m { spi_count: None }
                }
            },
        },
    );
    #[cfg(guest_arch = "x86_64")]
    let topology_arch =
        openvmm_defs::config::ArchTopologyConfig::X86(openvmm_defs::config::X86TopologyConfig {
            apic_id_offset: opt.apic_id_offset,
            x2apic: opt.x2apic,
        });

    let with_isolation = if let Some(isolation) = &opt.isolation {
        match isolation {
            cli_args::IsolationCli::Vbs => {
                // TODO: For now, VBS isolation is only supported with VTL2.
                if !opt.vtl2 {
                    anyhow::bail!("VBS isolation is only currently supported with vtl2");
                }

                // TODO: Alias map support is not yet implemented with isolation.
                if !opt.no_alias_map {
                    anyhow::bail!("alias map not supported with isolation");
                }

                Some(openvmm_defs::config::IsolationType::Vbs)
            }
            cli_args::IsolationCli::Snp => Some(openvmm_defs::config::IsolationType::Snp {
                // SNP host data is currently only configurable via TTRPC.
                host_data: None,
            }),
        }
    } else {
        None
    };

    if with_hv && !opt.no_vmbus {
        let (shutdown_send, shutdown_recv) = mesh::channel();
        resources.shutdown_ic = Some(shutdown_send);
        let (kvp_send, kvp_recv) = mesh::channel();
        resources.kvp_ic = Some(kvp_send);
        vmbus_devices.extend(
            [
                hyperv_ic_resources::shutdown::ShutdownIcHandle {
                    recv: shutdown_recv,
                }
                .into_resource(),
                hyperv_ic_resources::kvp::KvpIcHandle { recv: kvp_recv }.into_resource(),
                hyperv_ic_resources::timesync::TimesyncIcHandle.into_resource(),
            ]
            .map(|r| (DeviceVtl::Vtl0, r)),
        );
    }

    if let Some(hive_path) = &opt.imc {
        let file = fs_err::File::open(hive_path).context("failed to open imc hive")?;
        vmbus_devices.push((
            DeviceVtl::Vtl0,
            vmbfs_resources::VmbfsImcDeviceHandle { file: file.into() }.into_resource(),
        ));
    }

    let mut virtio_devices = Vec::new();
    let mut add_virtio_device =
        |bus, resource: Resource<VirtioDeviceHandle>, pcie_devices: &mut Vec<_>| match bus {
            VirtioBusCli::Auto => {
                // Use VPCI when possible (currently only on Windows and macOS due
                // to KVM backend limitations).
                if with_hv && (cfg!(windows) || cfg!(target_os = "macos")) {
                    vpci_devices.push(VpciDeviceConfig {
                        vtl: DeviceVtl::Vtl0,
                        instance_id: Guid::new_random(),
                        resource: VirtioPciDeviceHandle(resource).into_resource(),
                        vnode: None,
                    });
                } else {
                    virtio_devices.push((VirtioBus::Pci, resource));
                }
            }
            VirtioBusCli::Mmio => virtio_devices.push((VirtioBus::Mmio, resource)),
            VirtioBusCli::Pci => virtio_devices.push((VirtioBus::Pci, resource)),
            VirtioBusCli::Pcie(port_name) => pcie_devices.push(PcieDeviceConfig {
                port_name,
                resource: VirtioPciDeviceHandle(resource).into_resource(),
            }),
            VirtioBusCli::Vpci => vpci_devices.push(VpciDeviceConfig {
                vtl: DeviceVtl::Vtl0,
                instance_id: Guid::new_random(),
                resource: VirtioPciDeviceHandle(resource).into_resource(),
                vnode: None,
            }),
        };

    for cli_cfg in &opt.virtio_net {
        if cli_cfg.underhill {
            anyhow::bail!("use --net uh:[...] to add underhill NICs")
        }
        let vport = parse_endpoint(cli_cfg, &mut nic_index, &mut resources)?;
        let resource = virtio_resources::net::VirtioNetHandle {
            max_queues: vport.max_queues,
            mac_address: vport.mac_address,
            endpoint: vport.endpoint,
        }
        .into_resource();
        if let Some(pcie_port) = &cli_cfg.pcie_port {
            pcie_devices.push(PcieDeviceConfig {
                port_name: pcie_port.clone(),
                resource: VirtioPciDeviceHandle(resource).into_resource(),
            });
        } else {
            add_virtio_device(VirtioBusCli::Auto, resource, &mut pcie_devices);
        }
    }

    for args in &opt.virtio_fs {
        let resource: Resource<VirtioDeviceHandle> = virtio_resources::fs::VirtioFsHandle {
            tag: args.tag.clone(),
            fs: virtio_resources::fs::VirtioFsBackend::HostFs {
                root_path: args.path.clone(),
                mount_options: args.options.clone(),
            },
        }
        .into_resource();
        if let Some(pcie_port) = &args.pcie_port {
            pcie_devices.push(PcieDeviceConfig {
                port_name: pcie_port.clone(),
                resource: VirtioPciDeviceHandle(resource).into_resource(),
            });
        } else {
            add_virtio_device(opt.virtio_fs_bus.clone(), resource, &mut pcie_devices);
        }
    }

    for args in &opt.virtio_fs_shmem {
        let resource: Resource<VirtioDeviceHandle> = virtio_resources::fs::VirtioFsHandle {
            tag: args.tag.clone(),
            fs: virtio_resources::fs::VirtioFsBackend::SectionFs {
                root_path: args.path.clone(),
            },
        }
        .into_resource();
        if let Some(pcie_port) = &args.pcie_port {
            pcie_devices.push(PcieDeviceConfig {
                port_name: pcie_port.clone(),
                resource: VirtioPciDeviceHandle(resource).into_resource(),
            });
        } else {
            add_virtio_device(opt.virtio_fs_bus.clone(), resource, &mut pcie_devices);
        }
    }

    for args in &opt.virtio_9p {
        let resource: Resource<VirtioDeviceHandle> = virtio_resources::p9::VirtioPlan9Handle {
            tag: args.tag.clone(),
            root_path: args.path.clone(),
            debug: opt.virtio_9p_debug,
        }
        .into_resource();
        if let Some(pcie_port) = &args.pcie_port {
            pcie_devices.push(PcieDeviceConfig {
                port_name: pcie_port.clone(),
                resource: VirtioPciDeviceHandle(resource).into_resource(),
            });
        } else {
            add_virtio_device(VirtioBusCli::Auto, resource, &mut pcie_devices);
        }
    }

    if let Some(pmem_args) = &opt.virtio_pmem {
        let resource: Resource<VirtioDeviceHandle> = virtio_resources::pmem::VirtioPmemHandle {
            path: pmem_args.path.clone(),
        }
        .into_resource();
        if let Some(pcie_port) = &pmem_args.pcie_port {
            pcie_devices.push(PcieDeviceConfig {
                port_name: pcie_port.clone(),
                resource: VirtioPciDeviceHandle(resource).into_resource(),
            });
        } else {
            add_virtio_device(VirtioBusCli::Auto, resource, &mut pcie_devices);
        }
    }

    if opt.virtio_rng {
        let resource: Resource<VirtioDeviceHandle> =
            virtio_resources::rng::VirtioRngHandle.into_resource();
        if let Some(pcie_port) = &opt.virtio_rng_pcie_port {
            pcie_devices.push(PcieDeviceConfig {
                port_name: pcie_port.clone(),
                resource: VirtioPciDeviceHandle(resource).into_resource(),
            });
        } else {
            add_virtio_device(opt.virtio_rng_bus.clone(), resource, &mut pcie_devices);
        }
    }

    if let Some(rtc) = &opt.virtio_rtc {
        let resource: Resource<VirtioDeviceHandle> =
            virtio_resources::rtc::VirtioRtcHandle.into_resource();
        add_virtio_device(rtc.bus.clone(), resource, &mut pcie_devices);
    }

    if let Some(backend) = virtio_console_backend {
        let resource: Resource<VirtioDeviceHandle> =
            virtio_resources::console::VirtioConsoleHandle { backend }.into_resource();
        if let Some(pcie_port) = &opt.virtio_console_pcie_port {
            pcie_devices.push(PcieDeviceConfig {
                port_name: pcie_port.clone(),
                resource: VirtioPciDeviceHandle(resource).into_resource(),
            });
        } else {
            add_virtio_device(VirtioBusCli::Auto, resource, &mut pcie_devices);
        }
    }

    // Handle --vhost-user arguments.
    #[cfg(target_os = "linux")]
    for vhost_cli in &opt.vhost_user {
        let stream =
            unix_socket::UnixStream::connect(&vhost_cli.socket_path).with_context(|| {
                format!(
                    "failed to connect to vhost-user socket: {}",
                    vhost_cli.socket_path
                )
            })?;

        use crate::cli_args::VhostUserDeviceTypeCli;
        let resource: Resource<VirtioDeviceHandle> = match vhost_cli.device_type {
            VhostUserDeviceTypeCli::Fs {
                ref tag,
                num_queues,
                queue_size,
            } => virtio_resources::vhost_user::VhostUserFsHandle {
                socket: stream.into(),
                tag: tag.clone(),
                num_queues,
                queue_size,
            }
            .into_resource(),
            VhostUserDeviceTypeCli::Blk {
                num_queues,
                queue_size,
            } => virtio_resources::vhost_user::VhostUserBlkHandle {
                socket: stream.into(),
                num_queues,
                queue_size,
            }
            .into_resource(),
            VhostUserDeviceTypeCli::Other {
                device_id,
                ref queue_sizes,
            } => virtio_resources::vhost_user::VhostUserGenericHandle {
                socket: stream.into(),
                device_id,
                queue_sizes: queue_sizes.clone(),
            }
            .into_resource(),
        };
        if let Some(pcie_port) = &vhost_cli.pcie_port {
            pcie_devices.push(PcieDeviceConfig {
                port_name: pcie_port.clone(),
                resource: VirtioPciDeviceHandle(resource).into_resource(),
            });
        } else {
            add_virtio_device(VirtioBusCli::Auto, resource, &mut pcie_devices);
        }
    }

    let virtio_vsock_bus = opt.virtio_vsock_bus.clone().unwrap_or(VirtioBusCli::Auto);

    if let Some(vsock_path) = &opt.virtio_vsock_path {
        let listener = vsock_listener(Some(vsock_path))?.unwrap();
        let resource: Resource<VirtioDeviceHandle> = virtio_resources::vsock::VirtioVsockHandle {
            // The guest CID does not matter since the UDS relay does not use it. It just needs
            // to be some non-reserved value for the guest to use.
            guest_cid: 0x3,
            base_path: vsock_path.clone(),
            listener,
        }
        .into_resource();
        add_virtio_device(virtio_vsock_bus.clone(), resource, &mut pcie_devices);
    }

    #[cfg(target_os = "linux")]
    if let Some(guest_cid) = opt.virtio_vsock_vhost_cid {
        let vhost = std::fs::OpenOptions::new()
            .read(true)
            .write(true)
            .open("/dev/vhost-vsock")
            .context("failed to open /dev/vhost-vsock")?
            .into();
        let resource =
            virtio_resources::vsock::VirtioVsockVhostHandle { vhost, guest_cid }.into_resource();
        add_virtio_device(virtio_vsock_bus, resource, &mut pcie_devices);
    }

    #[cfg(target_os = "linux")]
    pcie_devices.extend(vfio_pcie_devices);

    let mut cfg = Config {
        chipset,
        load_mode,
        floppy_disks,
        pcie_root_complexes,
        pcie_ecam_below_4gb: opt.pcie_ecam_below_4gb,
        #[cfg(target_os = "linux")]
        pcie_devices,
        #[cfg(not(target_os = "linux"))]
        pcie_devices,
        pcie_switches,
        pcie_generic_initiators,
        vpci_devices,
        ide_disks: Vec::new(),
        numa: {
            if let Some(ref nodes) = opt.numa {
                // --numa mode: each --numa flag defines a node.
                NumaTopology {
                    nodes: nodes
                        .iter()
                        .map(|n| {
                            let vps = match &n.vps {
                                Some(vps) if vps.0.is_empty() => VpAssignment::Empty,
                                Some(vps) => {
                                    VpAssignment::Explicit(vps.expand_below(opt.processors)?)
                                }
                                None => VpAssignment::FromTopology,
                            };
                            Ok(NumaNode {
                                mem: Some(MemoryConfig {
                                    mem_size: n
                                        .memory
                                        .size
                                        .expect("NUMA memory size was validated")
                                        .0,
                                    prefetch_memory: n.memory.prefetch,
                                    private_memory: n.memory.shared == Some(false),
                                    transparent_hugepages: n
                                        .memory
                                        .transparent_hugepages
                                        .unwrap_or(!n.memory.hugepages),
                                    hugepages: n.memory.hugepages,
                                    hugepage_size: n.memory.hugepage_size.map(|m| m.0),
                                    host_numa_node: n.host_numa_node,
                                }),
                                vps,
                            })
                        })
                        .collect::<anyhow::Result<Vec<_>>>()?,
                    distances: opt
                        .numa_distance
                        .as_deref()
                        .unwrap_or(&[])
                        .iter()
                        .map(|d| NumaDistance {
                            src: d.src,
                            dst: d.dst,
                            distance: d.distance,
                        })
                        .collect(),
                }
            } else {
                // Single-node default from --memory.
                NumaTopology {
                    nodes: vec![NumaNode {
                        mem: Some(MemoryConfig {
                            mem_size: opt.memory_size(),
                            prefetch_memory: opt.prefetch_memory(),
                            private_memory: opt.private_memory(),
                            transparent_hugepages: opt.transparent_hugepages(),
                            hugepages: opt.memory.hugepages,
                            hugepage_size: opt.memory.hugepage_size.map(|m| m.0),
                            host_numa_node: None,
                        }),
                        vps: VpAssignment::FromTopology,
                    }],
                    distances: vec![],
                }
            }
        },
        processor_topology: ProcessorTopologyConfig {
            proc_count: opt.processors,
            vps_per_socket: opt.vps_per_socket,
            enable_smt: match opt.smt {
                cli_args::SmtConfigCli::Auto => None,
                cli_args::SmtConfigCli::Force => Some(true),
                cli_args::SmtConfigCli::Off => Some(false),
            },
            arch: Some(topology_arch),
        },
        hypervisor: HypervisorConfig {
            with_hv,
            with_vtl2: opt.vtl2.then_some(Vtl2Config {
                vtl0_alias_map: !opt.no_alias_map,
                late_map_vtl0_memory: match opt.late_map_vtl0_policy {
                    cli_args::Vtl0LateMapPolicyCli::Off => None,
                    cli_args::Vtl0LateMapPolicyCli::Log => Some(LateMapVtl0MemoryPolicy::Log),
                    cli_args::Vtl0LateMapPolicyCli::Halt => Some(LateMapVtl0MemoryPolicy::Halt),
                    cli_args::Vtl0LateMapPolicyCli::Exception => {
                        Some(LateMapVtl0MemoryPolicy::InjectException)
                    }
                },
            }),
            with_isolation,
            nested_virt: opt.nested_virt,
        },
        #[cfg(windows)]
        kernel_vmnics,
        input: mesh::Receiver::new(),
        framebuffer,
        vga_firmware,
        vtl2_gfx: opt.vtl2_gfx,
        virtio_devices,
        vmbus: (with_hv && !opt.no_vmbus).then_some(VmbusConfig {
            vsock_listener: vtl0_vsock_listener,
            vsock_path: opt.vmbus_vsock_path.clone(),
            vtl2_redirect: opt.vmbus_redirect,
            vmbus_max_version: opt.vmbus_max_version,
            #[cfg(windows)]
            vmbusproxy_handle,
        }),
        vtl2_vmbus: (with_hv && opt.vtl2).then_some(VmbusConfig {
            vsock_listener: vtl2_vsock_listener,
            vsock_path: opt.vmbus_vtl2_vsock_path.clone(),
            ..Default::default()
        }),
        vmbus_devices,
        chipset_devices,
        pci_chipset_devices,
        isa_dma_controller,
        chipset_capabilities: capabilities,
        layout: layout_config,
        #[cfg(windows)]
        vpci_resources,
        vmgs,
        firmware_event_send: None,
        debugger_rpc: None,
        rtc_delta_milliseconds: 0,
    };

    storage.build_config(&mut cfg, &mut resources, opt.scsi_sub_channels)?;
    let mut pcie_port_names = HashSet::new();
    for device in &cfg.pcie_devices {
        anyhow::ensure!(
            pcie_port_names.insert(&device.port_name),
            "multiple devices use PCIe port '{}'",
            device.port_name
        );
    }
    resources.serial_driver = Some(serial_driver);
    validate_snp_config(&cfg)?;
    Ok((cfg, resources))
}

fn validate_snp_config(cfg: &Config) -> anyhow::Result<()> {
    if !matches!(
        cfg.hypervisor.with_isolation,
        Some(openvmm_defs::config::IsolationType::Snp { .. })
    ) {
        return Ok(());
    }

    if !matches!(
        cfg.load_mode,
        LoadMode::Linux { .. } | LoadMode::Igvm { .. }
    ) {
        anyhow::bail!("SNP isolation currently only supports Linux direct or IGVM boot");
    }
    if cfg.hypervisor.with_vtl2.is_some() {
        anyhow::bail!("SNP isolation currently does not support VTL2");
    }
    if cfg.vmbus.is_some() || cfg.vtl2_vmbus.is_some() || !cfg.vmbus_devices.is_empty() {
        anyhow::bail!("SNP isolation currently does not support VMBus devices");
    }

    let only_supported_chipset_devices = cfg.chipset_devices.iter().all(|device| {
        matches!(
            device.resource.id(),
            "serial_16550"
                | "pic"
                | "pit"
                | "generic-ioapic"
                | "hyperv_power_management"
                | "missing-dev"
        )
    });
    let only_virtio_pcie_devices = cfg
        .pcie_devices
        .iter()
        .all(|device| device.resource.id() == "virtio");
    if !cfg.floppy_disks.is_empty()
        || !cfg.ide_disks.is_empty()
        || !cfg.virtio_devices.is_empty()
        || !only_virtio_pcie_devices
        || !cfg.vpci_devices.is_empty()
        || !only_supported_chipset_devices
        || !cfg.pci_chipset_devices.is_empty()
    {
        anyhow::bail!("SNP isolation currently only supports virtio devices");
    }
    if cfg.framebuffer.is_some() || cfg.vga_firmware.is_some() || cfg.debugger_rpc.is_some() {
        anyhow::bail!("SNP isolation currently does not support this VM configuration");
    }

    Ok(())
}

/// Gets the terminal to use for externally launched console windows.
pub(crate) fn openvmm_terminal_app() -> Option<PathBuf> {
    std::env::var_os("OPENVMM_TERM")
        .or_else(|| std::env::var_os("HVLITE_TERM"))
        .map(Into::into)
}

// Tries to remove `path` if it is confirmed to be a Unix socket.
fn cleanup_socket(path: &Path) {
    #[cfg(windows)]
    let is_socket = pal::windows::fs::is_unix_socket(path).unwrap_or(false);
    #[cfg(not(windows))]
    let is_socket = path
        .metadata()
        .is_ok_and(|meta| std::os::unix::fs::FileTypeExt::is_socket(&meta.file_type()));

    if is_socket {
        let _ = std::fs::remove_file(path);
    }
}

#[cfg(windows)]
fn new_switch_port(
    switch_id: Option<&str>,
) -> anyhow::Result<(
    openvmm_defs::config::SwitchPortId,
    vmswitch::kernel::SwitchPort,
)> {
    let id = vmswitch::kernel::SwitchPortId {
        switch: match switch_id {
            Some(s) => s.parse().context("invalid switch id")?,
            None => vmswitch::hcn::DEFAULT_SWITCH,
        },
        port: Guid::new_random(),
    };
    let _ = vmswitch::hcn::Network::open(&id.switch)
        .with_context(|| format!("could not find switch {}", id.switch))?;

    let port = vmswitch::kernel::SwitchPort::new(&id).context("failed to create switch port")?;

    let id = openvmm_defs::config::SwitchPortId {
        switch: id.switch,
        port: id.port,
    };
    Ok((id, port))
}

fn parse_endpoint(
    cli_cfg: &NicConfigCli,
    index: &mut usize,
    resources: &mut VmResources,
) -> anyhow::Result<NicConfig> {
    let _ = resources;
    let endpoint = match &cli_cfg.endpoint {
        EndpointConfigCli::Consomme { cidr, host_fwd } => {
            let ports = host_fwd
                .iter()
                .map(|fwd| {
                    use net_backend_resources::consomme::HostPortProtocol;
                    net_backend_resources::consomme::HostPortConfig {
                        protocol: match fwd.protocol {
                            cli_args::HostPortProtocolCli::Tcp => HostPortProtocol::Tcp,
                            cli_args::HostPortProtocolCli::Udp => HostPortProtocol::Udp,
                        },
                        host_address: fwd
                            .host_address
                            .map(net_backend_resources::consomme::HostIpAddress::from),
                        host_port: net_backend_resources::consomme::HostPort::Fixed(fwd.host_port),
                        guest_port: fwd.guest_port,
                    }
                })
                .collect();
            // Only wire the bind/unbind RPC channel to the first consomme
            // endpoint. Additional consomme NICs work normally but cannot be
            // targeted by runtime bind/unbind commands.
            let recv = if resources.consomme_rpc.is_none() {
                let (send, recv) = mesh::channel();
                resources.consomme_rpc = Some(send);
                Some(recv)
            } else {
                None
            };
            net_backend_resources::consomme::ConsommeHandle {
                cidr: cidr.clone(),
                ports,
                recv,
            }
            .into_resource()
        }
        EndpointConfigCli::None => net_backend_resources::null::NullHandle.into_resource(),
        EndpointConfigCli::Dio { id } => {
            #[cfg(windows)]
            {
                let (port_id, port) = new_switch_port(id.as_deref())?;
                resources.switch_ports.push(port);
                net_backend_resources::dio::WindowsDirectIoHandle {
                    switch_port_id: net_backend_resources::dio::SwitchPortId {
                        switch: port_id.switch,
                        port: port_id.port,
                    },
                }
                .into_resource()
            }

            #[cfg(not(windows))]
            {
                let _ = id;
                bail!("cannot use dio on non-windows platforms")
            }
        }
        EndpointConfigCli::Tap { name } => {
            #[cfg(target_os = "linux")]
            {
                let fd = net_tap::tap::open_tap(name)
                    .with_context(|| format!("failed to open TAP device '{name}'"))?;
                net_backend_resources::tap::TapHandle { fd }.into_resource()
            }

            #[cfg(not(target_os = "linux"))]
            {
                let _ = name;
                bail!("TAP backend is only supported on Linux")
            }
        }
    };

    // Pick a random MAC address.
    let mut mac_address = [0x00, 0x15, 0x5D, 0, 0, 0];
    getrandom::fill(&mut mac_address[3..]).expect("rng failure");

    // Pick a fixed instance ID based on the index.
    const BASE_INSTANCE_ID: Guid = guid::guid!("00000000-da43-11ed-936a-00155d6db52f");
    let instance_id = Guid {
        data1: *index as u32,
        ..BASE_INSTANCE_ID
    };
    *index += 1;

    Ok(NicConfig {
        vtl: cli_cfg.vtl,
        instance_id,
        endpoint,
        mac_address: mac_address.into(),
        max_queues: cli_cfg.max_queues,
        pcie_port: cli_cfg.pcie_port.clone(),
    })
}

#[derive(Debug)]
struct NicConfig {
    vtl: DeviceVtl,
    instance_id: Guid,
    mac_address: MacAddress,
    endpoint: Resource<NetEndpointHandleKind>,
    max_queues: Option<u16>,
    pcie_port: Option<String>,
}

impl NicConfig {
    fn into_netvsp_handle(self) -> (DeviceVtl, Resource<VmbusDeviceHandleKind>) {
        (
            self.vtl,
            netvsp_resources::NetvspHandle {
                instance_id: self.instance_id,
                mac_address: self.mac_address,
                endpoint: self.endpoint,
                max_queues: self.max_queues,
            }
            .into_resource(),
        )
    }
}

enum LayerOrDisk {
    Layer(DiskLayerDescription),
    Disk(Resource<DiskHandleKind>),
}

async fn disk_open(
    disk_cli: &DiskCliKind,
    read_only: bool,
) -> anyhow::Result<Resource<DiskHandleKind>> {
    let mut layers = Vec::new();
    disk_open_inner(disk_cli, read_only, &mut layers).await?;
    if layers.len() == 1 && matches!(layers[0], LayerOrDisk::Disk(_)) {
        let LayerOrDisk::Disk(disk) = layers.pop().unwrap() else {
            unreachable!()
        };
        Ok(disk)
    } else {
        Ok(Resource::new(disk_backend_resources::LayeredDiskHandle {
            layers: layers
                .into_iter()
                .map(|layer| match layer {
                    LayerOrDisk::Layer(layer) => layer,
                    LayerOrDisk::Disk(disk) => DiskLayerDescription {
                        layer: DiskLayerHandle(disk).into_resource(),
                        read_cache: false,
                        write_through: false,
                    },
                })
                .collect(),
        }))
    }
}

fn disk_open_inner<'a>(
    disk_cli: &'a DiskCliKind,
    read_only: bool,
    layers: &'a mut Vec<LayerOrDisk>,
) -> futures::future::BoxFuture<'a, anyhow::Result<()>> {
    Box::pin(async move {
        fn layer<T: IntoResource<DiskLayerHandleKind>>(layer: T) -> LayerOrDisk {
            LayerOrDisk::Layer(layer.into_resource().into())
        }
        fn disk<T: IntoResource<DiskHandleKind>>(disk: T) -> LayerOrDisk {
            LayerOrDisk::Disk(disk.into_resource())
        }
        match disk_cli {
            &DiskCliKind::Memory(len) => {
                layers.push(layer(RamDiskLayerHandle {
                    len: Some(len),
                    sector_size: None,
                }));
            }
            DiskCliKind::File {
                path,
                create_with_len,
                direct,
            } => layers.push(LayerOrDisk::Disk(if let Some(size) = create_with_len {
                create_disk_type(
                    path,
                    *size,
                    OpenDiskOptions {
                        read_only: false,
                        direct: *direct,
                    },
                )
                .with_context(|| format!("failed to create {}", path.display()))?
            } else {
                open_disk_type(
                    path,
                    OpenDiskOptions {
                        read_only,
                        direct: *direct,
                    },
                )
                .await
                .with_context(|| format!("failed to open {}", path.display()))?
            })),
            DiskCliKind::Blob { kind, url } => {
                layers.push(disk(disk_backend_resources::BlobDiskHandle {
                    url: url.to_owned(),
                    format: match kind {
                        cli_args::BlobKind::Flat => disk_backend_resources::BlobDiskFormat::Flat,
                        cli_args::BlobKind::Vhd1 => {
                            disk_backend_resources::BlobDiskFormat::FixedVhd1
                        }
                    },
                }))
            }
            DiskCliKind::MemoryDiff(inner) => {
                layers.push(layer(RamDiskLayerHandle {
                    len: None,
                    sector_size: None,
                }));
                disk_open_inner(inner, true, layers).await?;
            }
            DiskCliKind::PersistentReservationsWrapper(inner) => {
                layers.push(disk(disk_backend_resources::DiskWithReservationsHandle(
                    disk_open(inner, read_only).await?,
                )))
            }
            DiskCliKind::DelayDiskWrapper {
                delay_ms,
                disk: inner,
            } => layers.push(disk(DelayDiskHandle {
                delay: CellUpdater::new(Duration::from_millis(*delay_ms)).cell(),
                disk: disk_open(inner, read_only).await?,
            })),
            DiskCliKind::Crypt {
                disk: inner,
                cipher,
                key_file,
            } => layers.push(disk(disk_crypt_resources::DiskCryptHandle {
                disk: disk_open(inner, read_only).await?,
                cipher: match cipher {
                    cli_args::DiskCipher::XtsAes256 => disk_crypt_resources::Cipher::XtsAes256,
                },
                key: fs_err::read(key_file).context("failed to read key file")?,
            })),
            DiskCliKind::Sqlite {
                path,
                create_with_len,
            } => {
                // FUTURE: this code should be responsible for opening
                // file-handle(s) itself, and passing them into sqlite via a custom
                // vfs. For now though - simply check if the file exists or not, and
                // perform early validation of filesystem-level create options.
                match (create_with_len.is_some(), path.exists()) {
                    (true, true) => anyhow::bail!(
                        "cannot create new sqlite disk at {} - file already exists",
                        path.display()
                    ),
                    (false, false) => anyhow::bail!(
                        "cannot open sqlite disk at {} - file not found",
                        path.display()
                    ),
                    _ => {}
                }

                layers.push(layer(SqliteDiskLayerHandle {
                    dbhd_path: path.display().to_string(),
                    format_dbhd: create_with_len.map(|len| {
                        disk_backend_resources::layer::SqliteDiskLayerFormatParams {
                            logically_read_only: false,
                            len: Some(len),
                        }
                    }),
                }));
            }
            DiskCliKind::SqliteDiff { path, create, disk } => {
                // FUTURE: this code should be responsible for opening
                // file-handle(s) itself, and passing them into sqlite via a custom
                // vfs. For now though - simply check if the file exists or not, and
                // perform early validation of filesystem-level create options.
                match (create, path.exists()) {
                    (true, true) => anyhow::bail!(
                        "cannot create new sqlite disk at {} - file already exists",
                        path.display()
                    ),
                    (false, false) => anyhow::bail!(
                        "cannot open sqlite disk at {} - file not found",
                        path.display()
                    ),
                    _ => {}
                }

                layers.push(layer(SqliteDiskLayerHandle {
                    dbhd_path: path.display().to_string(),
                    format_dbhd: create.then_some(
                        disk_backend_resources::layer::SqliteDiskLayerFormatParams {
                            logically_read_only: false,
                            len: None,
                        },
                    ),
                }));
                disk_open_inner(disk, true, layers).await?;
            }
            DiskCliKind::AutoCacheSqlite {
                cache_path,
                key,
                disk,
            } => {
                layers.push(LayerOrDisk::Layer(DiskLayerDescription {
                    read_cache: true,
                    write_through: false,
                    layer: SqliteAutoCacheDiskLayerHandle {
                        cache_path: cache_path.clone(),
                        cache_key: key.clone(),
                    }
                    .into_resource(),
                }));
                disk_open_inner(disk, read_only, layers).await?;
            }
        }
        Ok(())
    })
}

/// Get the system page size.
pub(crate) fn system_page_size() -> u32 {
    sparse_mmap::SparseMapping::page_size() as u32
}

/// The guest architecture string, derived from the compile-time `guest_arch` cfg.
pub(crate) const GUEST_ARCH: &str = if cfg!(guest_arch = "x86_64") {
    "x86_64"
} else {
    "aarch64"
};

/// Open a snapshot directory and validate it against the current VM config.
/// Returns the shared memory fd (from memory.bin) and the saved device state.
fn prepare_snapshot_restore(
    snapshot_dir: &Path,
    opt: &Options,
) -> anyhow::Result<(
    openvmm_defs::worker::SharedMemoryFd,
    mesh::payload::message::ProtobufMessage,
)> {
    let (manifest, state_bytes) = openvmm_helpers::snapshot::read_snapshot(snapshot_dir)?;

    // Validate manifest against current VM config.
    openvmm_helpers::snapshot::validate_manifest(
        &manifest,
        GUEST_ARCH,
        opt.memory_size(),
        opt.processors,
        system_page_size(),
    )?;

    // Open memory.bin (existing file, no create, no resize).
    let memory_file = fs_err::OpenOptions::new()
        .read(true)
        .write(true)
        .open(snapshot_dir.join("memory.bin"))?;

    // Validate file size matches expected memory size.
    let file_size = memory_file.metadata()?.len();
    if file_size != manifest.memory_size_bytes {
        anyhow::bail!(
            "memory.bin size ({file_size} bytes) doesn't match manifest ({} bytes)",
            manifest.memory_size_bytes,
        );
    }

    let shared_memory_fd =
        openvmm_helpers::shared_memory::file_to_shared_memory_fd(memory_file.into())?;

    // Reconstruct ProtobufMessage from the saved state bytes.
    // The save side wrote mesh::payload::encode(ProtobufMessage), so we decode
    // back to ProtobufMessage.
    let state_msg: mesh::payload::message::ProtobufMessage = mesh::payload::decode(&state_bytes)
        .context("failed to decode saved state from snapshot")?;

    Ok((shared_memory_fd, state_msg))
}

fn do_main(pidfile_guard: &mut Option<pidfile::Pidfile>) -> anyhow::Result<i32> {
    #[cfg(windows)]
    pal::windows::disable_hard_error_dialog();

    tracing_init::enable_tracing()?;

    // Try to run as a worker host.
    // On success the worker runs to completion and then exits the process (does
    // not return). Any worker host setup errors are return and bubbled up.
    meshworker::run_vmm_mesh_host()?;

    let opt = cli_args::parse_options();

    // Print the version number. This comes after argument parsing to not interfere
    // with --version and --help.
    tracing::info!(version = openvmm_build_info::get().version());

    if let Some(path) = &opt.write_saved_state_proto {
        mesh::payload::protofile::DescriptorWriter::new(vmcore::save_restore::saved_state_roots())
            .write_to_path(path)
            .context("failed to write protobuf descriptors")?;
        return Ok(0);
    }

    if let Some(ref path) = opt.pidfile {
        *pidfile_guard = Some(pidfile::Pidfile::new(path).context("failed to create pidfile")?);
    }

    if let Some(path) = opt.relay_console_path {
        let console_title = opt.relay_console_title.unwrap_or_default();
        return console_relay::relay_console(&path, console_title.as_str()).map(|()| 0);
    }

    #[cfg(any(feature = "grpc", feature = "ttrpc"))]
    {
        let rpc = opt
            .rpc
            .as_ref()
            .map(|rpc| {
                let transport = match rpc.transport {
                    cli_args::RpcTransportCli::Auto => ttrpc::RpcTransport::Auto,
                    cli_args::RpcTransportCli::Ttrpc => ttrpc::RpcTransport::Ttrpc,
                    cli_args::RpcTransportCli::Grpc => ttrpc::RpcTransport::Grpc,
                };
                (rpc.path.as_path(), transport)
            })
            .or_else(|| {
                opt.ttrpc
                    .as_deref()
                    .map(|p| (p, ttrpc::RpcTransport::Ttrpc))
            })
            .or_else(|| opt.grpc.as_deref().map(|p| (p, ttrpc::RpcTransport::Grpc)));

        if let Some((path, transport)) = rpc {
            return block_on(async {
                let _ = std::fs::remove_file(path);
                let listener =
                    unix_socket::UnixListener::bind(path).context("failed to bind to socket")?;

                // This is a local launch
                let mut handle =
                    mesh_worker::launch_local_worker::<ttrpc::TtrpcWorker>(ttrpc::Parameters {
                        listener,
                        transport,
                    })
                    .await?;

                tracing::info!(%transport, path = %path.display(), "listening");

                // Signal the parent process that the server is ready.
                pal::close_stdout().context("failed to close stdout")?;

                handle.join().await?;

                Ok(0)
            });
        }
    }

    DefaultPool::run_with(async |driver| run_control(&driver, opt).await)
}

fn new_hvsock_service_id(port: u32) -> Guid {
    // This GUID is an embedding of the AF_VSOCK port into an
    // AF_HYPERV service ID.
    Guid {
        data1: port,
        .."00000000-facb-11e6-bd58-64006a7986d3".parse().unwrap()
    }
}

async fn run_control(driver: &DefaultDriver, opt: Options) -> anyhow::Result<i32> {
    let mut mesh = Some(VmmMesh::new(&driver, opt.single_process)?);
    let result = run_control_inner(driver, &mut mesh, opt).await;
    // If setup failed before the mesh was handed to the controller, shut it
    // down so the child host process exits cleanly without noisy logs.
    if let Some(mesh) = mesh {
        mesh.shutdown().await;
    }
    result
}

async fn run_control_inner(
    driver: &DefaultDriver,
    mesh_slot: &mut Option<VmmMesh>,
    opt: Options,
) -> anyhow::Result<i32> {
    let mesh = mesh_slot.as_ref().unwrap();
    let (mut vm_config, mut resources) = vm_config_from_command_line(driver, mesh, &opt).await?;

    let mut vnc_worker = None;
    if opt.gfx || opt.vnc.vnc {
        // Parse the listen address. Try as a full SocketAddr (host:port) first;
        // fall back to a bare IP, using the configured port.
        let addr: std::net::SocketAddr = if let Ok(sa) =
            opt.vnc.vnc_listen.parse::<std::net::SocketAddr>()
        {
            sa
        } else {
            let ip: std::net::IpAddr = opt.vnc.vnc_listen.parse().with_context(|| {
                format!(
                    "invalid VNC listen address: {} (expected IP address or socket address like [::1]:5900)",
                    opt.vnc.vnc_listen
                )
            })?;
            std::net::SocketAddr::new(ip, opt.vnc.vnc_port)
        };

        let socket = socket2::Socket::new(
            if addr.is_ipv6() {
                socket2::Domain::IPV6
            } else {
                socket2::Domain::IPV4
            },
            socket2::Type::STREAM,
            None,
        )
        .with_context(|| format!("creating VNC socket for {}", addr))?;

        if addr.is_ipv6() {
            if let Err(e) = socket.set_only_v6(false) {
                tracing::warn!(
                    error = %e,
                    "failed to enable dual-stack on IPv6 VNC socket, IPv4 clients may not be able to connect"
                );
            }
        }
        socket.set_reuse_address(true)?;
        socket
            .bind(&addr.into())
            .with_context(|| format!("binding VNC socket to {}", addr))?;
        socket
            .listen(128)
            .with_context(|| format!("listening on VNC socket {}", addr))?;
        let listener: TcpListener = socket.into();

        if !addr.ip().is_loopback() {
            tracing::warn!(
                address = %addr,
                "VNC server listening on non-localhost address without authentication"
            );
        }

        let input_send = vm_config.input.sender();
        let framebuffer = resources
            .framebuffer_access
            .take()
            .expect("synth video enabled");

        let vnc_host = mesh
            .make_host("vnc", None)
            .await
            .context("spawning vnc process failed")?;

        vnc_worker = Some(
            vnc_host
                .launch_worker(
                    vnc_worker_defs::VNC_WORKER_TCP,
                    VncParameters {
                        listener,
                        framebuffer,
                        input_send,
                        dirty_recv: resources.dirty_rect_recv.take(),
                        max_clients: opt.vnc.vnc_max_clients,
                        evict_oldest: opt.vnc.vnc_evict_oldest,
                    },
                )
                .await?,
        )
    }

    // spin up the debug worker
    let gdb_worker = if let Some(port) = opt.gdb {
        let listener = TcpListener::bind(format!("127.0.0.1:{}", port))
            .with_context(|| format!("binding to gdb port {}", port))?;

        let (req_tx, req_rx) = mesh::channel();
        vm_config.debugger_rpc = Some(req_rx);

        let gdb_host = mesh
            .make_host("gdb", None)
            .await
            .context("spawning gdbstub process failed")?;

        Some(
            gdb_host
                .launch_worker(
                    debug_worker_defs::DEBUGGER_WORKER,
                    debug_worker_defs::DebuggerParameters {
                        listener,
                        req_chan: req_tx,
                        vp_count: vm_config.processor_topology.proc_count,
                        target_arch: if cfg!(guest_arch = "x86_64") {
                            debug_worker_defs::TargetArch::X86_64
                        } else {
                            debug_worker_defs::TargetArch::Aarch64
                        },
                    },
                )
                .await
                .context("failed to launch gdbstub worker")?,
        )
    } else {
        None
    };

    // spin up the VM
    let (vm_rpc, rpc_recv) = mesh::channel();
    let (notify_send, notify_recv) = mesh::channel();
    let vm_worker = {
        let vm_host = mesh.make_host("vm", opt.log_file.clone()).await?;

        let (shared_memory, saved_state) = if let Some(snapshot_dir) = &opt.restore_snapshot {
            let (fd, state_msg) = prepare_snapshot_restore(snapshot_dir, &opt)?;
            (Some(fd), Some(state_msg))
        } else {
            let shared_memory = opt
                .memory_backing_file()
                .map(|path| {
                    openvmm_helpers::shared_memory::open_memory_backing_file(
                        path,
                        opt.memory_size(),
                    )
                })
                .transpose()?;
            (shared_memory, None)
        };

        let params = VmWorkerParameters {
            hypervisor: match &opt.hypervisor {
                Some(name) => openvmm_helpers::hypervisor::hypervisor_resource(name)?,
                None => openvmm_helpers::hypervisor::choose_hypervisor()?,
            },
            cfg: vm_config,
            saved_state,
            shared_memory,
            rpc: rpc_recv,
            notify: notify_send,
        };
        vm_host
            .launch_worker(VM_WORKER, params)
            .await
            .context("failed to launch vm worker")?
    };

    if opt.restore_snapshot.is_some() {
        tracing::info!("restoring VM from snapshot");
    }

    if !opt.paused {
        vm_rpc.call(VmRpc::Resume, ()).await?;
    }

    let paravisor_diag = Arc::new(diag_client::DiagClient::from_dialer(
        driver.clone(),
        DiagDialer {
            driver: driver.clone(),
            vm_rpc: vm_rpc.clone(),
            openhcl_vtl: if opt.vtl2 {
                DeviceVtl::Vtl2
            } else {
                DeviceVtl::Vtl0
            },
        },
    ));

    let diag_inspector = DiagInspector::new(driver.clone(), paravisor_diag.clone());

    // Create channels between the REPL and VmController.
    let (vm_controller_send, vm_controller_recv) = mesh::channel();
    let (vm_controller_event_send, vm_controller_event_recv) = mesh::channel();

    let has_vtl2 = resources.vtl2_settings.is_some();
    let serial_driver = resources
        .serial_driver
        .take()
        .expect("serial driver must outlive serial resources");

    // Build the VmController with exclusive resources.
    let controller = vm_controller::VmController {
        mesh: mesh_slot.take().unwrap(),
        vm_worker,
        vnc_worker,
        gdb_worker,
        diag_inspector: Some(diag_inspector),
        vtl2_settings: resources.vtl2_settings,
        ged_rpc: resources.ged_rpc.clone(),
        vm_rpc: vm_rpc.clone(),
        paravisor_diag: Some(paravisor_diag),
        igvm_path: opt.igvm.as_ref().map(|igvm| igvm.firmware.clone()),
        memory_backing_file: opt.memory_backing_file().cloned(),
        memory: opt.memory_size(),
        processors: opt.processors,
        log_file: opt.log_file.clone(),
        crash_dump_path: opt.crash_dump_path.clone(),
        guest_power_actions: vm_controller::GuestPowerActions {
            shutdown: opt.guest_shutdown_action,
            reset: opt.guest_reset_action,
            crash: opt.guest_crash_action,
            watchdog: opt.guest_watchdog_action,
        },
    };

    // Spawn the VmController as a task.
    let controller_task = driver.spawn(
        "vm-controller",
        controller.run(vm_controller_recv, vm_controller_event_send, notify_recv),
    );

    // Run the REPL with shareable resources.
    let repl_result = repl::run_repl(
        driver,
        repl::ReplResources {
            vm_rpc,
            vm_controller: vm_controller_send,
            vm_controller_events: vm_controller_event_recv,
            scsi_rpc: resources.scsi_rpc,
            nvme_vtl2_rpc: resources.nvme_vtl2_rpc,
            consomme_rpc: resources.consomme_rpc,
            shutdown_ic: resources.shutdown_ic,
            kvp_ic: resources.kvp_ic,
            console_in: resources.console_in,
            has_vtl2,
        },
    )
    .await;

    // Wait for the controller task to finish (it stops the VM worker and
    // shuts down the mesh).
    controller_task.await;
    drop(serial_driver);

    // run_repl returns the exit status: the code the guest drove via an opt-in
    // exit (VmControllerEvent::ExitRequested), or 0 when the VM stopped normally.
    repl_result
}

struct DiagDialer {
    driver: DefaultDriver,
    vm_rpc: mesh::Sender<VmRpc>,
    openhcl_vtl: DeviceVtl,
}

impl mesh_rpc::client::Dial for DiagDialer {
    type Stream = PolledSocket<unix_socket::UnixStream>;

    async fn dial(&mut self) -> io::Result<Self::Stream> {
        let service_id = new_hvsock_service_id(1);
        let socket = self
            .vm_rpc
            .call_failable(
                VmRpc::ConnectHvsock,
                (
                    CancelContext::new().with_timeout(Duration::from_secs(2)),
                    service_id,
                    self.openhcl_vtl,
                ),
            )
            .await
            .map_err(io::Error::other)?;

        PolledSocket::new(&self.driver, socket)
    }
}

/// An object that implements [`InspectMut`] by sending an inspect request over
/// TTRPC to the guest (typically the paravisor running in VTL2), then stitching
/// the response back into the inspect tree.
///
/// This also caches the TTRPC connection to the guest so that only the first
/// inspect request has to wait for the connection to be established.
pub(crate) struct DiagInspector(DiagInspectorInner);

enum DiagInspectorInner {
    NotStarted(DefaultDriver, Arc<diag_client::DiagClient>),
    Started {
        send: mesh::Sender<inspect::Deferred>,
        _task: Task<()>,
    },
    Invalid,
}

impl DiagInspector {
    pub fn new(driver: DefaultDriver, diag_client: Arc<diag_client::DiagClient>) -> Self {
        Self(DiagInspectorInner::NotStarted(driver, diag_client))
    }

    fn start(&mut self) -> &mesh::Sender<inspect::Deferred> {
        loop {
            match self.0 {
                DiagInspectorInner::NotStarted { .. } => {
                    let DiagInspectorInner::NotStarted(driver, client) =
                        std::mem::replace(&mut self.0, DiagInspectorInner::Invalid)
                    else {
                        unreachable!()
                    };
                    let (send, recv) = mesh::channel();
                    let task = driver.clone().spawn("diag-inspect", async move {
                        Self::run(&client, recv).await
                    });

                    self.0 = DiagInspectorInner::Started { send, _task: task };
                }
                DiagInspectorInner::Started { ref send, .. } => break send,
                DiagInspectorInner::Invalid => unreachable!(),
            }
        }
    }

    async fn run(
        diag_client: &diag_client::DiagClient,
        mut recv: mesh::Receiver<inspect::Deferred>,
    ) {
        while let Some(deferred) = recv.next().await {
            let info = deferred.external_request();
            let result = match info.request_type {
                inspect::ExternalRequestType::Inspect { depth } => {
                    if depth == 0 {
                        Ok(inspect::Node::Unevaluated)
                    } else {
                        // TODO: Support taking timeouts from the command line
                        diag_client
                            .inspect(info.path, Some(depth - 1), Some(Duration::from_secs(1)))
                            .await
                    }
                }
                inspect::ExternalRequestType::Update { value } => {
                    (diag_client.update(info.path, value).await).map(inspect::Node::Value)
                }
            };
            deferred.complete_external(
                result.unwrap_or_else(|err| {
                    inspect::Node::Failed(inspect::Error::Mesh(format!("{err:#}")))
                }),
                inspect::SensitivityLevel::Unspecified,
            )
        }
    }
}

impl InspectMut for DiagInspector {
    fn inspect_mut(&mut self, req: inspect::Request<'_>) {
        self.start().send(req.defer());
    }
}

#[cfg(test)]
mod tests {
    use super::*;
    use clap::Parser;
    use std::fs::File;
    use test_with_tracing::test;

    #[test]
    fn maps_igvm_personalities_to_chipsets() {
        for (args, expected) in [
            (
                vec!["openvmm", "--igvm", "firmware=guest.igvm,personality=uefi"],
                BaseChipsetType::HypervGen2Uefi,
            ),
            (
                vec![
                    "openvmm",
                    "--igvm",
                    "firmware=guest.igvm,personality=linux-direct",
                ],
                BaseChipsetType::UnenlightenedLinuxDirect,
            ),
            (
                vec![
                    "openvmm",
                    "--igvm",
                    "firmware=guest.igvm,personality=linux-direct",
                    "--hv",
                ],
                BaseChipsetType::HyperVGen2LinuxDirect,
            ),
            (
                vec![
                    "openvmm",
                    "--igvm",
                    "firmware=guest.igvm,personality=linux-direct",
                    "--isolation",
                    "snp",
                ],
                BaseChipsetType::EnlightenedLinuxDirect,
            ),
            (
                vec![
                    "openvmm",
                    "--igvm",
                    "firmware=guest.igvm,personality=openhcl",
                    "--hv",
                    "--vtl2",
                ],
                BaseChipsetType::HclHost,
            ),
        ] {
            let opt = Options::try_parse_from(args).unwrap();
            opt.validate_igvm_options().unwrap();
            assert!(
                std::mem::discriminant(&base_chipset_type(&opt))
                    == std::mem::discriminant(&expected)
            );
        }
    }

    #[test]
    fn builds_openhcl_igvm_config_without_host_uefi() {
        DefaultPool::run_with(async |driver| {
            let temp_dir = tempfile::tempdir().unwrap();
            let igvm_path = temp_dir.path().join("guest.igvm");
            File::create(&igvm_path).unwrap();
            let igvm_options = format!("firmware={},personality=openhcl", igvm_path.display());
            let mesh = VmmMesh::new(&driver, true).unwrap();

            for (extra, expected_alias, expected_policy) in [
                (vec![], true, Some(LateMapVtl0MemoryPolicy::Halt)),
                (
                    vec!["--uefi", "console=com1,disable_frontpage"],
                    true,
                    Some(LateMapVtl0MemoryPolicy::Halt),
                ),
                (
                    vec!["--net", "uh:consomme", "--no-alias-map"],
                    false,
                    Some(LateMapVtl0MemoryPolicy::Halt),
                ),
                (
                    vec!["--isolation", "vbs", "--no-alias-map"],
                    false,
                    Some(LateMapVtl0MemoryPolicy::Halt),
                ),
                (
                    vec!["--no-alias-map", "--late-map-vtl0-policy", "exception"],
                    false,
                    Some(LateMapVtl0MemoryPolicy::InjectException),
                ),
                (
                    vec!["--late-map-vtl0-policy", "log"],
                    true,
                    Some(LateMapVtl0MemoryPolicy::Log),
                ),
                (
                    vec!["--igvm-vtl2-relocation-type", "vtl2=filesize"],
                    true,
                    Some(LateMapVtl0MemoryPolicy::Halt),
                ),
                (
                    vec![
                        "--igvm-vtl2-relocation-type",
                        "vtl2=filesize",
                        "--late-map-vtl0-policy",
                        "off",
                    ],
                    true,
                    None,
                ),
            ] {
                let mut args = vec![
                    "openvmm",
                    "--igvm",
                    &igvm_options,
                    "--hv",
                    "--vtl2",
                    "--single-process",
                ];
                args.extend(extra);
                let opt = Options::try_parse_from(args).unwrap();
                let (config, resources) = vm_config_from_command_line(driver.clone(), &mesh, &opt)
                    .await
                    .unwrap();

                assert!(matches!(config.load_mode, LoadMode::Igvm { .. }));
                assert!(config.hypervisor.with_hv);
                let vtl2 = config.hypervisor.with_vtl2.as_ref().unwrap();
                assert_eq!(vtl2.vtl0_alias_map, expected_alias);
                assert_eq!(vtl2.late_map_vtl0_memory, expected_policy);
                assert!(config.vmbus.is_some());
                assert!(config.vtl2_vmbus.is_some());
                assert!(resources.ged_rpc.is_some());
                for id in ["ged", "gel"] {
                    assert!(
                        config.vmbus_devices.iter().any(|(vtl, resource)| {
                            *vtl == DeviceVtl::Vtl2 && resource.id() == id
                        })
                    );
                }
                assert!(
                    config
                        .chipset_devices
                        .iter()
                        .all(|device| { device.resource.id() != "hyperv_firmware_uefi" })
                );
            }
            for (extra, expected_error) in [
                (
                    ["--net", "uh:consomme"],
                    "must specify --no-alias-map to offer NICs to VTL2",
                ),
                (
                    ["--isolation", "vbs"],
                    "alias map not supported with isolation",
                ),
            ] {
                let mut args = vec![
                    "openvmm",
                    "--igvm",
                    &igvm_options,
                    "--hv",
                    "--vtl2",
                    "--single-process",
                ];
                args.extend(extra);
                let opt = Options::try_parse_from(args).unwrap();
                let err = vm_config_from_command_line(driver.clone(), &mesh, &opt)
                    .await
                    .err()
                    .unwrap();
                assert_eq!(err.to_string(), expected_error);
            }
            mesh.shutdown().await;
        });
    }

    #[test]
    fn maps_virtio_vsock_to_named_pcie_port() {
        DefaultPool::run_with(async |driver| {
            let temp_dir = tempfile::tempdir().unwrap();
            let kernel_path = temp_dir.path().join("kernel");
            File::create(&kernel_path).unwrap();
            let initrd_path = temp_dir.path().join("initrd");
            File::create(&initrd_path).unwrap();
            let socket_path = temp_dir.path().join("vsock");
            let opt = Options::try_parse_from([
                "openvmm",
                "--kernel",
                kernel_path.to_str().unwrap(),
                "--initrd",
                initrd_path.to_str().unwrap(),
                "--virtio-vsock-path",
                socket_path.to_str().unwrap(),
                "--virtio-vsock-bus",
                "pcie:custom",
                "--single-process",
            ])
            .unwrap();
            let mesh = VmmMesh::new(&driver, true).unwrap();

            let (config, _resources) = vm_config_from_command_line(driver, &mesh, &opt)
                .await
                .unwrap();

            assert_eq!(config.pcie_devices.len(), 1);
            assert_eq!(config.pcie_devices[0].port_name, "custom");
            mesh.shutdown().await;
        });
    }

    #[test]
    fn maps_virtio_fs_and_rng_to_named_pcie_ports() {
        DefaultPool::run_with(async |driver| {
            let temp_dir = tempfile::tempdir().unwrap();
            let kernel_path = temp_dir.path().join("kernel");
            File::create(&kernel_path).unwrap();
            let initrd_path = temp_dir.path().join("initrd");
            File::create(&initrd_path).unwrap();
            let root_path = temp_dir.path().to_str().unwrap();
            let opt = Options::try_parse_from([
                "openvmm",
                "--kernel",
                kernel_path.to_str().unwrap(),
                "--initrd",
                initrd_path.to_str().unwrap(),
                "--virtio-fs",
                &format!("fs,{root_path}"),
                "--virtio-fs-bus",
                "pcie:fs",
                "--virtio-rng",
                "--virtio-rng-bus",
                "pcie:rng",
                "--single-process",
            ])
            .unwrap();
            let mesh = VmmMesh::new(&driver, true).unwrap();

            let (config, _resources) = vm_config_from_command_line(driver, &mesh, &opt)
                .await
                .unwrap();

            let port_names: Vec<_> = config
                .pcie_devices
                .iter()
                .map(|device| device.port_name.as_str())
                .collect();
            assert_eq!(port_names, ["fs", "rng"]);
            mesh.shutdown().await;
        });
    }

    #[test]
    fn rejects_duplicate_pcie_port_assignments() {
        DefaultPool::run_with(async |driver| {
            let temp_dir = tempfile::tempdir().unwrap();
            let kernel_path = temp_dir.path().join("kernel");
            File::create(&kernel_path).unwrap();
            let initrd_path = temp_dir.path().join("initrd");
            File::create(&initrd_path).unwrap();
            let root_path = temp_dir.path().to_str().unwrap();
            let opt = Options::try_parse_from([
                "openvmm",
                "--kernel",
                kernel_path.to_str().unwrap(),
                "--initrd",
                initrd_path.to_str().unwrap(),
                "--virtio-fs",
                &format!("pcie_port=custom:fs,{root_path}"),
                "--virtio-rng",
                "--virtio-rng-bus",
                "pcie:custom",
                "--single-process",
            ])
            .unwrap();
            let mesh = VmmMesh::new(&driver, true).unwrap();

            let error = vm_config_from_command_line(driver, &mesh, &opt)
                .await
                .err()
                .unwrap();

            assert_eq!(error.to_string(), "multiple devices use PCIe port 'custom'");
            mesh.shutdown().await;
        });
    }

    #[test]
    fn rejects_duplicate_pcie_port_assignment_from_storage() {
        DefaultPool::run_with(async |driver| {
            let temp_dir = tempfile::tempdir().unwrap();
            let kernel_path = temp_dir.path().join("kernel");
            File::create(&kernel_path).unwrap();
            let initrd_path = temp_dir.path().join("initrd");
            File::create(&initrd_path).unwrap();
            let opt = Options::try_parse_from([
                "openvmm",
                "--kernel",
                kernel_path.to_str().unwrap(),
                "--initrd",
                initrd_path.to_str().unwrap(),
                "--nvme-pci",
                "id=nvme0,pcie_port=custom",
                "--virtio-rng",
                "--virtio-rng-bus",
                "pcie:custom",
                "--single-process",
            ])
            .unwrap();
            let mesh = VmmMesh::new(&driver, true).unwrap();

            let error = vm_config_from_command_line(driver, &mesh, &opt)
                .await
                .err()
                .unwrap();

            assert_eq!(error.to_string(), "multiple devices use PCIe port 'custom'");
            mesh.shutdown().await;
        });
    }
}