petri/vm/openvmm/
start.rs

1// Copyright (c) Microsoft Corporation.
2// Licensed under the MIT License.
3
4//! Methods to start a [`PetriVmConfigOpenVmm`] and produce a running [`PetriVmOpenVmm`].
5
6use super::PetriVmConfigOpenVmm;
7use super::PetriVmOpenVmm;
8use super::PetriVmResourcesOpenVmm;
9use crate::OpenvmmLogConfig;
10use crate::PetriLogFile;
11use crate::PetriVmRuntimeConfig;
12use crate::worker::Worker;
13use anyhow::Context;
14use mesh_process::Mesh;
15use mesh_process::ProcessConfig;
16use mesh_worker::WorkerHost;
17use openvmm_defs::config::DeviceVtl;
18use pal_async::pipe::PolledPipe;
19use pal_async::task::Spawn;
20use petri_artifacts_common::tags::MachineArch;
21use petri_artifacts_common::tags::OsFlavor;
22use std::collections::BTreeMap;
23use std::ffi::OsString;
24use std::io::Write;
25use std::sync::Arc;
26use vm_resource::IntoResource;
27
28impl PetriVmConfigOpenVmm {
29    async fn run_core(self) -> anyhow::Result<(PetriVmOpenVmm, PetriVmRuntimeConfig)> {
30        let Self {
31            runtime_config,
32            arch,
33            host_log_levels,
34            mut config,
35
36            mesh,
37
38            mut resources,
39
40            openvmm_log_file,
41
42            memory_backing_file,
43            requested_private_memory: _,
44
45            ged,
46            framebuffer_view,
47
48            pending_iommu,
49        } = self;
50
51        // Resolve deferred IOMMU assignments.
52        for (name, iommu_config) in &pending_iommu {
53            let rc = config
54                .pcie_root_complexes
55                .iter_mut()
56                .find(|rc| rc.name == *name)
57                .with_context(|| format!("IOMMU configured for unknown root complex '{name}'"))?;
58            rc.iommu = Some(iommu_config.clone());
59        }
60
61        // TODO: OpenHCL needs virt_whp support
62        // TODO: PCAT needs vga device support
63        // TODO: arm64 is broken?
64        // TODO: VPCI and some PCIe endpoints (NVMe/GDMA) don't support
65        // TODO: virtio vsock doesn't support save/restore yet
66        // save/restore yet.
67        let has_unsupported_pcie_save_restore_device = config
68            .pcie_devices
69            .iter()
70            .any(|device| matches!(device.resource.id(), "nvme" | "gdma"));
71        let supports_save_restore = !resources.properties.is_openhcl
72            && !resources.properties.is_pcat
73            && !matches!(arch, MachineArch::Aarch64)
74            && !resources.properties.using_vpci
75            && !has_unsupported_pcie_save_restore_device
76            && !resources.properties.use_virtio_vsock;
77
78        // Add the GED and VTL 2 settings.
79        if let Some(mut ged) = ged {
80            ged.vtl2_settings = Some(prost::Message::encode_to_vec(
81                runtime_config.vtl2_settings.as_ref().unwrap(),
82            ));
83            config
84                .vmbus_devices
85                .push((DeviceVtl::Vtl2, ged.into_resource()));
86        }
87
88        tracing::debug!(?config, "OpenVMM config");
89
90        let log_env = match host_log_levels {
91            None | Some(OpenvmmLogConfig::TestDefault) => BTreeMap::<OsString, OsString>::from([
92                // Quiet down `hyper_util`'s connection-pool debug spam that
93                // `disk_blob` triggers on every HTTP range request.
94                ("OPENVMM_LOG".into(), "debug,hyper_util=info".into()),
95                ("OPENVMM_SHOW_SPANS".into(), "true".into()),
96            ]),
97            Some(OpenvmmLogConfig::BuiltInDefault) => BTreeMap::new(),
98            Some(OpenvmmLogConfig::Custom(levels)) => levels
99                .iter()
100                .map(|(k, v)| (OsString::from(k), OsString::from(v)))
101                .collect::<BTreeMap<OsString, OsString>>(),
102        };
103
104        let (host, pid) = Self::openvmm_host(&mut resources, &mesh, openvmm_log_file, log_env)
105            .await
106            .context("failed to create host process")?;
107        // If a memory backing file was requested, open/create it and size
108        // it to match the configured guest RAM.
109        let shared_memory = memory_backing_file
110            .as_ref()
111            .map(|mem_path| {
112                let total_mem_size: u64 = config
113                    .numa
114                    .nodes
115                    .iter()
116                    .filter_map(|n| n.mem.as_ref())
117                    .map(|m| m.mem_size)
118                    .sum();
119                openvmm_helpers::shared_memory::open_memory_backing_file(mem_path, total_mem_size)
120            })
121            .transpose()?;
122
123        // Log the resolved guest RAM backing mode for diagnostics. Read from
124        // the final config so it reflects any backend overrides (e.g.
125        // `with_hugepages` / `with_memory_backing_file` forcing shared). Log
126        // per node, since NUMA nodes can have heterogeneous backings.
127        for (node, mem) in config
128            .numa
129            .nodes
130            .iter()
131            .enumerate()
132            .filter_map(|(i, n)| n.mem.as_ref().map(|m| (i, m)))
133        {
134            tracing::info!(
135                node,
136                mem_size = mem.mem_size,
137                backing_mode = if mem.private_memory {
138                    "private"
139                } else {
140                    "shared"
141                },
142                transparent_hugepages = mem.transparent_hugepages,
143                hugepages = mem.hugepages,
144                "guest RAM backing"
145            );
146        }
147
148        let (worker, halt_notif) = Worker::launch(&host, config, shared_memory)
149            .await
150            .context("failed to launch vm worker")?;
151
152        let worker = Arc::new(worker);
153
154        let is_minimal = resources.properties.minimal_mode;
155
156        // Resolve the TCP pipette port now, while the VM is starting.
157        // Consomme binds the port during launch, so the oneshot should
158        // be ready.  Caching the resolved port here lets wait_for_agent
159        // reconnect after a reset without needing the oneshot again.
160        let tcp_pipette_port = match resources.tcp_pipette_port.take() {
161            Some(recv) => Some(
162                recv.await
163                    .context("failed to receive TCP pipette port from consomme")?,
164            ),
165            None => None,
166        };
167
168        let mut vm = PetriVmOpenVmm::new(
169            super::runtime::PetriVmInner {
170                resources,
171                mesh,
172                worker,
173                framebuffer_view,
174                cidata_mounted: false,
175                tcp_pipette_port,
176                pid,
177            },
178            halt_notif,
179        );
180
181        tracing::info!("Resuming VM");
182        vm.resume().await?;
183
184        // Run basic save/restore test if it is supported
185        if supports_save_restore && !is_minimal {
186            tracing::info!("Testing save/restore");
187            vm.verify_save_restore().await?;
188        }
189
190        tracing::info!("VM ready");
191        Ok((vm, runtime_config))
192    }
193
194    /// Run the VM, configuring pipette to automatically start if it is
195    /// included in the config
196    pub async fn run(mut self) -> anyhow::Result<(PetriVmOpenVmm, PetriVmRuntimeConfig)> {
197        // Set up the IMC hive for Windows guests that use pipette in VTL0.
198        // Skip when VMBus is disabled — the no-vmbus prepped image has
199        // pipette pre-configured via offline registry injection.
200        if self.resources.properties.using_vtl0_pipette
201            && matches!(self.resources.properties.os_flavor, OsFlavor::Windows)
202            && !self.resources.properties.is_isolated
203            && !self.resources.properties.no_vmbus
204        {
205            let mut imc_hive_file = tempfile::tempfile().context("failed to create temp file")?;
206            imc_hive_file
207                .write_all(include_bytes!("../../../guest-bootstrap/imc.hiv"))
208                .context("failed to write imc hive")?;
209
210            self.config.vmbus_devices.push((
211                DeviceVtl::Vtl0,
212                vmbfs_resources::VmbfsImcDeviceHandle {
213                    file: imc_hive_file,
214                }
215                .into_resource(),
216            ));
217        }
218
219        // On non-pipette-as-init Linux direct, launch pipette via the serial
220        // agent. (When pipette is PID 1, it auto-starts on boot and the
221        // serial agent is not present.)
222        let launch_via_serial = self.resources.linux_direct_serial_agent.is_some()
223            && self.resources.properties.using_vtl0_pipette;
224
225        // Start the VM.
226        let (mut vm, config) = self.run_core().await?;
227
228        if launch_via_serial {
229            vm.launch_linux_direct_pipette().await?;
230        }
231
232        Ok((vm, config))
233    }
234
235    async fn openvmm_host(
236        resources: &mut PetriVmResourcesOpenVmm,
237        mesh: &Mesh,
238        log_file: PetriLogFile,
239        vmm_env: BTreeMap<OsString, OsString>,
240    ) -> anyhow::Result<(WorkerHost, i32)> {
241        // Copy the child's stderr to this process's, since internally this is
242        // wrapped by the test harness.
243        let (stderr_read, stderr_write) = pal::pipe_pair()?;
244        let task = resources.driver.spawn(
245            "serial log",
246            crate::log_task(
247                log_file,
248                PolledPipe::new(&resources.driver, stderr_read)
249                    .context("failed to create polled pipe")?,
250                "openvmm stderr",
251            ),
252        );
253        resources.log_stream_tasks.push(task);
254
255        let (host, runner) = mesh_worker::worker_host();
256        let pid = mesh
257            .launch_host(
258                ProcessConfig::new("vmm")
259                    .process_name(&resources.openvmm_path)
260                    .stderr(Some(stderr_write))
261                    .env(vmm_env),
262                openvmm_defs::entrypoint::MeshHostParams { runner },
263            )
264            .await?;
265        Ok((host, pid))
266    }
267}
petri/vm/openvmm/start.rs

petri/vm/openvmm/
start.rs
