1pub mod devices;
9
10use crate::Drive;
11use crate::Firmware;
12use crate::ModifyFn;
13use crate::NoPetriVmFramebufferAccess;
14use crate::NoPetriVmInspector;
15use crate::OpenHclServicingFlags;
16use crate::PetriHaltReason;
17use crate::PetriHaltReasonDetail;
18use crate::PetriInitrd;
19use crate::PetriVmConfig;
20use crate::PetriVmResources;
21use crate::PetriVmRuntime;
22use crate::PetriVmRuntimeConfig;
23use crate::PetriVmmBackend;
24use crate::ShutdownKind;
25use crate::TestChild;
26use crate::VmmQuirks;
27use crate::openhcl_diag::OpenHclDiagHandler;
28use crate::vm::PetriVmProperties;
29use anyhow::Context;
30use async_trait::async_trait;
31use devices::DeviceConfig;
32use futures::lock::Mutex;
33use futures_concurrency::future::Race;
34use get_resources::ged::FirmwareEvent;
35use guid::Guid;
36use pal_async::DefaultDriver;
37use pal_async::pipe::PolledPipe;
38use pal_async::process::PolledChild;
39use pal_async::task::Spawn;
40use pal_async::task::Task;
41use pal_async::timer::PolledTimer;
42use petri_artifacts_common::tags::GuestQuirksInner;
43use petri_artifacts_common::tags::MachineArch;
44use petri_artifacts_core::ArtifactResolver;
45use petri_artifacts_core::ResolvedArtifact;
46use pipette_client::PipetteClient;
47use std::path::Path;
48use std::path::PathBuf;
49use std::process::Command;
50use std::sync::Arc;
51use std::time::Duration;
52use tempfile::TempPath;
53use vtl2_settings_proto::Vtl2Settings;
54
55#[derive(Debug)]
57pub struct QemuPetriBackend {
58 qemu_path: ResolvedArtifact,
59}
60
61#[derive(Debug, Default)]
63pub struct QemuPetriConfig {
64 share_9p: Option<PathBuf>,
65 devices: Vec<DeviceConfig>,
66 machine: Option<String>,
67 cpu: Option<String>,
68 smp: Option<String>,
69 memory: Option<String>,
70 command_line: String,
71}
72
73impl QemuPetriBackend {
74 pub fn from_binary(path: PathBuf) -> Self {
76 Self {
77 qemu_path: ResolvedArtifact::from_local_path(path),
78 }
79 }
80}
81
82pub struct QemuPetriRuntime {
84 driver: DefaultDriver,
85 qemu_process: Arc<Mutex<TestChild>>,
86 host_pipette_port: u16,
87 log_tasks: Vec<Task<anyhow::Result<()>>>,
88 output_dir: PathBuf,
89}
90
91#[async_trait]
92impl PetriVmmBackend for QemuPetriBackend {
93 type VmmConfig = QemuPetriConfig;
94 type VmRuntime = QemuPetriRuntime;
95 const SUPPORTS_VMBUS: bool = false;
96 const SUPPORTS_CPU_EMULATION: bool = true;
97 const SUPPORTS_FILE_SHARING: bool = true;
98
99 fn check_compat(_firmware: &Firmware, _arch: MachineArch) -> bool {
100 MachineArch::X86_64 == MachineArch::host() && cfg!(target_os = "linux")
102 }
103
104 fn quirks(_firmware: &Firmware) -> (GuestQuirksInner, VmmQuirks) {
105 (GuestQuirksInner::default(), VmmQuirks::default())
106 }
107
108 fn default_servicing_flags() -> OpenHclServicingFlags {
109 OpenHclServicingFlags {
110 enable_nvme_keepalive: false,
111 enable_mana_keepalive: false,
112 override_version_checks: true,
113 stop_timeout_hint_secs: None,
114 }
115 }
116
117 fn create_guest_dump_disk() -> anyhow::Result<
118 Option<(
119 Arc<TempPath>,
120 Box<dyn FnOnce() -> anyhow::Result<Box<dyn fatfs::ReadWriteSeek>>>,
121 )>,
122 > {
123 Ok(None)
124 }
125
126 fn build_custom_init_script(pipette_path: &str) -> Option<String> {
127 let mut script = String::from(
128 "#!/bin/sh\n\
129 ip link set eth0 up\n\
130 ip addr add 10.0.2.15/24 dev eth0\n\
131 ip route add default via 10.0.2.2\n\
132 echo 'nameserver 10.0.2.3' > /etc/resolv.conf\n",
133 );
134
135 script.push_str(&format!(
136 "exec '/{}' --transport tcp\n",
137 pipette_path.replace('\'', "'\\''")
138 ));
139
140 Some(script)
141 }
142
143 fn new(resolver: &ArtifactResolver<'_>, arch: MachineArch) -> Self {
144 assert_eq!(arch, MachineArch::Aarch64);
148 QemuPetriBackend {
149 qemu_path: resolver
150 .require(petri_artifacts_vmm_test::artifacts::QEMU_SYSTEM_AARCH64_LINUX_X64)
151 .erase(),
152 }
153 }
154
155 async fn run(
156 self,
157 config: PetriVmConfig,
158 modify_vmm_config: Option<ModifyFn<Self::VmmConfig>>,
159 resources: &PetriVmResources,
160 _properties: &PetriVmProperties,
161 ) -> anyhow::Result<(Self::VmRuntime, PetriVmRuntimeConfig)> {
162 let PetriVmResources {
163 driver,
164 log_source,
165 prebuilt_initrd,
166 } = resources;
167
168 let mut qemu_config = QemuPetriConfig::default();
169 if let Some(f) = modify_vmm_config {
170 qemu_config = f.0(qemu_config);
171 }
172
173 let host_pipette_port = pick_free_port().context("failed to find a free port")?;
174
175 let mut cmd = build_qemu_command(
176 self.qemu_path.get(),
177 &config,
178 &qemu_config,
179 host_pipette_port,
180 prebuilt_initrd
181 .as_ref()
182 .context("QEMU requires a prebuilt initrd")?,
183 )?;
184 cmd.stdin(std::process::Stdio::null());
185 cmd.stdout(std::process::Stdio::piped());
186 cmd.stderr(std::process::Stdio::piped());
187
188 tracing::info!(?cmd, "launching qemu");
189 let mut qemu_process = cmd.spawn().context("failed to launch QEMU")?;
190 let qemu_stdout = qemu_process.stdout.take().expect("stdout should be piped");
191 let qemu_stderr = qemu_process.stderr.take().expect("stderr should be piped");
192
193 let qemu_process = PolledChild::<std::process::Child>::new(driver, qemu_process)
194 .context("failed to create PolledChild")?;
195 let qemu_process = TestChild::new(qemu_process);
196
197 let mut log_tasks = Vec::new();
198
199 let qemu_stdout_pipe = PolledPipe::new(driver, child_pipe_to_file(qemu_stdout))
200 .context("failed to create polled pipe for qemu stdout")?;
201 let qemu_stdout_log_file = log_source.log_file("guest")?;
203 let qemu_stdout_task = driver.spawn(
204 "qemu_stdout",
205 crate::log_task(qemu_stdout_log_file, qemu_stdout_pipe, "qemu_stdout"),
206 );
207 log_tasks.push(qemu_stdout_task);
208
209 let qemu_stderr_pipe = PolledPipe::new(driver, child_pipe_to_file(qemu_stderr))
210 .context("failed to create polled pipe for qemu stderr")?;
211 let qemu_stderr_log_file = log_source.log_file("qemu")?;
213 let qemu_stderr_task = driver.spawn(
214 "qemu_stderr",
215 crate::log_task(qemu_stderr_log_file, qemu_stderr_pipe, "qemu_stderr"),
216 );
217 log_tasks.push(qemu_stderr_task);
218
219 Ok((
220 QemuPetriRuntime {
221 driver: driver.clone(),
222 qemu_process: Arc::new(Mutex::new(qemu_process)),
223 host_pipette_port,
224 log_tasks,
225 output_dir: log_source.output_dir().to_owned(),
226 },
227 config
228 .firmware
229 .into_runtime_config(config.vmbus_storage_controllers),
230 ))
231 }
232}
233
234impl QemuPetriConfig {
235 pub fn with_platform(
237 mut self,
238 machine: String,
239 cpu: String,
240 smp: String,
241 memory: String,
242 command_line: String,
243 ) -> Self {
244 self.machine = Some(machine);
245 self.cpu = Some(cpu);
246 self.smp = Some(smp);
247 self.memory = Some(memory);
248 self.command_line = command_line;
249 self
250 }
251
252 pub fn with_share_9p(mut self, path: impl AsRef<Path>) -> Self {
254 self.share_9p = Some(path.as_ref().to_path_buf());
255 self
256 }
257
258 pub fn with_devices(mut self, devices: impl IntoIterator<Item = DeviceConfig>) -> Self {
260 self.devices.extend(devices);
261 self
262 }
263}
264
265#[async_trait]
266impl PetriVmRuntime for QemuPetriRuntime {
267 type VmInspector = NoPetriVmInspector;
268 type VmFramebufferAccess = NoPetriVmFramebufferAccess;
269
270 async fn teardown(mut self) -> anyhow::Result<()> {
271 futures::future::join_all(self.log_tasks.into_iter().map(|t| t.cancel())).await;
272 let mut qemu_process = self.qemu_process.lock().await;
273 qemu_process
274 .get_mut()
275 .kill()
276 .context("unable to kill qemu process")?;
277 Ok(())
278 }
279
280 async fn wait_for_halt(&mut self, _allow_reset: bool) -> anyhow::Result<PetriHaltReasonDetail> {
281 let status = self.qemu_process.lock().await.wait().await?;
282 Ok(PetriHaltReasonDetail {
283 reason: if status.success() {
284 PetriHaltReason::PowerOff
285 } else {
286 PetriHaltReason::Other
287 },
288 detail: format!("QEMU exited with status: {:?}", status.code()),
289 })
290 }
291
292 async fn wait_for_agent(&mut self, _set_high_vtl: bool) -> anyhow::Result<PipetteClient> {
293 let driver = self.driver.clone();
294 let output_dir = self.output_dir.clone();
295 let addr = std::net::SocketAddr::from(([127, 0, 0, 1], self.host_pipette_port));
296 let client_core = async move || {
297 tracing::debug!("connecting to pipette");
298 let socket = pal_async::socket::PolledSocket::connect_tcp(&driver, addr)
299 .await
300 .context("failed to connect to pipette")?;
301 tracing::debug!("handshaking with pipette");
302 PipetteClient::new(&driver, socket, &output_dir)
303 .await
304 .context("failed to create pipette client")
305 };
306
307 self.poll_until_exit_or_timeout(None, async move |_| match client_core().await {
308 Ok(client) => {
309 tracing::info!("completed pipette handshake");
310 Ok(Some(client))
311 }
312 Err(_err) => {
313 tracing::debug!("failed to connect to pipette server, retrying");
314 Ok(None)
315 }
316 })
317 .await
318 }
319
320 fn openhcl_diag(&self) -> Option<OpenHclDiagHandler> {
321 None
323 }
324
325 async fn wait_for_boot_event(
326 &mut self,
327 _timeout: Option<Duration>,
328 ) -> anyhow::Result<Option<FirmwareEvent>> {
329 anyhow::bail!("QEMU backend only supports linux direct, which doesn't emit a boot event")
330 }
331
332 async fn wait_for_enlightened_shutdown_ready(&mut self) -> anyhow::Result<()> {
333 tracing::info!("QEMU doesn't support enlightened shutdown, immediately returning");
334 Ok(())
335 }
336
337 async fn send_enlightened_shutdown(&mut self, _kind: ShutdownKind) -> anyhow::Result<()> {
338 anyhow::bail!("QEMU doesn't support enlightened shutdown")
339 }
340
341 async fn restart_openhcl(
342 &mut self,
343 _new_openhcl: &ResolvedArtifact,
344 _flags: OpenHclServicingFlags,
345 ) -> anyhow::Result<()> {
346 anyhow::bail!("QEMU doesn't support OpenHCL")
347 }
348
349 async fn save_openhcl(
350 &mut self,
351 _new_openhcl: &ResolvedArtifact,
352 _flags: OpenHclServicingFlags,
353 ) -> anyhow::Result<()> {
354 anyhow::bail!("QEMU doesn't support OpenHCL")
355 }
356
357 async fn restore_openhcl(&mut self) -> anyhow::Result<()> {
358 anyhow::bail!("QEMU doesn't support OpenHCL")
359 }
360
361 async fn update_command_line(&mut self, _command_line: &str) -> anyhow::Result<()> {
362 todo!()
363 }
364
365 fn take_framebuffer_access(&mut self) -> Option<NoPetriVmFramebufferAccess> {
366 None
367 }
368
369 async fn reset(&mut self) -> anyhow::Result<()> {
370 todo!()
371 }
372
373 async fn get_guest_state_file(&self) -> anyhow::Result<Option<PathBuf>> {
374 todo!()
375 }
376
377 async fn set_vtl2_settings(&mut self, _settings: &Vtl2Settings) -> anyhow::Result<()> {
378 anyhow::bail!("QEMU doesn't support OpenHCL")
379 }
380
381 async fn set_vmbus_drive(
382 &mut self,
383 _drive: &Drive,
384 _controller_id: &Guid,
385 _controller_location: u32,
386 ) -> anyhow::Result<()> {
387 anyhow::bail!("QEMU doesn't support VMBus")
388 }
389}
390
391impl QemuPetriRuntime {
392 async fn poll_until_exit_or_timeout<T>(
395 &mut self,
396 timeout: Option<Duration>,
397 f: impl AsyncFn(&Self) -> anyhow::Result<Option<T>>,
398 ) -> anyhow::Result<T> {
399 let mut qemu_process = self.qemu_process.lock().await;
400 (
401 async {
402 let mut timer = PolledTimer::new(&self.driver);
403 loop {
404 if let Some(v) = f(self).await? {
405 return Ok(v);
406 }
407 timer.sleep(Duration::from_secs(10)).await;
408 }
409 },
410 async {
411 let status = qemu_process.wait().await?;
412 Err(anyhow::anyhow!(
413 "QEMU exited unexpectedly (status: {status})"
414 ))
415 },
416 async {
417 PolledTimer::new(&self.driver)
418 .sleep(timeout.unwrap_or(Duration::from_mins(10)))
419 .await;
420 Err(anyhow::anyhow!("Timed out waiting for operation"))
421 },
422 )
423 .race()
424 .await
425 }
426}
427
428#[cfg(unix)]
432pub fn child_pipe_to_file(pipe: impl Into<std::os::unix::io::OwnedFd>) -> std::fs::File {
433 std::fs::File::from(pipe.into())
434}
435
436#[cfg(windows)]
440pub fn child_pipe_to_file(pipe: impl Into<std::os::windows::io::OwnedHandle>) -> std::fs::File {
441 std::fs::File::from(pipe.into())
442}
443
444fn pick_free_port() -> anyhow::Result<u16> {
446 let listener =
447 std::net::TcpListener::bind("127.0.0.1:0").context("failed to bind ephemeral port")?;
448 let port = listener
449 .local_addr()
450 .context("failed to get local addr")?
451 .port();
452 Ok(port)
453}
454
455pub const EXTRA_DEVICE_ADDR_BASE: usize = 16;
461
462pub const SHARE_9P_MOUNT_TAG: &str = "hostshare";
464
465pub fn build_qemu_command(
467 binary: &Path,
468 config: &PetriVmConfig,
469 qemu_config: &QemuPetriConfig,
470 host_pipette_port: u16,
471 prebuilt_initrd: &PetriInitrd,
472) -> anyhow::Result<Command> {
473 let PetriVmConfig {
474 name: _,
475 arch,
476 host_log_levels,
477 firmware,
478 hibernation_enabled,
479 ipmi_enabled,
480 memory:
481 crate::MemoryConfig {
482 startup_bytes,
483 dynamic_memory_range,
484 numa_mem_sizes,
485 private_memory,
486 transparent_hugepages: _, },
488 proc_topology:
489 crate::ProcessorTopology {
490 vp_count,
491 enable_smt,
492 vps_per_socket,
493 apic_mode,
494 },
495 nested_virt_enabled: _, vmgs,
497 tpm,
498 vmbus_storage_controllers,
499 pcie_nvme_drives,
500 pcie_virtio_blk_drives,
501 physical_nvme_devices,
502 guest_shares,
503 } = config;
504
505 let QemuPetriConfig {
506 share_9p,
507 devices,
508 machine,
509 cpu,
510 smp,
511 memory,
512 command_line,
513 } = qemu_config;
514
515 assert!(matches!(arch, MachineArch::Aarch64));
517 assert!(matches!(
518 host_log_levels,
519 None | Some(crate::OpenvmmLogConfig::TestDefault)
520 ));
521 assert!(!*hibernation_enabled);
522 assert!(!*ipmi_enabled);
523 assert!(dynamic_memory_range.is_none());
524 assert!(numa_mem_sizes.is_none());
525 assert!(private_memory.is_none());
526 assert!(enable_smt.is_none());
527 assert!(vps_per_socket.is_none());
528 assert!(apic_mode.is_none());
529 assert!(matches!(vmgs, crate::PetriVmgsResource::Ephemeral));
530 assert!(tpm.is_none());
531 assert!(vmbus_storage_controllers.is_empty());
532 assert!(pcie_nvme_drives.is_empty());
533 assert!(pcie_virtio_blk_drives.is_empty());
534 assert!(physical_nvme_devices.is_empty());
535
536 let mut cmd = Command::new(binary);
537
538 cmd.arg("-machine").arg(
539 machine
540 .as_deref()
541 .unwrap_or("virt,virtualization=on,iommu=smmuv3,gic-version=3"),
542 );
543 cmd.arg("-cpu").arg(cpu.as_deref().unwrap_or("max"));
544 cmd.arg("-m").arg(
546 memory
547 .clone()
548 .unwrap_or_else(|| (startup_bytes / (1024 * 1024)).to_string()),
549 );
550 cmd.arg("-smp")
552 .arg(smp.clone().unwrap_or_else(|| vp_count.to_string()));
553 cmd.arg("-nographic");
554
555 let PetriInitrd { path, rdinit_param } = prebuilt_initrd;
556
557 match firmware {
558 Firmware::LinuxDirect { kernel, .. } => {
559 cmd.arg("-kernel").arg(kernel);
560 cmd.arg("-initrd").arg(path.as_ref());
561 }
562 _ => anyhow::bail!("qemu backend only supports linux direct"),
563 };
564
565 cmd.arg("-append")
566 .arg(format!("{command_line} rdinit={rdinit_param}").trim());
567 cmd.arg("-no-reboot");
568
569 let mut guest_shares = guest_shares
571 .iter()
572 .map(|(k, v)| (k.as_str(), v.host_path.as_path()))
573 .collect::<Vec<_>>();
574 if let Some(custom_share) = share_9p {
575 guest_shares.push((SHARE_9P_MOUNT_TAG, custom_share.as_path()));
576 }
577 for (i, (tag, path)) in guest_shares.iter().enumerate() {
578 cmd.arg("-fsdev").arg(format!(
579 "local,id=fsdev{i},path={},security_model=none",
580 path.to_str().context("share path not utf8")?
581 ));
582 cmd.arg("-device")
583 .arg(format!("virtio-9p-pci,fsdev=fsdev{i},mount_tag={tag}"));
584 }
585
586 cmd.arg("-netdev").arg(format!(
588 "user,id=net0,hostfwd=tcp::{host_pipette_port}-:{guest_port}",
589 guest_port = pipette_client::PIPETTE_PORT,
590 ));
591 cmd.arg("-device")
592 .arg("virtio-net-pci,netdev=net0,romfile=");
593
594 cmd.arg("-serial").arg("mon:stdio");
596
597 for (i, device) in devices.iter().enumerate() {
602 let rp_id = format!("hosting_rp{i}");
603 let addr = EXTRA_DEVICE_ADDR_BASE + i;
604 let slot = i + 1;
609 cmd.arg("-device").arg(format!(
610 "pcie-root-port,id={rp_id},addr={addr:#x},slot={slot}"
611 ));
612
613 match device {
614 DeviceConfig::VirtioBlk(cfg) => {
615 let node_name = format!("disk{i}");
616 let size_bytes = cfg.size;
617 cmd.arg("-blockdev")
618 .arg(format!("null-co,node-name={node_name},size={size_bytes}"));
619 cmd.arg("-device")
620 .arg(format!("virtio-blk-pci,drive={node_name},bus={rp_id},iommu_platform=on,disable-legacy=on,romfile="));
621 }
622 DeviceConfig::Edu(cfg) => {
623 let mut dev = format!("edu,bus={rp_id}");
628 if let Some(mask) = &cfg.dma_mask {
629 dev.push_str(&format!(",dma_mask={mask:#x}"));
630 }
631 cmd.arg("-device").arg(dev);
632 }
633 DeviceConfig::IvshmemPlain(cfg) => {
634 let mem_id = format!("ivshmem_mem{i}");
637 let size_bytes = cfg.size;
638 cmd.arg("-object")
639 .arg(format!("memory-backend-ram,id={mem_id},size={size_bytes}"));
640 cmd.arg("-device")
641 .arg(format!("ivshmem-plain,memdev={mem_id},bus={rp_id}"));
642 }
643 }
644 }
645
646 Ok(cmd)
647}
648
649#[cfg(test)]
650mod tests {
651 use super::*;
652 use test_with_tracing::test;
653
654 #[test]
655 fn preserves_standalone_platform_options() -> anyhow::Result<()> {
656 let config = PetriVmConfig {
657 name: "test".into(),
658 arch: MachineArch::Aarch64,
659 host_log_levels: None,
660 firmware: Firmware::LinuxDirect {
661 kernel: ResolvedArtifact::from_local_path("kernel with spaces".into()),
662 initrd: ResolvedArtifact::from_local_path("base-initrd".into()),
663 },
664 hibernation_enabled: false,
665 ipmi_enabled: false,
666 memory: Default::default(),
667 proc_topology: Default::default(),
668 nested_virt_enabled: true,
669 vmgs: crate::PetriVmgsResource::Ephemeral,
670 tpm: None,
671 vmbus_storage_controllers: Default::default(),
672 pcie_nvme_drives: Vec::new(),
673 pcie_virtio_blk_drives: Vec::new(),
674 physical_nvme_devices: Default::default(),
675 guest_shares: Default::default(),
676 };
677 let platform = QemuPetriConfig::default().with_platform(
678 "virt,gic-version=3".into(),
679 "max,sve=off".into(),
680 "4,sockets=2,cores=2".into(),
681 "4096".into(),
682 "console=ttyAMA0 debug".into(),
683 );
684 let initrd = PetriInitrd {
685 path: crate::TempOrPersistentPath::Persistent("patched initrd".into()),
686 rdinit_param: "/tcg-init.sh".into(),
687 };
688 let cmd = build_qemu_command(Path::new("qemu"), &config, &platform, 12345, &initrd)?;
689 let args = cmd
690 .get_args()
691 .map(|s| s.to_str().unwrap())
692 .collect::<Vec<_>>();
693 for pair in [
694 ["-machine", "virt,gic-version=3"],
695 ["-cpu", "max,sve=off"],
696 ["-m", "4096"],
697 ["-smp", "4,sockets=2,cores=2"],
698 ["-kernel", "kernel with spaces"],
699 ["-initrd", "patched initrd"],
700 ["-append", "console=ttyAMA0 debug rdinit=/tcg-init.sh"],
701 ] {
702 assert!(args.windows(2).any(|args| args == pair), "{pair:?}");
703 }
704 let default_cmd = build_qemu_command(
705 Path::new("qemu"),
706 &config,
707 &QemuPetriConfig::default(),
708 12345,
709 &initrd,
710 )?;
711 assert!(default_cmd.get_args().any(|a| a == "rdinit=/tcg-init.sh"));
712 Ok(())
713 }
714}