1pub mod mshv;
8mod nice;
9mod vp_state;
10
11cfg_if::cfg_if! {
12 if #[cfg(guest_arch = "x86_64")] {
13 mod hardware_cvm;
14 pub mod snp;
15 pub mod tdx;
16
17 use crate::TlbFlushLockAccess;
18 use crate::VtlCrash;
19 use bitvec::prelude::BitArray;
20 use bitvec::prelude::Lsb0;
21 use hv1_emulator::synic::ProcessorSynic;
22 use hvdef::HvRegisterCrInterceptControl;
23 use hvdef::HvX64RegisterName;
24 use virt::vp::MpState;
25 use virt::x86::MsrError;
26 use virt_support_apic::LocalApic;
27 use virt_support_x86emu::translate::TranslationRegisters;
28 use virt::vp::AccessVpState;
29 use zerocopy::IntoBytes;
30 } else if #[cfg(guest_arch = "aarch64")] {
31 use hv1_hypercall::Arm64RegisterState;
32 use hvdef::HvArm64RegisterName;
33 } else {
34 compile_error!("unsupported guest architecture");
35 }
36}
37
38use super::Error;
39use super::UhPartitionInner;
40use super::UhVpInner;
41use crate::ExitActivity;
42use crate::GuestVtl;
43use crate::WakeReason;
44use cvm_tracing::CVM_ALLOWED;
45use cvm_tracing::CVM_CONFIDENTIAL;
46use hcl::ioctl::Hcl;
47use hcl::ioctl::ProcessorRunner;
48use hv1_emulator::message_queues::MessageQueues;
49use hv1_hypercall::HvRepResult;
50use hv1_structs::ProcessorSet;
51use hv1_structs::VtlArray;
52use hvdef::HvError;
53use hvdef::HvMessage;
54use hvdef::HvSynicSint;
55use hvdef::NUM_SINTS;
56use hvdef::Vtl;
57use inspect::Inspect;
58use inspect::InspectMut;
59use pal::unix::affinity;
60use pal::unix::affinity::CpuSet;
61use pal_async::driver::Driver;
62use pal_async::driver::PollImpl;
63use pal_async::timer::PollTimer;
64use pal_uring::IdleControl;
65use private::BackingPrivate;
66use std::convert::Infallible;
67use std::future::poll_fn;
68use std::marker::PhantomData;
69use std::sync::Arc;
70use std::sync::atomic::Ordering;
71use std::task::Poll;
72use virt::Processor;
73use virt::StopVp;
74use virt::VpHaltReason;
75use virt::VpIndex;
76use virt::io::CpuIo;
77use vm_topology::processor::TargetVpInfo;
78use vmcore::vmtime::VmTimeAccess;
79
80#[derive(InspectMut)]
87#[inspect(extra = "UhProcessor::inspect_extra", bound = "T: Backing")]
88pub struct UhProcessor<'a, T: Backing> {
89 _not_send: PhantomData<*mut ()>,
90
91 #[inspect(flatten)]
92 inner: &'a UhVpInner,
93 #[inspect(skip)]
94 partition: &'a UhPartitionInner,
95 #[inspect(skip)]
96 idle_control: Option<&'a mut IdleControl>,
97 #[inspect(skip)]
98 kernel_returns: u64,
99 #[inspect(hex, iter_by_index)]
100 crash_reg: [u64; hvdef::HV_X64_GUEST_CRASH_PARAMETER_MSRS],
101 vmtime: VmTimeAccess,
102 #[inspect(skip)]
103 timer: PollImpl<dyn PollTimer>,
104 #[inspect(mut)]
105 force_exit_sidecar: bool,
106 signaled_sidecar_exit: bool,
107 vtls_tlb_locked: VtlsTlbLocked,
109 #[inspect(skip)]
110 shared: &'a T::Shared,
111 #[inspect(hex, with = "|x| inspect::iter_by_index(x.iter()).map_value(|a| a.0)")]
112 exit_activities: VtlArray<ExitActivity, 2>,
113
114 #[inspect(skip)]
118 runner: ProcessorRunner<'a, T::HclBacking<'a>>,
119 #[inspect(mut)]
120 backing: T,
121}
122
123#[derive(Inspect)]
124struct VtlsTlbLocked {
125 vtl1: VtlArray<bool, 1>,
127 vtl2: VtlArray<bool, 2>,
128}
129
130impl VtlsTlbLocked {
131 fn get(&self, requesting_vtl: Vtl, target_vtl: GuestVtl) -> bool {
132 match requesting_vtl {
133 Vtl::Vtl0 => unreachable!(),
134 Vtl::Vtl1 => self.vtl1[target_vtl],
135 Vtl::Vtl2 => self.vtl2[target_vtl],
136 }
137 }
138
139 fn set(&mut self, requesting_vtl: Vtl, target_vtl: GuestVtl, value: bool) {
140 match requesting_vtl {
141 Vtl::Vtl0 => unreachable!(),
142 Vtl::Vtl1 => self.vtl1[target_vtl] = value,
143 Vtl::Vtl2 => self.vtl2[target_vtl] = value,
144 }
145 }
146
147 fn fill(&mut self, requesting_vtl: Vtl, value: bool) {
148 match requesting_vtl {
149 Vtl::Vtl0 => unreachable!(),
150 Vtl::Vtl1 => self.vtl1.fill(value),
151 Vtl::Vtl2 => self.vtl2.fill(value),
152 }
153 }
154}
155
156#[cfg(guest_arch = "x86_64")]
157#[derive(Inspect)]
158pub(crate) struct LapicState {
159 lapic: LocalApic,
160 activity: MpState,
161 nmi_pending: bool,
162}
163
164#[cfg(guest_arch = "x86_64")]
165impl LapicState {
166 pub fn new(lapic: LocalApic, activity: MpState) -> Self {
167 Self {
168 lapic,
169 activity,
170 nmi_pending: false,
171 }
172 }
173}
174
175struct BackingParams<'a, 'b, T: Backing> {
176 partition: &'a UhPartitionInner,
177 vp_info: &'a TargetVpInfo,
178 runner: &'a mut ProcessorRunner<'b, T::HclBacking<'b>>,
179}
180
181mod private {
182 use super::BackingParams;
183 use super::UhRunVpError;
184 use super::vp_state;
185 use crate::BackingShared;
186 use crate::Error;
187 use crate::GuestVtl;
188 use crate::processor::UhProcessor;
189 use hv1_emulator::hv::ProcessorVtlHv;
190 use hv1_structs::VtlArray;
191 use inspect::InspectMut;
192 use std::future::Future;
193 use virt::StopVp;
194 use virt::VpHaltReason;
195 use virt::io::CpuIo;
196 use virt::vp::AccessVpState;
197
198 #[expect(private_interfaces)]
199 pub trait BackingPrivate: 'static + Sized + InspectMut + Sized {
200 type HclBacking<'b>: hcl::ioctl::Backing<'b>;
201 type EmulationCache;
202 type Shared;
203
204 fn shared(shared: &BackingShared) -> &Self::Shared;
205
206 fn new(params: BackingParams<'_, '_, Self>, shared: &Self::Shared) -> Result<Self, Error>;
207
208 type StateAccess<'p, 'a>: AccessVpState<Error = vp_state::Error>
209 where
210 Self: 'a + 'p,
211 'p: 'a;
212
213 fn init(this: &mut UhProcessor<'_, Self>);
214
215 fn access_vp_state<'a, 'p>(
216 this: &'a mut UhProcessor<'p, Self>,
217 vtl: GuestVtl,
218 ) -> Self::StateAccess<'p, 'a>;
219
220 fn run_vp(
221 this: &mut UhProcessor<'_, Self>,
222 dev: &impl CpuIo,
223 stop: &mut StopVp<'_>,
224 ) -> impl Future<Output = Result<(), VpHaltReason<UhRunVpError>>>;
225
226 fn process_interrupts(
228 this: &mut UhProcessor<'_, Self>,
229 scan_irr: VtlArray<bool, 2>,
230 first_scan_irr: &mut bool,
231 dev: &impl CpuIo,
232 ) -> Result<bool, VpHaltReason<UhRunVpError>>;
233
234 fn poll_apic(
236 this: &mut UhProcessor<'_, Self>,
237 vtl: GuestVtl,
238 scan_irr: bool,
239 ) -> Result<(), UhRunVpError>;
240
241 fn request_extint_readiness(this: &mut UhProcessor<'_, Self>);
246
247 fn request_untrusted_sint_readiness(this: &mut UhProcessor<'_, Self>, sints: u16);
252
253 fn handle_vp_start_enable_vtl_wake(
254 _this: &mut UhProcessor<'_, Self>,
255 _vtl: GuestVtl,
256 ) -> Result<(), UhRunVpError>;
257
258 fn inspect_extra(_this: &mut UhProcessor<'_, Self>, _resp: &mut inspect::Response<'_>) {}
259
260 fn hv(&self, vtl: GuestVtl) -> Option<&ProcessorVtlHv>;
261 fn hv_mut(&mut self, vtl: GuestVtl) -> Option<&mut ProcessorVtlHv>;
262
263 fn vtl1_inspectable(this: &UhProcessor<'_, Self>) -> bool;
264 }
265}
266
267pub trait Backing: BackingPrivate {}
269
270impl<T: BackingPrivate> Backing for T {}
271
272#[cfg_attr(not(guest_arch = "x86_64"), expect(dead_code))]
273pub(crate) struct BackingSharedParams<'a> {
274 pub cvm_state: Option<crate::UhCvmPartitionState>,
275 #[cfg(guest_arch = "x86_64")]
276 pub cpuid: &'a virt::CpuidLeafSet,
277 pub hcl: &'a Hcl,
278 pub guest_vsm_available: bool,
279}
280
281#[cfg_attr(guest_arch = "aarch64", expect(dead_code))]
283enum InterceptMessageType {
284 #[cfg(guest_arch = "x86_64")]
285 Register {
286 reg: HvX64RegisterName,
287 value: u64,
288 },
289 Msr {
290 msr: u32,
291 },
292 #[cfg(guest_arch = "x86_64")]
293 IoPort {
294 port_number: u16,
295 access_size: u8,
296 string_access: bool,
297 rep_access: bool,
298 },
299}
300
301#[cfg_attr(guest_arch = "aarch64", expect(dead_code))]
303pub(crate) struct InterceptMessageState {
304 instruction_length_and_cr8: u8,
305 cpl: u8,
306 efer_lma: bool,
307 cs: hvdef::HvX64SegmentRegister,
308 rip: u64,
309 rflags: u64,
310 rax: u64,
311 rdx: u64,
312 rcx: u64,
313 rsi: u64,
314 rdi: u64,
315 optional: Option<InterceptMessageOptionalState>,
316}
317
318#[cfg_attr(guest_arch = "aarch64", expect(dead_code))]
319struct InterceptMessageOptionalState {
323 ds: hvdef::HvX64SegmentRegister,
324 es: hvdef::HvX64SegmentRegister,
325}
326
327impl InterceptMessageType {
328 #[cfg(guest_arch = "x86_64")]
329 fn generate_hv_message(
330 &self,
331 vp_index: VpIndex,
332 vtl: GuestVtl,
333 state: InterceptMessageState,
334 is_read: bool,
335 ) -> HvMessage {
336 let header = hvdef::HvX64InterceptMessageHeader {
337 vp_index: vp_index.index(),
338 instruction_length_and_cr8: state.instruction_length_and_cr8,
339 intercept_access_type: if is_read {
340 hvdef::HvInterceptAccessType::READ
341 } else {
342 hvdef::HvInterceptAccessType::WRITE
343 },
344 execution_state: hvdef::HvX64VpExecutionState::new()
345 .with_cpl(state.cpl)
346 .with_vtl(vtl.into())
347 .with_efer_lma(state.efer_lma),
348 cs_segment: state.cs,
349 rip: state.rip,
350 rflags: state.rflags,
351 };
352 match self {
353 InterceptMessageType::Register { reg, value } => {
354 let intercept_message = hvdef::HvX64RegisterInterceptMessage {
355 header,
356 flags: hvdef::HvX64RegisterInterceptMessageFlags::new(),
357 rsvd: 0,
358 rsvd2: 0,
359 register_name: *reg,
360 access_info: hvdef::HvX64RegisterAccessInfo::new_source_value(
361 hvdef::HvRegisterValue::from(*value),
362 ),
363 };
364 HvMessage::new(
365 hvdef::HvMessageType::HvMessageTypeRegisterIntercept,
366 0,
367 intercept_message.as_bytes(),
368 )
369 }
370 InterceptMessageType::Msr { msr } => {
371 let intercept_message = hvdef::HvX64MsrInterceptMessage {
372 header,
373 msr_number: *msr,
374 rax: state.rax,
375 rdx: state.rdx,
376 reserved: 0,
377 };
378
379 HvMessage::new(
380 hvdef::HvMessageType::HvMessageTypeMsrIntercept,
381 0,
382 intercept_message.as_bytes(),
383 )
384 }
385 InterceptMessageType::IoPort {
386 port_number,
387 access_size,
388 string_access,
389 rep_access,
390 } => {
391 let access_info =
392 hvdef::HvX64IoPortAccessInfo::new(*access_size, *string_access, *rep_access);
393 let intercept_message = hvdef::HvX64IoPortInterceptMessage {
394 header,
395 port_number: *port_number,
396 access_info,
397 instruction_byte_count: 0,
398 reserved: 0,
399 rax: state.rax,
400 instruction_bytes: [0u8; 16],
401 ds_segment: state.optional.as_ref().unwrap().ds,
402 es_segment: state.optional.as_ref().unwrap().es,
403 rcx: state.rcx,
404 rsi: state.rsi,
405 rdi: state.rdi,
406 };
407
408 HvMessage::new(
409 hvdef::HvMessageType::HvMessageTypeX64IoPortIntercept,
410 0,
411 intercept_message.as_bytes(),
412 )
413 }
414 }
415 }
416}
417
418#[cfg(guest_arch = "x86_64")]
420pub(crate) trait HardwareIsolatedBacking: Backing {
421 fn cvm_state(&self) -> &crate::UhCvmVpState;
423 fn cvm_state_mut(&mut self) -> &mut crate::UhCvmVpState;
425 fn cvm_partition_state(shared: &Self::Shared) -> &crate::UhCvmPartitionState;
427 fn tlb_flush_lock_access<'a>(
436 vp_index: Option<VpIndex>,
437 partition: &'a UhPartitionInner,
438 shared: &'a Self::Shared,
439 ) -> impl TlbFlushLockAccess + 'a;
440 fn switch_vtl(this: &mut UhProcessor<'_, Self>, source_vtl: GuestVtl, target_vtl: GuestVtl);
443 fn translation_registers(
445 &self,
446 this: &UhProcessor<'_, Self>,
447 vtl: GuestVtl,
448 ) -> TranslationRegisters;
449 fn pending_event_vector(this: &UhProcessor<'_, Self>, vtl: GuestVtl) -> Option<u8>;
452 fn is_interrupt_pending(
455 this: &mut UhProcessor<'_, Self>,
456 vtl: GuestVtl,
457 check_rflags: bool,
458 dev: &impl CpuIo,
459 ) -> bool;
460 fn set_pending_exception(
465 this: &mut UhProcessor<'_, Self>,
466 vtl: GuestVtl,
467 event: hvdef::HvX64PendingExceptionEvent,
468 );
469
470 fn intercept_message_state(
471 this: &UhProcessor<'_, Self>,
472 vtl: GuestVtl,
473 include_optional_state: bool,
474 ) -> InterceptMessageState;
475
476 fn cr0(this: &UhProcessor<'_, Self>, vtl: GuestVtl) -> u64;
479 fn cr4(this: &UhProcessor<'_, Self>, vtl: GuestVtl) -> u64;
480
481 fn cr_intercept_registration(
482 this: &mut UhProcessor<'_, Self>,
483 intercept_control: HvRegisterCrInterceptControl,
484 );
485
486 fn untrusted_synic_mut(&mut self) -> Option<&mut ProcessorSynic>;
487}
488
489#[cfg_attr(guest_arch = "aarch64", expect(dead_code))]
490#[derive(Inspect, Debug)]
491#[inspect(tag = "reason")]
492pub(crate) enum SidecarExitReason {
493 #[inspect(transparent)]
494 Exit(SidecarRemoveExit),
495 #[inspect(transparent)]
496 TaskRequest(Arc<str>),
497 ManualRequest,
498}
499
500#[cfg_attr(guest_arch = "aarch64", expect(dead_code))]
501#[derive(Inspect, Debug)]
502#[inspect(tag = "exit")]
503pub(crate) enum SidecarRemoveExit {
504 Msr {
505 #[inspect(hex)]
506 msr: u32,
507 value: Option<u64>,
508 },
509 Io {
510 #[inspect(hex)]
511 port: u16,
512 write: bool,
513 },
514 Mmio {
515 #[inspect(hex)]
516 gpa: u64,
517 write: bool,
518 },
519 Hypercall {
520 #[inspect(debug)]
521 code: hvdef::HypercallCode,
522 },
523 Cpuid {
524 #[inspect(hex)]
525 leaf: u32,
526 #[inspect(hex)]
527 subleaf: u32,
528 },
529 Hypervisor {
530 #[inspect(debug)]
531 message: hvdef::HvMessageType,
532 },
533}
534
535impl UhVpInner {
536 pub fn new(cpu_index: u32, vp_info: TargetVpInfo) -> Self {
538 Self {
539 wake_reasons: Default::default(),
540 message_queues: VtlArray::from_fn(|_| MessageQueues::new()),
541 waker: Default::default(),
542 cpu_index,
543 vp_info,
544 sidecar_exit_reason: Default::default(),
545 }
546 }
547
548 pub fn post_message(&self, vtl: GuestVtl, sint: u8, message: &HvMessage) {
550 if self.message_queues[vtl].enqueue_message(sint, message) {
551 self.wake(vtl, WakeReason::MESSAGE_QUEUES);
552 }
553 }
554
555 pub fn wake(&self, vtl: GuestVtl, reason: WakeReason) {
556 let reason = u64::from(reason.0) << (vtl as u8 * 32);
557 if self.wake_reasons.fetch_or(reason, Ordering::Release) & reason == 0 {
558 if let Some(waker) = &*self.waker.read() {
559 waker.wake_by_ref();
560 }
561 }
562 }
563
564 pub fn wake_vtl2(&self) {
565 if let Some(waker) = &*self.waker.read() {
566 waker.wake_by_ref();
567 }
568 }
569
570 pub fn set_sidecar_exit_reason(&self, reason: SidecarExitReason) {
571 self.sidecar_exit_reason.lock().get_or_insert_with(|| {
572 tracing::info!(CVM_ALLOWED, "sidecar exit");
573 tracing::info!(CVM_CONFIDENTIAL, ?reason, "sidecar exit");
574 reason
575 });
576 }
577}
578
579#[derive(Debug, Error)]
581pub enum UhRunVpError {
582 #[error("failed to run")]
584 Run(#[source] hcl::ioctl::Error),
585 #[error("sidecar run error")]
586 Sidecar(#[source] sidecar_client::SidecarError),
587 #[error("failed to access state for emulation")]
589 EmulationState(#[source] hcl::ioctl::Error),
590 #[error("failed to translate GVA")]
592 TranslateGva(#[source] hcl::ioctl::Error),
593 #[error("failed VTL access check")]
595 VtlAccess(#[source] hcl::ioctl::Error),
596 #[error("failed to advance rip")]
598 AdvanceRip(#[source] hcl::ioctl::Error),
599 #[error("failed to set pending event")]
601 Event(#[source] hcl::ioctl::Error),
602 #[error("guest accessed unaccepted gpa {0}")]
604 UnacceptedMemoryAccess(u64),
605 #[error("state access error")]
607 State(#[source] vp_state::Error),
608 #[error("invalid vmcb")]
610 InvalidVmcb,
611 #[error("unknown exit {0:#x?}")]
612 UnknownVmxExit(x86defs::vmx::VmxExit),
613 #[error("bad guest state on VP.ENTER")]
614 VmxBadGuestState,
615 #[error("failed to read hypercall parameters")]
616 HypercallParameters(#[source] guestmem::GuestMemoryError),
617 #[error("failed to write hypercall result")]
618 HypercallResult(#[source] guestmem::GuestMemoryError),
619 #[error("unexpected debug exception with dr6 value {0:#x}")]
620 UnexpectedDebugException(u64),
621 #[error("invalid intercepted vtl {0:?}")]
623 InvalidInterceptedVtl(u8),
624 #[error("access to state blocked by another vtl")]
625 StateAccessDenied,
626}
627
628#[derive(Debug, Error)]
630pub enum ProcessorError {
631 #[error("hcl error")]
633 Ioctl(#[from] hcl::ioctl::Error),
634 #[error("state access error")]
636 State(#[from] vp_state::Error),
637 #[error("operation not supported")]
639 NotSupported,
640}
641
642impl<T: Backing> UhProcessor<'_, T> {
643 fn inspect_extra(&mut self, resp: &mut inspect::Response<'_>) {
644 resp.child("stats", |req| {
645 if let Ok(stats) = hcl::stats::vp_stats() {
647 let stats = &stats[self.vp_index().index() as usize];
648 req.respond()
649 .counter("vtl_transitions", stats.vtl_transitions)
650 .counter(
651 "spurious_exits",
652 stats.vtl_transitions.saturating_sub(self.kernel_returns),
653 );
654 }
655 })
656 .field(
657 "last_enter_modes",
658 self.runner
659 .enter_mode()
660 .map(|&mut v| inspect::AsHex(u8::from(v))),
661 )
662 .field("sidecar", self.runner.is_sidecar())
663 .field(
664 "sidecar_base_cpu",
665 self.partition.hcl.sidecar_base_cpu(self.vp_index().index()),
666 );
667
668 T::inspect_extra(self, resp);
669 }
670
671 #[cfg(guest_arch = "x86_64")]
672 fn handle_debug_exception(&mut self, vtl: GuestVtl) -> Result<(), VpHaltReason<UhRunVpError>> {
673 if vtl == GuestVtl::Vtl0 {
675 let debug_regs: virt::x86::vp::DebugRegisters = self
676 .access_state(Vtl::Vtl0)
677 .debug_regs()
678 .expect("register query should not fail");
679
680 let dr = [
681 debug_regs.dr0,
682 debug_regs.dr1,
683 debug_regs.dr2,
684 debug_regs.dr3,
685 ];
686
687 if debug_regs.dr6 & x86defs::DR6_SINGLE_STEP != 0 {
688 return Err(VpHaltReason::SingleStep);
689 }
690
691 const BREAKPOINT_INDEX_OFFSET: usize = 4;
693 let i = debug_regs.dr6.trailing_zeros() as usize;
694 if i >= BREAKPOINT_INDEX_OFFSET {
695 return Err(VpHaltReason::InvalidVmState(
697 UhRunVpError::UnexpectedDebugException(debug_regs.dr6),
698 ));
699 }
700 let bp = virt::x86::HardwareBreakpoint::from_dr7(debug_regs.dr7, dr[i], i);
701
702 return Err(VpHaltReason::HwBreak(bp));
703 }
704
705 panic!("unexpected debug exception in VTL {:?}", vtl);
706 }
707}
708
709impl<'p, T: Backing> Processor for UhProcessor<'p, T> {
710 type Error = ProcessorError;
711 type RunVpError = UhRunVpError;
712 type StateAccess<'a>
713 = T::StateAccess<'p, 'a>
714 where
715 Self: 'a;
716
717 #[cfg(guest_arch = "aarch64")]
718 fn set_debug_state(
719 &mut self,
720 _vtl: Vtl,
721 _state: Option<&virt::x86::DebugState>,
722 ) -> Result<(), Self::Error> {
723 Err(ProcessorError::NotSupported)
724 }
725
726 #[cfg(guest_arch = "x86_64")]
727 fn set_debug_state(
728 &mut self,
729 vtl: Vtl,
730 state: Option<&virt::x86::DebugState>,
731 ) -> Result<(), Self::Error> {
732 if vtl == Vtl::Vtl0 {
734 let mut db: [u64; 4] = [0; 4];
735 let mut rflags =
736 x86defs::RFlags::from(self.access_state(Vtl::Vtl0).registers().unwrap().rflags);
737 let mut dr7: u64 = 0;
738
739 if let Some(state) = state {
740 rflags.set_trap(state.single_step);
741 for (i, bp) in state.breakpoints.iter().enumerate() {
742 if let Some(bp) = bp {
743 db[i] = bp.address;
744 dr7 |= bp.dr7_bits(i);
745 }
746 }
747 }
748
749 let debug_registers = virt::x86::vp::DebugRegisters {
750 dr0: db[0],
751 dr1: db[1],
752 dr2: db[2],
753 dr3: db[3],
754 dr6: 0,
755 dr7,
756 };
757
758 let mut access_state = self.access_state(vtl);
759
760 access_state.set_debug_regs(&debug_registers)?;
761
762 let registers = {
763 let mut registers = access_state.registers().unwrap();
764 registers.rflags = rflags.into();
765 registers
766 };
767 access_state.set_registers(®isters)?;
768
769 return Ok(());
770 }
771
772 panic!("unexpected set debug state in VTL {:?}", vtl);
773 }
774
775 async fn run_vp(
776 &mut self,
777 mut stop: StopVp<'_>,
778 dev: &impl CpuIo,
779 ) -> Result<Infallible, VpHaltReason<UhRunVpError>> {
780 if self.runner.is_sidecar() {
781 if self.force_exit_sidecar && !self.signaled_sidecar_exit {
782 self.inner
783 .set_sidecar_exit_reason(SidecarExitReason::ManualRequest);
784 self.signaled_sidecar_exit = true;
785 return Err(VpHaltReason::Cancel);
786 }
787 } else {
788 let mut current = Default::default();
789 affinity::get_current_thread_affinity(&mut current).unwrap();
790 assert_eq!(¤t, CpuSet::new().set(self.inner.cpu_index));
791
792 nice::nice(1);
795 }
796
797 let mut last_waker = None;
798
799 let vtl0_wakes = WakeReason::new()
801 .with_message_queues(true)
802 .with_intcon(true);
803 let vtl1_wakes = WakeReason::new().with_message_queues(true);
804 self.inner.wake_reasons.fetch_or(
805 ((vtl1_wakes.0 as u64) << 32) | (vtl0_wakes.0 as u64),
806 Ordering::Relaxed,
807 );
808
809 let mut first_scan_irr = true;
810
811 loop {
812 poll_fn(|cx| {
814 loop {
815 stop.check()?;
816
817 self.runner.clear_cancel();
819
820 self.vmtime.cancel_timeout();
822
823 if !last_waker
825 .as_ref()
826 .is_some_and(|waker| cx.waker().will_wake(waker))
827 {
828 last_waker = Some(cx.waker().clone());
829 self.inner.waker.write().clone_from(&last_waker);
830 }
831
832 let scan_irr = if self.inner.wake_reasons.load(Ordering::Relaxed) != 0 {
834 self.handle_wake().map_err(VpHaltReason::Hypervisor)?
835 } else {
836 [false, false].into()
837 };
838
839 if T::process_interrupts(self, scan_irr, &mut first_scan_irr, dev)? {
840 continue;
841 }
842
843 if let Some(timeout) = self.vmtime.get_timeout() {
845 let deadline = self.vmtime.host_time(timeout);
846 if self.timer.poll_timer(cx, deadline).is_ready() {
847 continue;
848 }
849 }
850
851 return <Result<_, VpHaltReason<_>>>::Ok(()).into();
852 }
853 })
854 .await?;
855
856 if let Some(idle_control) = &mut self.idle_control {
858 if !idle_control.pre_block() {
859 yield_now().await;
860 continue;
861 }
862 }
863
864 if let Some(mode) = self.runner.enter_mode() {
865 *mode = self
866 .partition
867 .enter_modes_atomic
868 .load(Ordering::Relaxed)
869 .into();
870 }
871
872 minircu::global().quiesce();
875
876 T::run_vp(self, dev, &mut stop).await?;
877 self.kernel_returns += 1;
878 }
879 }
880
881 fn flush_async_requests(&mut self) -> Result<(), Self::RunVpError> {
882 if self.inner.wake_reasons.load(Ordering::Relaxed) != 0 {
883 let scan_irr = self.handle_wake()?;
884 for vtl in [GuestVtl::Vtl1, GuestVtl::Vtl0] {
885 if scan_irr[vtl] {
886 T::poll_apic(self, vtl, true)?;
887 }
888 }
889 }
890 self.runner.flush_deferred_state();
891 Ok(())
892 }
893
894 fn access_state(&mut self, vtl: Vtl) -> Self::StateAccess<'_> {
895 T::access_vp_state(self, vtl.try_into().unwrap())
896 }
897
898 fn vtl_inspectable(&self, vtl: Vtl) -> bool {
899 match vtl {
900 Vtl::Vtl0 => true,
901 Vtl::Vtl1 => T::vtl1_inspectable(self),
902 Vtl::Vtl2 => false,
903 }
904 }
905}
906
907impl<'a, T: Backing> UhProcessor<'a, T> {
908 pub(super) fn new(
909 driver: &impl Driver,
910 partition: &'a UhPartitionInner,
911 vp_info: TargetVpInfo,
912 idle_control: Option<&'a mut IdleControl>,
913 ) -> Result<Self, Error> {
914 let inner = partition.vp(vp_info.base.vp_index).unwrap();
915 let mut runner = partition
916 .hcl
917 .runner(inner.vp_index().index(), idle_control.is_none())
918 .unwrap();
919
920 let backing_shared = T::shared(&partition.backing_shared);
921
922 let backing = T::new(
923 BackingParams {
924 partition,
925 vp_info: &vp_info,
926 runner: &mut runner,
927 },
928 backing_shared,
929 )?;
930
931 let mut vp = Self {
932 partition,
933 inner,
934 runner,
935 idle_control,
936 kernel_returns: 0,
937 crash_reg: [0; hvdef::HV_X64_GUEST_CRASH_PARAMETER_MSRS],
938 _not_send: PhantomData,
939 backing,
940 shared: backing_shared,
941 vmtime: partition
942 .vmtime
943 .access(format!("vp-{}", vp_info.base.vp_index.index())),
944 timer: driver.new_dyn_timer(),
945 force_exit_sidecar: false,
946 signaled_sidecar_exit: false,
947 vtls_tlb_locked: VtlsTlbLocked {
948 vtl1: VtlArray::new(false),
949 vtl2: VtlArray::new(false),
950 },
951 exit_activities: Default::default(),
952 };
953
954 T::init(&mut vp);
955
956 Ok(vp)
957 }
958
959 fn handle_wake(&mut self) -> Result<VtlArray<bool, 2>, UhRunVpError> {
961 let wake_reasons_raw = self.inner.wake_reasons.swap(0, Ordering::SeqCst);
962 let wake_reasons_vtl: [WakeReason; 2] = zerocopy::transmute!(wake_reasons_raw);
963 for (vtl, wake_reasons) in [
964 (GuestVtl::Vtl1, wake_reasons_vtl[1]),
965 (GuestVtl::Vtl0, wake_reasons_vtl[0]),
966 ] {
967 if wake_reasons.message_queues() {
968 let pending_sints = self.inner.message_queues[vtl].pending_sints();
969 if pending_sints != 0 {
970 let pending_sints = self.inner.message_queues[vtl].pending_sints();
972 let mut masked_sints = 0;
973
974 for sint in 0..NUM_SINTS as u8 {
976 if pending_sints & (1 << sint) == 0 {
977 continue;
978 }
979 let sint_msr = if let Some(hv) = self.backing.hv(vtl).as_ref() {
980 hv.synic.sint(sint)
981 } else {
982 #[cfg(guest_arch = "x86_64")]
983 let sint_reg =
984 HvX64RegisterName(HvX64RegisterName::Sint0.0 + sint as u32);
985 #[cfg(guest_arch = "aarch64")]
986 let sint_reg =
987 HvArm64RegisterName(HvArm64RegisterName::Sint0.0 + sint as u32);
988 self.runner.get_vp_register(vtl, sint_reg).unwrap().as_u64()
989 };
990 masked_sints |= (HvSynicSint::from(sint_msr).masked() as u16) << sint;
991 }
992
993 self.inner.message_queues[vtl].post_pending_messages(masked_sints, |_, _| {
995 Err(HvError::InvalidSynicState)
996 });
997
998 self.request_sint_notifications(vtl, pending_sints & !masked_sints);
999 }
1000 }
1001
1002 if wake_reasons.extint() {
1003 T::request_extint_readiness(self);
1004 }
1005
1006 #[cfg(guest_arch = "x86_64")]
1007 if wake_reasons.hv_start_enable_vtl_vp() {
1008 T::handle_vp_start_enable_vtl_wake(self, vtl)?;
1009 }
1010
1011 #[cfg(guest_arch = "x86_64")]
1012 if wake_reasons.update_proxy_irr_filter() {
1013 debug_assert!(self.partition.isolation.is_hardware_isolated());
1015 self.update_proxy_irr_filter(vtl);
1016 }
1017 }
1018
1019 Ok(wake_reasons_vtl.map(|w| w.intcon()).into())
1020 }
1021
1022 fn request_sint_notifications(&mut self, vtl: GuestVtl, sints: u16) {
1023 if sints == 0 {
1024 return;
1025 }
1026
1027 let untrusted_sints = if let Some(hv) = self.backing.hv_mut(vtl).as_mut() {
1029 let proxied_sints = hv.synic.proxied_sints();
1030 hv.synic.request_sint_readiness(sints & !proxied_sints);
1031 proxied_sints
1032 } else {
1033 !0
1034 };
1035
1036 if sints & untrusted_sints != 0 {
1037 assert_eq!(vtl, GuestVtl::Vtl0);
1038 T::request_untrusted_sint_readiness(self, sints & untrusted_sints);
1039 }
1040 }
1041
1042 fn vp_index(&self) -> VpIndex {
1043 self.inner.vp_index()
1044 }
1045
1046 #[cfg(guest_arch = "x86_64")]
1047 fn write_crash_msr(&mut self, msr: u32, value: u64, vtl: GuestVtl) -> Result<(), MsrError> {
1048 match msr {
1049 hvdef::HV_X64_MSR_GUEST_CRASH_CTL => {
1050 let crash = VtlCrash {
1051 vp_index: self.vp_index(),
1052 last_vtl: vtl,
1053 control: hvdef::GuestCrashCtl::from(value),
1054 parameters: self.crash_reg,
1055 };
1056 tracelimit::warn_ratelimited!(
1057 CVM_ALLOWED,
1058 ?crash,
1059 "Guest has reported system crash"
1060 );
1061
1062 if crash.control.crash_message() {
1063 let message_gpa = crash.parameters[3];
1064 let message_size = crash.parameters[4];
1065 let mut message = vec![0; message_size as usize];
1066 match self.partition.gm[vtl].read_at(message_gpa, &mut message) {
1067 Ok(()) => {
1068 let message = String::from_utf8_lossy(&message).into_owned();
1069 tracelimit::warn_ratelimited!(
1070 CVM_CONFIDENTIAL,
1071 message,
1072 "Guest has reported a system crash message"
1073 );
1074 }
1075 Err(e) => {
1076 tracelimit::warn_ratelimited!(
1077 CVM_ALLOWED,
1078 ?e,
1079 "Failed to read crash message"
1080 );
1081 }
1082 }
1083 }
1084
1085 self.partition.crash_notification_send.send(crash);
1086 }
1087 hvdef::HV_X64_MSR_GUEST_CRASH_P0
1088 | hvdef::HV_X64_MSR_GUEST_CRASH_P1
1089 | hvdef::HV_X64_MSR_GUEST_CRASH_P2
1090 | hvdef::HV_X64_MSR_GUEST_CRASH_P3
1091 | hvdef::HV_X64_MSR_GUEST_CRASH_P4 => {
1092 self.crash_reg[(msr - hvdef::HV_X64_MSR_GUEST_CRASH_P0) as usize] = value;
1093 }
1094 _ => return Err(MsrError::Unknown),
1095 }
1096 Ok(())
1097 }
1098
1099 #[cfg(guest_arch = "x86_64")]
1100 fn read_crash_msr(&self, msr: u32, _vtl: GuestVtl) -> Result<u64, MsrError> {
1101 let v = match msr {
1102 hvdef::HV_X64_MSR_GUEST_CRASH_CTL => hvdef::GuestCrashCtl::new()
1105 .with_crash_notify(true)
1106 .with_crash_message(true)
1107 .into(),
1108 hvdef::HV_X64_MSR_GUEST_CRASH_P0 => self.crash_reg[0],
1109 hvdef::HV_X64_MSR_GUEST_CRASH_P1 => self.crash_reg[1],
1110 hvdef::HV_X64_MSR_GUEST_CRASH_P2 => self.crash_reg[2],
1111 hvdef::HV_X64_MSR_GUEST_CRASH_P3 => self.crash_reg[3],
1112 hvdef::HV_X64_MSR_GUEST_CRASH_P4 => self.crash_reg[4],
1113 _ => return Err(MsrError::Unknown),
1114 };
1115 Ok(v)
1116 }
1117
1118 #[cfg(guest_arch = "x86_64")]
1120 async fn emulate<D: CpuIo>(
1121 &mut self,
1122 devices: &D,
1123 interruption_pending: bool,
1124 vtl: GuestVtl,
1125 cache: T::EmulationCache,
1126 ) -> Result<(), VpHaltReason<UhRunVpError>>
1127 where
1128 for<'b> UhEmulationState<'b, 'a, D, T>:
1129 virt_support_x86emu::emulate::EmulatorSupport<Error = UhRunVpError>,
1130 {
1131 let guest_memory = &self.partition.gm[vtl];
1132 let (kx_guest_memory, ux_guest_memory) = match vtl {
1133 GuestVtl::Vtl0 => (
1134 &self.partition.vtl0_kernel_exec_gm,
1135 &self.partition.vtl0_user_exec_gm,
1136 ),
1137 GuestVtl::Vtl1 => (guest_memory, guest_memory),
1138 };
1139 let emu_mem = virt_support_x86emu::emulate::EmulatorMemoryAccess {
1140 gm: guest_memory,
1141 kx_gm: kx_guest_memory,
1142 ux_gm: ux_guest_memory,
1143 };
1144 let mut emulation_state = UhEmulationState {
1145 vp: &mut *self,
1146 interruption_pending,
1147 devices,
1148 vtl,
1149 cache,
1150 };
1151
1152 virt_support_x86emu::emulate::emulate(&mut emulation_state, &emu_mem, devices).await
1153 }
1154
1155 #[cfg(guest_arch = "aarch64")]
1157 async fn emulate<D: CpuIo>(
1158 &mut self,
1159 devices: &D,
1160 intercept_state: &aarch64emu::InterceptState,
1161 vtl: GuestVtl,
1162 cache: T::EmulationCache,
1163 ) -> Result<(), VpHaltReason<UhRunVpError>>
1164 where
1165 for<'b> UhEmulationState<'b, 'a, D, T>:
1166 virt_support_aarch64emu::emulate::EmulatorSupport<Error = UhRunVpError>,
1167 {
1168 let guest_memory = &self.partition.gm[vtl];
1169 virt_support_aarch64emu::emulate::emulate(
1170 &mut UhEmulationState {
1171 vp: &mut *self,
1172 interruption_pending: intercept_state.interruption_pending,
1173 devices,
1174 vtl,
1175 cache,
1176 },
1177 intercept_state,
1178 guest_memory,
1179 devices,
1180 )
1181 .await
1182 }
1183
1184 #[cfg(guest_arch = "x86_64")]
1185 fn update_proxy_irr_filter(&mut self, vtl: GuestVtl) {
1186 assert_eq!(vtl, GuestVtl::Vtl0);
1187 let mut irr_bits: BitArray<[u32; 8], Lsb0> = BitArray::new(Default::default());
1188
1189 if let Some(hv) = self.backing.hv(vtl).as_ref() {
1191 for sint in 0..NUM_SINTS as u8 {
1192 let sint_msr = hv.synic.sint(sint);
1193 let hv_sint = HvSynicSint::from(sint_msr);
1194 if (hv_sint.proxy() || self.partition.vmbus_relay) && !hv_sint.masked() {
1197 irr_bits.set(hv_sint.vector() as usize, true);
1198 }
1199 }
1200 }
1201
1202 self.partition.fill_device_vectors(vtl, &mut irr_bits);
1204
1205 self.runner
1207 .update_proxy_irr_filter_vtl0(&irr_bits.into_inner());
1208 }
1209}
1210
1211fn signal_mnf(dev: &impl CpuIo, connection_id: u32) {
1212 if let Err(err) = dev.signal_synic_event(Vtl::Vtl0, connection_id, 0) {
1213 tracelimit::warn_ratelimited!(
1214 CVM_ALLOWED,
1215 error = &err as &dyn std::error::Error,
1216 connection_id,
1217 "failed to signal mnf"
1218 );
1219 }
1220}
1221
1222async fn yield_now() {
1224 let mut yielded = false;
1225 poll_fn(|cx| {
1226 if !yielded {
1227 cx.waker().wake_by_ref();
1229 yielded = true;
1230 Poll::Pending
1231 } else {
1232 Poll::Ready(())
1233 }
1234 })
1235 .await;
1236}
1237
1238struct UhEmulationState<'a, 'b, T: CpuIo, U: Backing> {
1239 vp: &'a mut UhProcessor<'b, U>,
1240 interruption_pending: bool,
1241 devices: &'a T,
1242 vtl: GuestVtl,
1243 cache: U::EmulationCache,
1244}
1245
1246struct UhHypercallHandler<'a, 'b, T, B: Backing> {
1247 vp: &'a mut UhProcessor<'b, B>,
1248 bus: &'a T,
1249 trusted: bool,
1258 intercepted_vtl: GuestVtl,
1259}
1260
1261impl<T, B: Backing> UhHypercallHandler<'_, '_, T, B> {
1262 fn target_vtl_no_higher(&self, target_vtl: Vtl) -> Result<GuestVtl, HvError> {
1263 if Vtl::from(self.intercepted_vtl) < target_vtl {
1264 return Err(HvError::AccessDenied);
1265 }
1266 Ok(target_vtl.try_into().unwrap())
1267 }
1268}
1269
1270impl<T, B: Backing> hv1_hypercall::GetVpIndexFromApicId for UhHypercallHandler<'_, '_, T, B> {
1271 fn get_vp_index_from_apic_id(
1272 &mut self,
1273 partition_id: u64,
1274 target_vtl: Vtl,
1275 apic_ids: &[u32],
1276 vp_indices: &mut [u32],
1277 ) -> HvRepResult {
1278 tracing::debug!(partition_id, ?target_vtl, "HvGetVpIndexFromApicId");
1279
1280 if partition_id != hvdef::HV_PARTITION_ID_SELF {
1281 return Err((HvError::InvalidPartitionId, 0));
1282 }
1283
1284 let _target_vtl = self.target_vtl_no_higher(target_vtl).map_err(|e| (e, 0))?;
1285
1286 #[cfg(guest_arch = "aarch64")]
1287 if true {
1288 let _ = apic_ids;
1289 let _ = vp_indices;
1290 todo!("AARCH64_TODO");
1291 }
1292
1293 #[cfg(guest_arch = "x86_64")]
1294 for (i, (&apic_id, vp_index)) in apic_ids.iter().zip(vp_indices).enumerate() {
1295 *vp_index = self
1296 .vp
1297 .partition
1298 .vps
1299 .iter()
1300 .find(|vp| vp.vp_info.apic_id == apic_id)
1301 .ok_or((HvError::InvalidParameter, i))?
1302 .vp_info
1303 .base
1304 .vp_index
1305 .index()
1306 }
1307
1308 Ok(())
1309 }
1310}
1311
1312#[cfg(guest_arch = "aarch64")]
1313impl<T: CpuIo, B: Backing> Arm64RegisterState for UhHypercallHandler<'_, '_, T, B> {
1314 fn pc(&mut self) -> u64 {
1315 self.vp
1316 .runner
1317 .get_vp_register(self.intercepted_vtl, HvArm64RegisterName::XPc)
1318 .expect("get vp register cannot fail")
1319 .as_u64()
1320 }
1321
1322 fn set_pc(&mut self, pc: u64) {
1323 self.vp
1324 .runner
1325 .set_vp_register(self.intercepted_vtl, HvArm64RegisterName::XPc, pc.into())
1326 .expect("set vp register cannot fail");
1327 }
1328
1329 fn x(&mut self, n: u8) -> u64 {
1330 self.vp
1331 .runner
1332 .get_vp_register(
1333 self.intercepted_vtl,
1334 HvArm64RegisterName(HvArm64RegisterName::X0.0 + n as u32),
1335 )
1336 .expect("get vp register cannot fail")
1337 .as_u64()
1338 }
1339
1340 fn set_x(&mut self, n: u8, v: u64) {
1341 self.vp
1342 .runner
1343 .set_vp_register(
1344 self.intercepted_vtl,
1345 HvArm64RegisterName(HvArm64RegisterName::X0.0 + n as u32),
1346 v.into(),
1347 )
1348 .expect("set vp register cannot fail")
1349 }
1350}
1351
1352impl<T: CpuIo, B: Backing> hv1_hypercall::PostMessage for UhHypercallHandler<'_, '_, T, B> {
1353 fn post_message(&mut self, connection_id: u32, message: &[u8]) -> hvdef::HvResult<()> {
1354 tracing::trace!(
1355 connection_id,
1356 self.trusted,
1357 "handling post message intercept"
1358 );
1359
1360 self.bus.post_synic_message(
1361 self.intercepted_vtl.into(),
1362 connection_id,
1363 self.trusted,
1364 message,
1365 )
1366 }
1367}
1368
1369impl<T: CpuIo, B: Backing> hv1_hypercall::SignalEvent for UhHypercallHandler<'_, '_, T, B> {
1370 fn signal_event(&mut self, connection_id: u32, flag: u16) -> hvdef::HvResult<()> {
1371 tracing::trace!(connection_id, "handling signal event intercept");
1372
1373 self.bus
1374 .signal_synic_event(self.intercepted_vtl.into(), connection_id, flag)
1375 }
1376}
1377
1378impl<T: CpuIo, B: Backing> UhHypercallHandler<'_, '_, T, B> {
1379 fn retarget_virtual_interrupt(
1380 &mut self,
1381 device_id: u64,
1382 address: u64,
1383 data: u32,
1384 vector: u32,
1385 multicast: bool,
1386 target_processors: ProcessorSet<'_>,
1387 ) -> hvdef::HvResult<()> {
1388 let target_processors = Vec::from_iter(target_processors);
1389 let vpci_params = vmcore::vpci_msi::VpciInterruptParameters {
1390 vector,
1391 multicast,
1392 target_processors: &target_processors,
1393 };
1394
1395 self.vp
1396 .partition
1397 .software_devices
1398 .as_ref()
1399 .expect("should exist if this intercept is registered or this is a CVM")
1400 .retarget_interrupt(device_id, address, data, &vpci_params)
1401 }
1402}
1403
1404impl<T, B: Backing> hv1_hypercall::ExtendedQueryCapabilities for UhHypercallHandler<'_, '_, T, B> {
1405 fn query_extended_capabilities(&mut self) -> hvdef::HvResult<u64> {
1406 Err(HvError::InvalidHypercallCode)
1410 }
1411}