1#![expect(missing_docs)]
11#![forbid(unsafe_code)]
12
13pub mod ring_buffer;
14
15use crate::ring_buffer::MemoryBlockRingBuffer;
16use anyhow::Context;
17use anyhow::Result;
18use futures::StreamExt;
19use futures_concurrency::stream::Merge;
20use guid::Guid;
21use inspect::InspectMut;
22use mesh::rpc::RpcSend;
23use pal_async::driver::SpawnDriver;
24use std::future::Future;
25use std::future::pending;
26use std::pin::pin;
27use std::sync::Arc;
28use task_control::AsyncRun;
29use task_control::Cancelled;
30use task_control::InspectTaskMut;
31use task_control::StopTask;
32use task_control::TaskControl;
33use tracing::Instrument;
34use user_driver::DmaClient;
35use user_driver::memory::MemoryBlock;
36use vmbus_channel::ChannelClosed;
37use vmbus_channel::RawAsyncChannel;
38use vmbus_channel::SignalVmbusChannel;
39use vmbus_channel::bus::GpadlRequest;
40use vmbus_channel::bus::OpenData;
41use vmbus_client::ChannelRequest;
42use vmbus_client::OfferInfo;
43use vmbus_client::OpenOutput;
44use vmbus_client::OpenRequest;
45use vmbus_core::protocol::GpadlId;
46use vmbus_core::protocol::UserDefinedData;
47use vmbus_relay::InterceptChannelRequest;
48use vmbus_ring::IncomingRing;
49use vmbus_ring::OutgoingRing;
50use vmbus_ring::PAGE_SIZE;
51use vmcore::interrupt::Interrupt;
52use vmcore::notify::Notify;
53use vmcore::notify::PolledNotify;
54use vmcore::save_restore::NoSavedState;
55use vmcore::save_restore::SavedStateBlob;
56use vmcore::save_restore::SavedStateRoot;
57use zerocopy::FromZeros;
58
59pub enum OfferResponse {
60 Ignore,
61 Open,
62}
63
64pub trait SimpleVmbusClientDevice {
65 type SavedState: SavedStateRoot + Send + Sync;
67
68 type Runner: 'static + Send + Sync;
70
71 fn inspect(&mut self, req: inspect::Request<'_>, runner: Option<&mut Self::Runner>);
73
74 fn instance_id(&self) -> Guid;
76
77 fn offer(&self, offer: &vmbus_core::protocol::OfferChannel) -> OfferResponse;
79
80 fn open(
84 &mut self,
85 channel_idx: u16,
86 channel: RawAsyncChannel<MemoryBlockRingBuffer>,
87 ) -> Result<Self::Runner>;
88
89 fn close(&mut self, channel_idx: u16);
92
93 fn supports_save_restore(
95 &mut self,
96 ) -> Option<
97 &mut dyn SaveRestoreSimpleVmbusClientDevice<
98 SavedState = Self::SavedState,
99 Runner = Self::Runner,
100 >,
101 >;
102}
103
104pub trait SimpleVmbusClientDeviceAsync: SimpleVmbusClientDevice + 'static + Send + Sync {
105 fn run(
110 &mut self,
111 stop: &mut StopTask<'_>,
112 runner: &mut Self::Runner,
113 ) -> impl Send + Future<Output = Result<(), Cancelled>>;
114}
115
116pub trait SaveRestoreSimpleVmbusClientDevice: SimpleVmbusClientDevice {
121 fn save_open(&mut self, runner: &Self::Runner) -> Self::SavedState;
125
126 fn restore_open(
130 &mut self,
131 state: Self::SavedState,
132 channel: RawAsyncChannel<MemoryBlockRingBuffer>,
133 ) -> Result<Self::Runner>;
134}
135
136#[derive(InspectMut)]
137pub struct SimpleVmbusClientDeviceWrapper<T: SimpleVmbusClientDeviceAsync> {
138 instance_id: Guid,
139 #[inspect(skip)]
140 spawner: Arc<dyn SpawnDriver>,
141 #[inspect(mut)]
142 vmbus_listener: TaskControl<SimpleVmbusClientDeviceTask<T>, SimpleVmbusClientDeviceTaskState>,
143}
144
145impl<T: SimpleVmbusClientDeviceAsync> SimpleVmbusClientDeviceWrapper<T> {
146 pub fn new(
148 driver: impl SpawnDriver + Clone,
149 dma_alloc: Arc<dyn DmaClient>,
150 device: T,
151 ) -> Result<Self> {
152 let spawner = Arc::new(driver.clone());
153 Ok(Self {
154 instance_id: device.instance_id(),
155 vmbus_listener: TaskControl::new(SimpleVmbusClientDeviceTask::new(
156 device,
157 spawner.clone(),
158 dma_alloc,
159 )),
160 spawner,
161 })
162 }
163
164 pub fn instance_id(&self) -> Guid {
165 self.instance_id
166 }
167
168 pub fn detach(
169 mut self,
170 driver: impl SpawnDriver,
171 recv_relay: mesh::Receiver<InterceptChannelRequest>,
172 ) -> Result<()> {
173 let (send_disconnected, recv_disconnected) = mesh::oneshot();
174 self.vmbus_listener.insert(
175 &self.spawner,
176 format!("{}", self.instance_id),
177 SimpleVmbusClientDeviceTaskState {
178 offer: None,
179 recv_relay,
180 send_disconnected: Some(send_disconnected),
181 vtl_pages: None,
182 },
183 );
184 driver
185 .spawn(
186 format!("vmbus_relay_device {}", self.instance_id),
187 async move {
188 self.vmbus_listener.start();
189 let _ = recv_disconnected.await;
190 assert!(!self.vmbus_listener.stop().await);
191 if self.vmbus_listener.state().unwrap().vtl_pages.is_some() {
192 pending::<()>().await;
197 }
198 },
199 )
200 .detach();
201 Ok(())
202 }
203}
204
205struct RelayDeviceTask<T>(T);
206
207impl<T: SimpleVmbusClientDeviceAsync> AsyncRun<T::Runner> for RelayDeviceTask<T> {
208 async fn run(
209 &mut self,
210 stop: &mut StopTask<'_>,
211 runner: &mut T::Runner,
212 ) -> Result<(), Cancelled> {
213 stop.check()?;
214 self.0.run(stop, runner).await
215 }
216}
217
218impl<T: SimpleVmbusClientDeviceAsync> InspectTaskMut<T::Runner> for RelayDeviceTask<T> {
219 fn inspect_mut(&mut self, req: inspect::Request<'_>, runner: Option<&mut T::Runner>) {
220 self.0.inspect(req, runner)
221 }
222}
223
224#[derive(InspectMut)]
225struct SimpleVmbusClientDeviceTaskState {
226 offer: Option<OfferInfo>,
227 #[inspect(skip)]
228 recv_relay: mesh::Receiver<InterceptChannelRequest>,
229 #[inspect(skip)]
230 send_disconnected: Option<mesh::OneshotSender<()>>,
231 #[inspect(hex, with = "|x| x.as_ref().map(|x| inspect::iter_by_index(x.pfns()))")]
232 vtl_pages: Option<MemoryBlock>,
233}
234
235struct SimpleVmbusClientDeviceTask<T: SimpleVmbusClientDeviceAsync> {
236 device: TaskControl<RelayDeviceTask<T>, T::Runner>,
237 saved_state: Option<T::SavedState>,
238 spawner: Arc<dyn SpawnDriver>,
239 dma_alloc: Arc<dyn DmaClient>,
240}
241
242impl<T: SimpleVmbusClientDeviceAsync> AsyncRun<SimpleVmbusClientDeviceTaskState>
243 for SimpleVmbusClientDeviceTask<T>
244{
245 async fn run(
246 &mut self,
247 stop: &mut StopTask<'_>,
248 state: &mut SimpleVmbusClientDeviceTaskState,
249 ) -> Result<(), Cancelled> {
250 let check = stop.check_handle()?;
251 stop.until_stopped(self.process_messages(state, &check))
252 .await??;
253 state
254 .send_disconnected
255 .take()
256 .expect("task should not be restarted")
257 .send(());
258 Ok(())
259 }
260}
261
262impl<T: SimpleVmbusClientDeviceAsync> InspectTaskMut<SimpleVmbusClientDeviceTaskState>
263 for SimpleVmbusClientDeviceTask<T>
264{
265 fn inspect_mut(
266 &mut self,
267 req: inspect::Request<'_>,
268 state: Option<&mut SimpleVmbusClientDeviceTaskState>,
269 ) {
270 req.respond()
271 .merge(state)
272 .field_mut("device", &mut self.device)
273 .field("dma_alloc", &self.dma_alloc);
274 }
275}
276
277impl<T: SimpleVmbusClientDeviceAsync> SimpleVmbusClientDeviceTask<T> {
278 pub fn new(device: T, spawner: Arc<dyn SpawnDriver>, dma_alloc: Arc<dyn DmaClient>) -> Self {
279 Self {
280 device: TaskControl::new(RelayDeviceTask(device)),
281 saved_state: None,
282 spawner,
283 dma_alloc,
284 }
285 }
286
287 fn insert_runner(&mut self, state: &SimpleVmbusClientDeviceTaskState, runner: T::Runner) {
288 let offer = state.offer.as_ref().unwrap().offer;
289 self.device.insert(
290 &self.spawner,
291 format!("{}-{}", offer.interface_id, offer.instance_id),
292 runner,
293 );
294 }
295
296 async fn handle_offer(
298 &mut self,
299 offer: OfferInfo,
300 state: &mut SimpleVmbusClientDeviceTaskState,
301 ) -> Result<()> {
302 tracing::info!(?offer, "matching channel offered");
303
304 if offer.offer.is_dedicated != 1 {
305 tracing::warn!(offer = ?offer.offer, "All offers should be dedicated with Win8+ host")
306 }
307
308 if matches!(
309 self.device.task_mut().0.offer(&offer.offer),
310 OfferResponse::Ignore
311 ) {
312 return Ok(());
313 }
314
315 let interrupt_event = pal_event::Event::new();
316 let (memory, ring_gpadl_id) = self
317 .reserve_memory(state, &offer.request_send, 4)
318 .await
319 .context("reserve memory")?;
320 let guest_to_host_interrupt = offer.guest_to_host_interrupt.clone();
321 state.offer = Some(offer);
322 let offer = state.offer.as_ref().unwrap();
323 self.open_channel(&offer.request_send, ring_gpadl_id, &interrupt_event)
324 .await
325 .context("open channel")?;
326 let channel = self
327 .create_vmbus_channel(&memory, &interrupt_event, guest_to_host_interrupt)
328 .context("create vmbus queue")?;
329
330 let save_restore = self.device.task_mut().0.supports_save_restore();
331 let saved_state = self.saved_state.take();
332 let device_runner = if let Some(save_restore) = save_restore
333 && let Some(saved_state) = saved_state
334 {
335 save_restore
336 .restore_open(saved_state, channel)
337 .context("device restore_open callback")?
338 } else {
339 self.device
340 .task_mut()
341 .0
342 .open(offer.offer.subchannel_index, channel)
343 .context("device open callback")?
344 };
345 self.insert_runner(state, device_runner);
346 self.device.start();
347 Ok(())
348 }
349
350 async fn handle_start(&mut self, state: &mut SimpleVmbusClientDeviceTaskState) {
352 if self.device.is_running() {
353 return;
354 }
355
356 let offer = state.offer.take();
357 if offer.is_none() {
358 return;
359 }
360
361 if let Err(err) = self.handle_offer(offer.unwrap(), state).await {
363 tracing::error!(
364 err = err.as_ref() as &dyn std::error::Error,
365 "Failed to reconnect vmbus channel"
366 );
367 }
368 }
369
370 async fn cleanup_device_resources(&mut self, state: &mut SimpleVmbusClientDeviceTaskState) {
371 let Some(offer) = state.offer.as_mut() else {
372 return;
373 };
374
375 if let Some(vtl_pages) = &state.vtl_pages {
376 match offer
377 .request_send
378 .call(
379 ChannelRequest::TeardownGpadl,
380 GpadlId(vtl_pages.pfns()[1] as u32),
381 )
382 .await
383 {
384 Ok(()) => {
385 state.vtl_pages = None;
386 }
387 Err(err) => {
388 tracing::error!(
392 error = &err as &dyn std::error::Error,
393 "Failed to teardown gpadl -- leaking memory."
394 );
395 }
396 }
397 }
398 }
399
400 async fn handle_stop(&mut self, state: &mut SimpleVmbusClientDeviceTaskState) {
402 if !self.device.stop().await {
403 return;
404 }
405
406 {
416 let offer = state.offer.as_ref().expect("device opened");
417 offer
418 .request_send
419 .call(ChannelRequest::Close, ())
420 .await
421 .ok();
422 }
423 self.cleanup_device_resources(state).await;
427 let runner = self.device.remove();
428 let device = self.device.task_mut();
429 if let Some(save_restore) = device.0.supports_save_restore() {
430 self.saved_state = Some(save_restore.save_open(&runner));
431 }
432 drop(runner);
433 let offer = state.offer.as_ref().expect("device opened");
434 device.0.close(offer.offer.subchannel_index);
435 }
436
437 async fn reserve_memory(
440 &mut self,
441 state: &mut SimpleVmbusClientDeviceTaskState,
442 request_send: &mesh::Sender<ChannelRequest>,
443 page_count: usize,
444 ) -> Result<(MemoryBlock, GpadlId)> {
445 assert!(page_count >= 4);
448
449 let mem = self
450 .dma_alloc
451 .allocate_dma_buffer(page_count * PAGE_SIZE)
452 .context("allocating memory for vmbus rings")?;
453 state.vtl_pages = Some(mem.clone());
454 let buf: Vec<_> = [mem.len() as u64]
455 .iter()
456 .chain(mem.pfns())
457 .copied()
458 .collect();
459
460 let gpadl_id = GpadlId(state.vtl_pages.as_ref().unwrap().pfns()[1] as u32);
461 request_send
462 .call_failable(
463 ChannelRequest::Gpadl,
464 GpadlRequest {
465 id: gpadl_id,
466 count: 1,
467 buf,
468 },
469 )
470 .await
471 .context("registering gpadl")?;
472 Ok((mem, gpadl_id))
473 }
474
475 async fn open_channel(
477 &self,
478 request_send: &mesh::Sender<ChannelRequest>,
479 ring_gpadl_id: GpadlId,
480 event: &pal_event::Event,
481 ) -> Result<OpenOutput> {
482 let open_request = OpenRequest {
483 open_data: OpenData {
484 target_vp: Some(0),
485 ring_offset: 2,
486 ring_gpadl_id,
487 event_flag: !0,
488 connection_id: !0,
489 user_data: UserDefinedData::new_zeroed(),
490 },
491 incoming_event: Some(event.clone()),
492 use_vtl2_connection_id: true,
493 };
494
495 request_send
496 .call_failable(ChannelRequest::Open, open_request)
497 .instrument(tracing::info_span!(
498 "opening vmbus channel for intercepted device"
499 ))
500 .await
501 .context("open vmbus channel")
502 }
503
504 fn create_vmbus_channel(
506 &self,
507 mem: &MemoryBlock,
508 host_to_guest_event: &pal_event::Event,
509 guest_to_host_interrupt: Interrupt,
510 ) -> Result<RawAsyncChannel<MemoryBlockRingBuffer>> {
511 let (out_ring_mem, in_ring_mem) = (
512 mem.subblock(0, 2 * PAGE_SIZE),
513 mem.subblock(2 * PAGE_SIZE, 2 * PAGE_SIZE),
514 );
515 let (in_ring, out_ring) = (
516 IncomingRing::new(in_ring_mem.into()).unwrap(),
517 OutgoingRing::new(out_ring_mem.into()).unwrap(),
518 );
519
520 let signal = MemoryBlockChannelSignal {
521 event: Notify::from_event(host_to_guest_event.clone())
522 .pollable(self.spawner.as_ref())
523 .unwrap(),
524 interrupt: guest_to_host_interrupt,
525 };
526 Ok(RawAsyncChannel {
527 in_ring,
528 out_ring,
529 signal: Box::new(signal),
530 })
531 }
532
533 async fn handle_revoke(&mut self, state: &mut SimpleVmbusClientDeviceTaskState) {
535 let Some(offer) = state.offer.as_ref() else {
536 return;
537 };
538 tracing::info!("device revoked");
539 if self.device.stop().await {
540 drop(self.device.remove());
541 self.device.task_mut().0.close(offer.offer.subchannel_index);
542 }
543 self.cleanup_device_resources(state).await;
544 drop(state.offer.take());
545 }
546
547 fn handle_save(&mut self) -> SavedStateBlob {
548 let saved_state = self.saved_state.take();
549 if let Some(saved_state) = saved_state {
550 let blob = SavedStateBlob::new(saved_state);
551 self.handle_restore(&blob);
552 blob
553 } else {
554 SavedStateBlob::new(NoSavedState)
555 }
556 }
557
558 fn handle_restore(&mut self, saved_state_blob: &SavedStateBlob) {
559 self.saved_state = match saved_state_blob.parse() {
560 Ok(saved_state) => Some(saved_state),
561 Err(err) => {
562 tracing::error!(
563 err = &err as &dyn std::error::Error,
564 "Protobuf conversion error saving state"
565 );
566 None
567 }
568 };
569 }
570
571 pub async fn process_messages(
574 &mut self,
575 state: &mut SimpleVmbusClientDeviceTaskState,
576 check: &task_control::StopCheck<'_>,
577 ) -> Result<(), Cancelled> {
578 loop {
579 check.check()?;
580 #[expect(clippy::large_enum_variant)]
581 enum Event {
582 Request(InterceptChannelRequest),
583 Revoke,
584 }
585 let r = if let Some(offer) = &mut state.offer {
586 (
587 (&mut state.recv_relay).map(Event::Request),
588 futures::stream::once(&mut offer.revoke_recv).map(|_| Event::Revoke),
589 )
590 .merge()
591 .next()
592 .await
593 } else {
594 let mut recv_relay = pin!(&mut state.recv_relay);
595 recv_relay.next().await.map(Event::Request)
596 };
597 let Some(r) = r else {
598 break;
599 };
600 match r {
601 Event::Revoke => {
602 self.handle_revoke(state).await;
603 }
604 Event::Request(InterceptChannelRequest::Offer(offer)) => {
605 if !self.device.is_running() {
608 if let Err(err) = self.handle_offer(offer, state).await {
609 tracing::error!(
610 error = err.as_ref() as &dyn std::error::Error,
611 "failed offer handling"
612 );
613 }
614 }
615 }
616 Event::Request(InterceptChannelRequest::Start) => {
617 self.handle_start(state).await;
618 }
619 Event::Request(InterceptChannelRequest::Stop(rpc)) => {
620 rpc.handle(async |()| self.handle_stop(state).await).await;
621 }
622 Event::Request(InterceptChannelRequest::Save(rpc)) => {
623 rpc.handle_sync(|()| self.handle_save());
624 }
625 Event::Request(InterceptChannelRequest::Restore(saved_state)) => {
626 self.handle_restore(&saved_state);
627 }
628 }
629 }
630 Ok(())
631 }
632}
633
634struct MemoryBlockChannelSignal {
635 event: PolledNotify,
636 interrupt: Interrupt,
637}
638
639impl SignalVmbusChannel for MemoryBlockChannelSignal {
640 fn signal_remote(&self) {
641 self.interrupt.deliver();
642 }
643
644 fn poll_for_signal(
645 &self,
646 cx: &mut std::task::Context<'_>,
647 ) -> std::task::Poll<Result<(), ChannelClosed>> {
648 self.event.poll_wait(cx).map(Ok)
649 }
650}