Skip to main content

vmbus_relay_intercept_device/
lib.rs

1// Copyright (c) Microsoft Corporation.
2// Licensed under the MIT License.
3
4//! This module contains logic used to intercept from VTL2 a vmbus device
5//! provided for a VTL0 guest. This requires the vmbus relay to be active,
6//! which will filter the device out from the list provided to the VTL0 guest
7//! and send any vmbus notifications for that device to the
8//! SimpleVmbusClientDeviceWrapper instance.
9
10#![expect(missing_docs)]
11#![forbid(unsafe_code)]
12
13pub mod ring_buffer;
14
15use crate::ring_buffer::MemoryBlockRingBuffer;
16use anyhow::Context;
17use anyhow::Result;
18use futures::StreamExt;
19use futures_concurrency::stream::Merge;
20use guid::Guid;
21use inspect::InspectMut;
22use mesh::rpc::RpcSend;
23use pal_async::driver::SpawnDriver;
24use std::future::Future;
25use std::future::pending;
26use std::pin::pin;
27use std::sync::Arc;
28use task_control::AsyncRun;
29use task_control::Cancelled;
30use task_control::InspectTaskMut;
31use task_control::StopTask;
32use task_control::TaskControl;
33use tracing::Instrument;
34use user_driver::DmaClient;
35use user_driver::memory::MemoryBlock;
36use vmbus_channel::ChannelClosed;
37use vmbus_channel::RawAsyncChannel;
38use vmbus_channel::SignalVmbusChannel;
39use vmbus_channel::bus::GpadlRequest;
40use vmbus_channel::bus::OpenData;
41use vmbus_client::ChannelRequest;
42use vmbus_client::OfferInfo;
43use vmbus_client::OpenOutput;
44use vmbus_client::OpenRequest;
45use vmbus_core::protocol::GpadlId;
46use vmbus_core::protocol::UserDefinedData;
47use vmbus_relay::InterceptChannelRequest;
48use vmbus_ring::IncomingRing;
49use vmbus_ring::OutgoingRing;
50use vmbus_ring::PAGE_SIZE;
51use vmcore::interrupt::Interrupt;
52use vmcore::notify::Notify;
53use vmcore::notify::PolledNotify;
54use vmcore::save_restore::NoSavedState;
55use vmcore::save_restore::SavedStateBlob;
56use vmcore::save_restore::SavedStateRoot;
57use zerocopy::FromZeros;
58
59pub enum OfferResponse {
60    Ignore,
61    Open,
62}
63
64pub trait SimpleVmbusClientDevice {
65    /// The saved state type.
66    type SavedState: SavedStateRoot + Send + Sync;
67
68    /// The type used to run an open channel.
69    type Runner: 'static + Send + Sync;
70
71    /// Inspects a channel.
72    fn inspect(&mut self, req: inspect::Request<'_>, runner: Option<&mut Self::Runner>);
73
74    /// Returns the instance ID of the matching device.
75    fn instance_id(&self) -> Guid;
76
77    /// Respond to a new channel offer for a device matching instance_id().
78    fn offer(&self, offer: &vmbus_core::protocol::OfferChannel) -> OfferResponse;
79
80    /// Open successful for the channel number `channel_idx`.
81    ///
82    /// When the channel is closed, the runner will be dropped.
83    fn open(
84        &mut self,
85        channel_idx: u16,
86        channel: RawAsyncChannel<MemoryBlockRingBuffer>,
87    ) -> Result<Self::Runner>;
88
89    /// Closes the channel number `channel_idx` after the runner has been
90    /// dropped.
91    fn close(&mut self, channel_idx: u16);
92
93    /// Returns a trait used to save/restore the device.
94    fn supports_save_restore(
95        &mut self,
96    ) -> Option<
97        &mut dyn SaveRestoreSimpleVmbusClientDevice<
98            SavedState = Self::SavedState,
99            Runner = Self::Runner,
100        >,
101    >;
102}
103
104pub trait SimpleVmbusClientDeviceAsync: SimpleVmbusClientDevice + 'static + Send + Sync {
105    /// Runs an open channel until `stop` is signaled.
106    ///
107    /// Check for stop at cancel-safe work boundaries even when channel work
108    /// remains continuously ready.
109    fn run(
110        &mut self,
111        stop: &mut StopTask<'_>,
112        runner: &mut Self::Runner,
113    ) -> impl Send + Future<Output = Result<(), Cancelled>>;
114}
115
116/// Trait implemented by simple vmbus client devices that support save/restore.
117///
118/// If you implement this, make sure to return `Some(self)` from
119/// [`SimpleVmbusClientDevice::supports_save_restore`].
120pub trait SaveRestoreSimpleVmbusClientDevice: SimpleVmbusClientDevice {
121    /// Saves the channel.
122    ///
123    /// Will only be called if the channel is open.
124    fn save_open(&mut self, runner: &Self::Runner) -> Self::SavedState;
125
126    /// Restores the channel.
127    ///
128    /// Will only be called if the channel was saved open.
129    fn restore_open(
130        &mut self,
131        state: Self::SavedState,
132        channel: RawAsyncChannel<MemoryBlockRingBuffer>,
133    ) -> Result<Self::Runner>;
134}
135
136#[derive(InspectMut)]
137pub struct SimpleVmbusClientDeviceWrapper<T: SimpleVmbusClientDeviceAsync> {
138    instance_id: Guid,
139    #[inspect(skip)]
140    spawner: Arc<dyn SpawnDriver>,
141    #[inspect(mut)]
142    vmbus_listener: TaskControl<SimpleVmbusClientDeviceTask<T>, SimpleVmbusClientDeviceTaskState>,
143}
144
145impl<T: SimpleVmbusClientDeviceAsync> SimpleVmbusClientDeviceWrapper<T> {
146    /// Create a new instance.
147    pub fn new(
148        driver: impl SpawnDriver + Clone,
149        dma_alloc: Arc<dyn DmaClient>,
150        device: T,
151    ) -> Result<Self> {
152        let spawner = Arc::new(driver.clone());
153        Ok(Self {
154            instance_id: device.instance_id(),
155            vmbus_listener: TaskControl::new(SimpleVmbusClientDeviceTask::new(
156                device,
157                spawner.clone(),
158                dma_alloc,
159            )),
160            spawner,
161        })
162    }
163
164    pub fn instance_id(&self) -> Guid {
165        self.instance_id
166    }
167
168    pub fn detach(
169        mut self,
170        driver: impl SpawnDriver,
171        recv_relay: mesh::Receiver<InterceptChannelRequest>,
172    ) -> Result<()> {
173        let (send_disconnected, recv_disconnected) = mesh::oneshot();
174        self.vmbus_listener.insert(
175            &self.spawner,
176            format!("{}", self.instance_id),
177            SimpleVmbusClientDeviceTaskState {
178                offer: None,
179                recv_relay,
180                send_disconnected: Some(send_disconnected),
181                vtl_pages: None,
182            },
183        );
184        driver
185            .spawn(
186                format!("vmbus_relay_device {}", self.instance_id),
187                async move {
188                    self.vmbus_listener.start();
189                    let _ = recv_disconnected.await;
190                    assert!(!self.vmbus_listener.stop().await);
191                    if self.vmbus_listener.state().unwrap().vtl_pages.is_some() {
192                        // The VTL pages were not freed. This can occur if an
193                        // error is hit that drops the vmbus parent tasks. Just
194                        // pend here and let the outer error cause the VM to
195                        // exit.
196                        pending::<()>().await;
197                    }
198                },
199            )
200            .detach();
201        Ok(())
202    }
203}
204
205struct RelayDeviceTask<T>(T);
206
207impl<T: SimpleVmbusClientDeviceAsync> AsyncRun<T::Runner> for RelayDeviceTask<T> {
208    async fn run(
209        &mut self,
210        stop: &mut StopTask<'_>,
211        runner: &mut T::Runner,
212    ) -> Result<(), Cancelled> {
213        stop.check()?;
214        self.0.run(stop, runner).await
215    }
216}
217
218impl<T: SimpleVmbusClientDeviceAsync> InspectTaskMut<T::Runner> for RelayDeviceTask<T> {
219    fn inspect_mut(&mut self, req: inspect::Request<'_>, runner: Option<&mut T::Runner>) {
220        self.0.inspect(req, runner)
221    }
222}
223
224#[derive(InspectMut)]
225struct SimpleVmbusClientDeviceTaskState {
226    offer: Option<OfferInfo>,
227    #[inspect(skip)]
228    recv_relay: mesh::Receiver<InterceptChannelRequest>,
229    #[inspect(skip)]
230    send_disconnected: Option<mesh::OneshotSender<()>>,
231    #[inspect(hex, with = "|x| x.as_ref().map(|x| inspect::iter_by_index(x.pfns()))")]
232    vtl_pages: Option<MemoryBlock>,
233}
234
235struct SimpleVmbusClientDeviceTask<T: SimpleVmbusClientDeviceAsync> {
236    device: TaskControl<RelayDeviceTask<T>, T::Runner>,
237    saved_state: Option<T::SavedState>,
238    spawner: Arc<dyn SpawnDriver>,
239    dma_alloc: Arc<dyn DmaClient>,
240}
241
242impl<T: SimpleVmbusClientDeviceAsync> AsyncRun<SimpleVmbusClientDeviceTaskState>
243    for SimpleVmbusClientDeviceTask<T>
244{
245    async fn run(
246        &mut self,
247        stop: &mut StopTask<'_>,
248        state: &mut SimpleVmbusClientDeviceTaskState,
249    ) -> Result<(), Cancelled> {
250        let check = stop.check_handle()?;
251        stop.until_stopped(self.process_messages(state, &check))
252            .await??;
253        state
254            .send_disconnected
255            .take()
256            .expect("task should not be restarted")
257            .send(());
258        Ok(())
259    }
260}
261
262impl<T: SimpleVmbusClientDeviceAsync> InspectTaskMut<SimpleVmbusClientDeviceTaskState>
263    for SimpleVmbusClientDeviceTask<T>
264{
265    fn inspect_mut(
266        &mut self,
267        req: inspect::Request<'_>,
268        state: Option<&mut SimpleVmbusClientDeviceTaskState>,
269    ) {
270        req.respond()
271            .merge(state)
272            .field_mut("device", &mut self.device)
273            .field("dma_alloc", &self.dma_alloc);
274    }
275}
276
277impl<T: SimpleVmbusClientDeviceAsync> SimpleVmbusClientDeviceTask<T> {
278    pub fn new(device: T, spawner: Arc<dyn SpawnDriver>, dma_alloc: Arc<dyn DmaClient>) -> Self {
279        Self {
280            device: TaskControl::new(RelayDeviceTask(device)),
281            saved_state: None,
282            spawner,
283            dma_alloc,
284        }
285    }
286
287    fn insert_runner(&mut self, state: &SimpleVmbusClientDeviceTaskState, runner: T::Runner) {
288        let offer = state.offer.as_ref().unwrap().offer;
289        self.device.insert(
290            &self.spawner,
291            format!("{}-{}", offer.interface_id, offer.instance_id),
292            runner,
293        );
294    }
295
296    /// Configures channel.
297    async fn handle_offer(
298        &mut self,
299        offer: OfferInfo,
300        state: &mut SimpleVmbusClientDeviceTaskState,
301    ) -> Result<()> {
302        tracing::info!(?offer, "matching channel offered");
303
304        if offer.offer.is_dedicated != 1 {
305            tracing::warn!(offer = ?offer.offer, "All offers should be dedicated with Win8+ host")
306        }
307
308        if matches!(
309            self.device.task_mut().0.offer(&offer.offer),
310            OfferResponse::Ignore
311        ) {
312            return Ok(());
313        }
314
315        let interrupt_event = pal_event::Event::new();
316        let (memory, ring_gpadl_id) = self
317            .reserve_memory(state, &offer.request_send, 4)
318            .await
319            .context("reserve memory")?;
320        let guest_to_host_interrupt = offer.guest_to_host_interrupt.clone();
321        state.offer = Some(offer);
322        let offer = state.offer.as_ref().unwrap();
323        self.open_channel(&offer.request_send, ring_gpadl_id, &interrupt_event)
324            .await
325            .context("open channel")?;
326        let channel = self
327            .create_vmbus_channel(&memory, &interrupt_event, guest_to_host_interrupt)
328            .context("create vmbus queue")?;
329
330        let save_restore = self.device.task_mut().0.supports_save_restore();
331        let saved_state = self.saved_state.take();
332        let device_runner = if let Some(save_restore) = save_restore
333            && let Some(saved_state) = saved_state
334        {
335            save_restore
336                .restore_open(saved_state, channel)
337                .context("device restore_open callback")?
338        } else {
339            self.device
340                .task_mut()
341                .0
342                .open(offer.offer.subchannel_index, channel)
343                .context("device open callback")?
344        };
345        self.insert_runner(state, device_runner);
346        self.device.start();
347        Ok(())
348    }
349
350    /// Start channel after it has been stopped.
351    async fn handle_start(&mut self, state: &mut SimpleVmbusClientDeviceTaskState) {
352        if self.device.is_running() {
353            return;
354        }
355
356        let offer = state.offer.take();
357        if offer.is_none() {
358            return;
359        }
360
361        // If there is a previous valid offer, open the channel again.
362        if let Err(err) = self.handle_offer(offer.unwrap(), state).await {
363            tracing::error!(
364                err = err.as_ref() as &dyn std::error::Error,
365                "Failed to reconnect vmbus channel"
366            );
367        }
368    }
369
370    async fn cleanup_device_resources(&mut self, state: &mut SimpleVmbusClientDeviceTaskState) {
371        let Some(offer) = state.offer.as_mut() else {
372            return;
373        };
374
375        if let Some(vtl_pages) = &state.vtl_pages {
376            match offer
377                .request_send
378                .call(
379                    ChannelRequest::TeardownGpadl,
380                    GpadlId(vtl_pages.pfns()[1] as u32),
381                )
382                .await
383            {
384                Ok(()) => {
385                    state.vtl_pages = None;
386                }
387                Err(err) => {
388                    // If the ring buffer pages are still in use by the host, which
389                    // has to be assumed, the memory pages cannot be used again as
390                    // they have been marked as visible to VTL0.
391                    tracing::error!(
392                        error = &err as &dyn std::error::Error,
393                        "Failed to teardown gpadl -- leaking memory."
394                    );
395                }
396            }
397        }
398    }
399
400    /// Stop channel
401    async fn handle_stop(&mut self, state: &mut SimpleVmbusClientDeviceTaskState) {
402        if !self.device.stop().await {
403            return;
404        }
405
406        // Close the channel on every stop. Overlay devices cannot be saved /
407        // restored because the physical pages used for the ring buffer, et al.
408        // would need to be reserved at boot, otherwise the host may end up
409        // scribbling on random memory as it continues updating a ring buffer it
410        // assumes it has ownership of.
411        //
412        // TODO: We could support save restore, if we had a pool of memory that
413        // supports that. This should be possible once the page_pool_alloc is
414        // available everywhere.
415        {
416            let offer = state.offer.as_ref().expect("device opened");
417            offer
418                .request_send
419                .call(ChannelRequest::Close, ())
420                .await
421                .ok();
422        }
423        // N.B. This will wait for a TeardownGpadl response which can be used
424        // as a signal that the channel is closed and the ring buffers are no
425        // longer in use.
426        self.cleanup_device_resources(state).await;
427        let runner = self.device.remove();
428        let device = self.device.task_mut();
429        if let Some(save_restore) = device.0.supports_save_restore() {
430            self.saved_state = Some(save_restore.save_open(&runner));
431        }
432        drop(runner);
433        let offer = state.offer.as_ref().expect("device opened");
434        device.0.close(offer.offer.subchannel_index);
435    }
436
437    /// Allocates memory to be shared with the host and registers it with a
438    /// GPADL ID.
439    async fn reserve_memory(
440        &mut self,
441        state: &mut SimpleVmbusClientDeviceTaskState,
442        request_send: &mesh::Sender<ChannelRequest>,
443        page_count: usize,
444    ) -> Result<(MemoryBlock, GpadlId)> {
445        // Incoming and outgoing rings require a minimum of two pages apiece:
446        // one for the control bytes and at least one for the ring.
447        assert!(page_count >= 4);
448
449        let mem = self
450            .dma_alloc
451            .allocate_dma_buffer(page_count * PAGE_SIZE)
452            .context("allocating memory for vmbus rings")?;
453        state.vtl_pages = Some(mem.clone());
454        let buf: Vec<_> = [mem.len() as u64]
455            .iter()
456            .chain(mem.pfns())
457            .copied()
458            .collect();
459
460        let gpadl_id = GpadlId(state.vtl_pages.as_ref().unwrap().pfns()[1] as u32);
461        request_send
462            .call_failable(
463                ChannelRequest::Gpadl,
464                GpadlRequest {
465                    id: gpadl_id,
466                    count: 1,
467                    buf,
468                },
469            )
470            .await
471            .context("registering gpadl")?;
472        Ok((mem, gpadl_id))
473    }
474
475    /// Open the channel offered by the host.
476    async fn open_channel(
477        &self,
478        request_send: &mesh::Sender<ChannelRequest>,
479        ring_gpadl_id: GpadlId,
480        event: &pal_event::Event,
481    ) -> Result<OpenOutput> {
482        let open_request = OpenRequest {
483            open_data: OpenData {
484                target_vp: Some(0),
485                ring_offset: 2,
486                ring_gpadl_id,
487                event_flag: !0,
488                connection_id: !0,
489                user_data: UserDefinedData::new_zeroed(),
490            },
491            incoming_event: Some(event.clone()),
492            use_vtl2_connection_id: true,
493        };
494
495        request_send
496            .call_failable(ChannelRequest::Open, open_request)
497            .instrument(tracing::info_span!(
498                "opening vmbus channel for intercepted device"
499            ))
500            .await
501            .context("open vmbus channel")
502    }
503
504    /// Create a raw vmbus channel.
505    fn create_vmbus_channel(
506        &self,
507        mem: &MemoryBlock,
508        host_to_guest_event: &pal_event::Event,
509        guest_to_host_interrupt: Interrupt,
510    ) -> Result<RawAsyncChannel<MemoryBlockRingBuffer>> {
511        let (out_ring_mem, in_ring_mem) = (
512            mem.subblock(0, 2 * PAGE_SIZE),
513            mem.subblock(2 * PAGE_SIZE, 2 * PAGE_SIZE),
514        );
515        let (in_ring, out_ring) = (
516            IncomingRing::new(in_ring_mem.into()).unwrap(),
517            OutgoingRing::new(out_ring_mem.into()).unwrap(),
518        );
519
520        let signal = MemoryBlockChannelSignal {
521            event: Notify::from_event(host_to_guest_event.clone())
522                .pollable(self.spawner.as_ref())
523                .unwrap(),
524            interrupt: guest_to_host_interrupt,
525        };
526        Ok(RawAsyncChannel {
527            in_ring,
528            out_ring,
529            signal: Box::new(signal),
530        })
531    }
532
533    /// Responds to the channel being revoked by the host.
534    async fn handle_revoke(&mut self, state: &mut SimpleVmbusClientDeviceTaskState) {
535        let Some(offer) = state.offer.as_ref() else {
536            return;
537        };
538        tracing::info!("device revoked");
539        if self.device.stop().await {
540            drop(self.device.remove());
541            self.device.task_mut().0.close(offer.offer.subchannel_index);
542        }
543        self.cleanup_device_resources(state).await;
544        drop(state.offer.take());
545    }
546
547    fn handle_save(&mut self) -> SavedStateBlob {
548        let saved_state = self.saved_state.take();
549        if let Some(saved_state) = saved_state {
550            let blob = SavedStateBlob::new(saved_state);
551            self.handle_restore(&blob);
552            blob
553        } else {
554            SavedStateBlob::new(NoSavedState)
555        }
556    }
557
558    fn handle_restore(&mut self, saved_state_blob: &SavedStateBlob) {
559        self.saved_state = match saved_state_blob.parse() {
560            Ok(saved_state) => Some(saved_state),
561            Err(err) => {
562                tracing::error!(
563                    err = &err as &dyn std::error::Error,
564                    "Protobuf conversion error saving state"
565                );
566                None
567            }
568        };
569    }
570
571    /// Handle vmbus messages from the host and control messages from the
572    /// device wrapper.
573    pub async fn process_messages(
574        &mut self,
575        state: &mut SimpleVmbusClientDeviceTaskState,
576        check: &task_control::StopCheck<'_>,
577    ) -> Result<(), Cancelled> {
578        loop {
579            check.check()?;
580            #[expect(clippy::large_enum_variant)]
581            enum Event {
582                Request(InterceptChannelRequest),
583                Revoke,
584            }
585            let r = if let Some(offer) = &mut state.offer {
586                (
587                    (&mut state.recv_relay).map(Event::Request),
588                    futures::stream::once(&mut offer.revoke_recv).map(|_| Event::Revoke),
589                )
590                    .merge()
591                    .next()
592                    .await
593            } else {
594                let mut recv_relay = pin!(&mut state.recv_relay);
595                recv_relay.next().await.map(Event::Request)
596            };
597            let Some(r) = r else {
598                break;
599            };
600            match r {
601                Event::Revoke => {
602                    self.handle_revoke(state).await;
603                }
604                Event::Request(InterceptChannelRequest::Offer(offer)) => {
605                    // Any extraneous offer notifications (e.g. from a request offers
606                    // query) are ignored.
607                    if !self.device.is_running() {
608                        if let Err(err) = self.handle_offer(offer, state).await {
609                            tracing::error!(
610                                error = err.as_ref() as &dyn std::error::Error,
611                                "failed offer handling"
612                            );
613                        }
614                    }
615                }
616                Event::Request(InterceptChannelRequest::Start) => {
617                    self.handle_start(state).await;
618                }
619                Event::Request(InterceptChannelRequest::Stop(rpc)) => {
620                    rpc.handle(async |()| self.handle_stop(state).await).await;
621                }
622                Event::Request(InterceptChannelRequest::Save(rpc)) => {
623                    rpc.handle_sync(|()| self.handle_save());
624                }
625                Event::Request(InterceptChannelRequest::Restore(saved_state)) => {
626                    self.handle_restore(&saved_state);
627                }
628            }
629        }
630        Ok(())
631    }
632}
633
634struct MemoryBlockChannelSignal {
635    event: PolledNotify,
636    interrupt: Interrupt,
637}
638
639impl SignalVmbusChannel for MemoryBlockChannelSignal {
640    fn signal_remote(&self) {
641        self.interrupt.deliver();
642    }
643
644    fn poll_for_signal(
645        &self,
646        cx: &mut std::task::Context<'_>,
647    ) -> std::task::Poll<Result<(), ChannelClosed>> {
648        self.event.poll_wait(cx).map(Ok)
649    }
650}