virtio/transport/
mmio.rs

1// Copyright (c) Microsoft Corporation.
2// Licensed under the MIT License.
3
4use super::StalledIo;
5use super::core::TransportOps;
6use super::core::VirtioTransportCore;
7use super::task::ConfigReadCompletion;
8use super::task::defer_config_read;
9use super::task::defer_config_write;
10use crate::DynVirtioDevice;
11use crate::MAX_QUEUE_SIZE;
12use crate::spec::VIRTIO_MMIO_INTERRUPT_STATUS_CONFIG_CHANGE;
13use crate::spec::VIRTIO_MMIO_INTERRUPT_STATUS_USED_BUFFER;
14use crate::spec::mmio::VirtioMmioRegister;
15use chipset_device::ChipsetDevice;
16use chipset_device::io::IoResult;
17use chipset_device::io::deferred::defer_read;
18use chipset_device::io::deferred::defer_write;
19use chipset_device::mmio::MmioIntercept;
20use chipset_device::poll_device::PollDevice;
21use device_emulators::ReadWriteRequestType;
22use device_emulators::read_as_u32_chunks;
23use device_emulators::write_as_u32_chunks;
24use guestmem::DoorbellRegistration;
25use guestmem::GuestMemory;
26use inspect::Inspect;
27use inspect::InspectMut;
28use pal_async::task::Spawn;
29use parking_lot::Mutex;
30use std::fmt;
31use std::ops::RangeInclusive;
32use std::sync::Arc;
33use vmcore::device_state::ChangeDeviceState;
34use vmcore::interrupt::Interrupt;
35use vmcore::line_interrupt::LineInterrupt;
36
37/// MMIO-specific transport state.
38#[derive(Inspect)]
39struct MmioTransport {
40    #[inspect(skip)]
41    fixed_mmio_region: (&'static str, RangeInclusive<u64>),
42    #[inspect(hex)]
43    device_id: u32,
44    #[inspect(hex)]
45    vendor_id: u32,
46    interrupt_state: Arc<Mutex<InterruptState>>,
47}
48
49#[derive(Inspect)]
50struct InterruptState {
51    interrupt: LineInterrupt,
52    status: u32,
53}
54
55impl InterruptState {
56    fn update(&mut self, is_set: bool, bits: u32) {
57        if is_set {
58            self.status |= bits;
59        } else {
60            self.status &= !bits;
61        }
62        self.interrupt.set_level(self.status != 0);
63    }
64}
65
66impl TransportOps for MmioTransport {
67    fn create_queue_interrupt(&mut self, _idx: usize, _msix_vector: u16) -> Interrupt {
68        let interrupt_state = self.interrupt_state.clone();
69        Interrupt::from_fn(move || {
70            interrupt_state
71                .lock()
72                .update(true, VIRTIO_MMIO_INTERRUPT_STATUS_USED_BUFFER);
73        })
74    }
75
76    fn signal_config_change(&mut self) {
77        self.interrupt_state
78            .lock()
79            .update(true, VIRTIO_MMIO_INTERRUPT_STATUS_CONFIG_CHANGE);
80    }
81
82    fn reset_interrupts(&mut self) {
83        self.interrupt_state.lock().update(false, !0);
84    }
85
86    fn doorbell_region(&mut self) -> Option<(u64, u32)> {
87        let base = (*self.fixed_mmio_region.1.start() & !0xfff)
88            + VirtioMmioRegister::QUEUE_NOTIFY.0 as u64;
89        Some((base, 4))
90    }
91}
92
93/// Run a virtio device over MMIO
94#[derive(InspectMut)]
95pub struct VirtioMmioDevice {
96    #[inspect(flatten)]
97    core: VirtioTransportCore,
98    #[inspect(flatten)]
99    mmio: MmioTransport,
100}
101
102impl fmt::Debug for VirtioMmioDevice {
103    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
104        f.debug_struct("VirtioMmioDevice").finish()
105    }
106}
107
108impl VirtioMmioDevice {
109    pub fn new(
110        device: Box<dyn DynVirtioDevice>,
111        driver: &impl Spawn,
112        guest_memory: GuestMemory,
113        interrupt: LineInterrupt,
114        doorbell_registration: Option<Arc<dyn DoorbellRegistration>>,
115        mmio_gpa: u64,
116        mmio_len: u64,
117    ) -> std::io::Result<Self> {
118        let traits = device.traits();
119        let interrupt_state = Arc::new(Mutex::new(InterruptState {
120            interrupt,
121            status: 0,
122        }));
123
124        let core = VirtioTransportCore::new(device, driver, guest_memory, doorbell_registration)?;
125
126        Ok(Self {
127            core,
128            mmio: MmioTransport {
129                fixed_mmio_region: ("virtio-chipset", mmio_gpa..=(mmio_gpa + mmio_len - 1)),
130                device_id: traits.device_id.0 as u32,
131                vendor_id: 0x1af4,
132                interrupt_state,
133            },
134        })
135    }
136
137    /// Synchronous transport register read for tests.
138    #[cfg(test)]
139    pub(crate) fn read_u32(&mut self, address: u64) -> u32 {
140        self.read_u32_local((address & 0xfff) as u16)
141    }
142
143    /// Synchronous transport register write for tests.
144    #[cfg(test)]
145    pub(crate) fn write_u32(&mut self, address: u64, val: u32) {
146        self.write_u32_local((address & 0xfff) as u16, val);
147    }
148
149    /// Read a transport register as a u32.
150    fn read_u32_local(&mut self, offset: u16) -> u32 {
151        assert!(offset & 3 == 0);
152        let queue_select = self.core.queue_select as usize;
153        match VirtioMmioRegister(offset) {
154            VirtioMmioRegister::MAGIC_VALUE => u32::from_le_bytes(*b"virt"),
155            VirtioMmioRegister::VERSION => 2,
156            VirtioMmioRegister::DEVICE_ID => self.mmio.device_id,
157            VirtioMmioRegister::VENDOR_ID => self.mmio.vendor_id,
158            VirtioMmioRegister::DEVICE_FEATURES => self
159                .core
160                .device_feature
161                .bank(self.core.device_feature_select as usize),
162            VirtioMmioRegister::DEVICE_FEATURES_SEL => self.core.device_feature_select,
163            VirtioMmioRegister::DRIVER_FEATURES => self
164                .core
165                .driver_feature
166                .bank(self.core.driver_feature_select as usize),
167            VirtioMmioRegister::DRIVER_FEATURES_SEL => self.core.driver_feature_select,
168            VirtioMmioRegister::QUEUE_SEL => self.core.queue_select,
169            VirtioMmioRegister::QUEUE_NUM_MAX => self
170                .core
171                .queues
172                .get(queue_select)
173                .map_or(0, |qd| qd.initial_size.into()),
174            VirtioMmioRegister::QUEUE_NUM => self
175                .core
176                .queues
177                .get(queue_select)
178                .map_or(0, |qd| qd.params.size as u32),
179            VirtioMmioRegister::QUEUE_READY => {
180                self.core
181                    .queues
182                    .get(queue_select)
183                    .is_some_and(|qd| qd.params.enable) as u32
184            }
185            VirtioMmioRegister::QUEUE_NOTIFY => 0,
186            VirtioMmioRegister::INTERRUPT_STATUS => self.mmio.interrupt_state.lock().status,
187            VirtioMmioRegister::INTERRUPT_ACK => 0,
188            VirtioMmioRegister::STATUS => self.core.device_status.as_u32(),
189            VirtioMmioRegister::QUEUE_DESC_LOW => self
190                .core
191                .queues
192                .get(queue_select)
193                .map_or(0, |qd| qd.params.desc_addr as u32),
194            VirtioMmioRegister::QUEUE_DESC_HIGH => self
195                .core
196                .queues
197                .get(queue_select)
198                .map_or(0, |qd| (qd.params.desc_addr >> 32) as u32),
199            VirtioMmioRegister::QUEUE_AVAIL_LOW => self
200                .core
201                .queues
202                .get(queue_select)
203                .map_or(0, |qd| qd.params.avail_addr as u32),
204            VirtioMmioRegister::QUEUE_AVAIL_HIGH => self
205                .core
206                .queues
207                .get(queue_select)
208                .map_or(0, |qd| (qd.params.avail_addr >> 32) as u32),
209            VirtioMmioRegister::QUEUE_USED_LOW => self
210                .core
211                .queues
212                .get(queue_select)
213                .map_or(0, |qd| qd.params.used_addr as u32),
214            VirtioMmioRegister::QUEUE_USED_HIGH => self
215                .core
216                .queues
217                .get(queue_select)
218                .map_or(0, |qd| (qd.params.used_addr >> 32) as u32),
219            VirtioMmioRegister::CONFIG_GENERATION => self.core.config_generation,
220            _ => 0xffffffff,
221        }
222    }
223
224    /// Write a transport register as a u32.
225    fn write_u32_local(&mut self, offset: u16, val: u32) {
226        assert!(offset & 3 == 0);
227        let queue_select = self.core.queue_select as usize;
228        let queues_locked = self.core.device_status.driver_ok();
229        let features_locked = queues_locked || self.core.device_status.features_ok();
230        match VirtioMmioRegister(offset) {
231            VirtioMmioRegister::DEVICE_FEATURES_SEL => self.core.device_feature_select = val,
232            VirtioMmioRegister::DRIVER_FEATURES => {
233                let bank = self.core.driver_feature_select as usize;
234                if !features_locked && bank < 2 {
235                    self.core
236                        .driver_feature
237                        .set_bank(bank, val & self.core.device_feature.bank(bank));
238                }
239            }
240            VirtioMmioRegister::DRIVER_FEATURES_SEL => self.core.driver_feature_select = val,
241            VirtioMmioRegister::QUEUE_SEL => self.core.queue_select = val,
242            VirtioMmioRegister::QUEUE_NUM => {
243                if !queues_locked && queue_select < self.core.queues.len() {
244                    let val = val as u16;
245                    let queue = &mut self.core.queues[queue_select].params;
246                    if val > MAX_QUEUE_SIZE {
247                        queue.size = MAX_QUEUE_SIZE;
248                    } else {
249                        queue.size = val;
250                    }
251                }
252            }
253            VirtioMmioRegister::QUEUE_READY => {
254                if !queues_locked && queue_select < self.core.queues.len() {
255                    self.core.queues[queue_select].params.enable = val != 0;
256                }
257            }
258            VirtioMmioRegister::QUEUE_NOTIFY => {
259                self.core.notify_queue(val);
260            }
261            VirtioMmioRegister::INTERRUPT_ACK => {
262                self.mmio.interrupt_state.lock().update(false, val);
263            }
264            VirtioMmioRegister::STATUS => {
265                self.core.write_device_status(&mut self.mmio, val as u8);
266            }
267            VirtioMmioRegister::QUEUE_DESC_LOW => {
268                if !queues_locked && queue_select < self.core.queues.len() {
269                    let queue = &mut self.core.queues[queue_select].params;
270                    queue.desc_addr = queue.desc_addr & 0xffffffff00000000 | val as u64;
271                }
272            }
273            VirtioMmioRegister::QUEUE_DESC_HIGH => {
274                if !queues_locked && queue_select < self.core.queues.len() {
275                    let queue = &mut self.core.queues[queue_select].params;
276                    queue.desc_addr = (val as u64) << 32 | queue.desc_addr & 0xffffffff;
277                }
278            }
279            VirtioMmioRegister::QUEUE_AVAIL_LOW => {
280                if !queues_locked && queue_select < self.core.queues.len() {
281                    let queue = &mut self.core.queues[queue_select].params;
282                    queue.avail_addr = queue.avail_addr & 0xffffffff00000000 | val as u64;
283                }
284            }
285            VirtioMmioRegister::QUEUE_AVAIL_HIGH => {
286                if !queues_locked && queue_select < self.core.queues.len() {
287                    let queue = &mut self.core.queues[queue_select].params;
288                    queue.avail_addr = (val as u64) << 32 | queue.avail_addr & 0xffffffff;
289                }
290            }
291            VirtioMmioRegister::QUEUE_USED_LOW => {
292                if !queues_locked && queue_select < self.core.queues.len() {
293                    let queue = &mut self.core.queues[queue_select].params;
294                    queue.used_addr = queue.used_addr & 0xffffffff00000000 | val as u64;
295                }
296            }
297            VirtioMmioRegister::QUEUE_USED_HIGH => {
298                if !queues_locked && queue_select < self.core.queues.len() {
299                    let queue = &mut self.core.queues[queue_select].params;
300                    queue.used_addr = (val as u64) << 32 | queue.used_addr & 0xffffffff;
301                }
302            }
303            _ => (),
304        }
305    }
306
307    /// Read transport registers via sub-word chunk handling.
308    fn read_transport(&mut self, offset: u16, data: &mut [u8]) {
309        read_as_u32_chunks(offset, data, |offset| self.read_u32_local(offset));
310    }
311
312    /// Write transport registers via sub-word chunk handling.
313    fn write_transport(&mut self, offset: u16, data: &[u8]) {
314        write_as_u32_chunks(offset, data, |offset, request_type| match request_type {
315            ReadWriteRequestType::Write(value) => {
316                self.write_u32_local(offset, value);
317                None
318            }
319            ReadWriteRequestType::Read => Some(self.read_u32_local(offset)),
320        });
321    }
322
323    /// Replay MMIO accesses that were stalled while the transport was busy.
324    fn replay_stalled_io(&mut self) {
325        let stalled = std::mem::take(&mut self.core.stalled_io);
326        let mut iter = stalled.into_iter();
327        for io in &mut iter {
328            match io {
329                StalledIo::Read {
330                    address,
331                    len,
332                    deferred,
333                } => {
334                    let mut buf = vec![0u8; len];
335                    self.read_transport((address & 0xfff) as u16, &mut buf);
336                    deferred.complete(&buf);
337                }
338                StalledIo::Write {
339                    address,
340                    data,
341                    len,
342                    deferred,
343                } => {
344                    self.write_transport((address & 0xfff) as u16, &data[..len]);
345                    if self.core.state.is_busy() {
346                        self.core.pending_status_deferred = Some(deferred);
347                        break;
348                    }
349                    deferred.complete();
350                }
351            }
352        }
353        self.core.stalled_io = iter.collect();
354    }
355}
356
357impl ChangeDeviceState for VirtioMmioDevice {
358    fn start(&mut self) {
359        self.core.start(&mut self.mmio);
360    }
361
362    async fn stop(&mut self) {
363        self.core.stop(&mut self.mmio).await;
364    }
365
366    async fn reset(&mut self) {
367        self.core.reset(&mut self.mmio).await;
368    }
369}
370
371impl PollDevice for VirtioMmioDevice {
372    fn poll_device(&mut self, cx: &mut std::task::Context<'_>) {
373        self.core.poll_device(&mut self.mmio, cx);
374        if !self.core.stalled_io.is_empty() && !self.core.state.is_busy() {
375            self.replay_stalled_io();
376        }
377    }
378}
379
380impl ChipsetDevice for VirtioMmioDevice {
381    fn supports_mmio(&mut self) -> Option<&mut dyn MmioIntercept> {
382        Some(self)
383    }
384
385    fn supports_poll_device(&mut self) -> Option<&mut dyn PollDevice> {
386        Some(self)
387    }
388}
389
390mod saved_state {
391    mod state {
392        use crate::transport::saved_state::state::CommonQueueState;
393        use crate::transport::saved_state::state::CommonSavedState;
394        use mesh::payload::Protobuf;
395        use vmcore::save_restore::SavedStateRoot;
396
397        #[derive(Protobuf)]
398        #[mesh(package = "virtio.transport.mmio")]
399        pub struct SavedQueueState {
400            #[mesh(1)]
401            pub common: CommonQueueState,
402        }
403
404        #[derive(Protobuf, SavedStateRoot)]
405        #[mesh(package = "virtio.transport.mmio")]
406        pub struct SavedState {
407            #[mesh(1)]
408            pub common: CommonSavedState,
409            #[mesh(2)]
410            pub queues: Vec<SavedQueueState>,
411            #[mesh(3)]
412            pub interrupt_status: u32,
413        }
414    }
415
416    use super::*;
417    use vmcore::save_restore::SaveRestore;
418
419    impl SaveRestore for VirtioMmioDevice {
420        type SavedState = state::SavedState;
421
422        fn save(&mut self) -> Result<Self::SavedState, vmcore::save_restore::SaveError> {
423            Ok(state::SavedState {
424                common: self.core.save_common()?,
425                queues: (0..self.core.queues.len())
426                    .map(|i| state::SavedQueueState {
427                        common: self.core.save_queue_common(i),
428                    })
429                    .collect(),
430                interrupt_status: self.mmio.interrupt_state.lock().status,
431            })
432        }
433
434        fn restore(
435            &mut self,
436            state: Self::SavedState,
437        ) -> Result<(), vmcore::save_restore::RestoreError> {
438            let saved_queue_count = state.queues.len();
439            self.core.restore_common(
440                &mut self.mmio,
441                &state.common,
442                state.queues.into_iter().map(|sq| (sq.common, 0)),
443                saved_queue_count,
444            )?;
445
446            // Restore MMIO-specific interrupt state.
447            {
448                let mut is = self.mmio.interrupt_state.lock();
449                is.status = state.interrupt_status;
450                is.interrupt.set_level(is.status != 0);
451            }
452
453            Ok(())
454        }
455    }
456}
457
458impl MmioIntercept for VirtioMmioDevice {
459    fn mmio_read(&mut self, address: u64, data: &mut [u8]) -> IoResult {
460        let offset = (address & 0xfff) as u16;
461        if offset >= VirtioMmioRegister::CONFIG.0 {
462            return defer_config_read(
463                &self.core.device_sender,
464                offset - VirtioMmioRegister::CONFIG.0,
465                data.len() as u8,
466                ConfigReadCompletion::Exact,
467            );
468        }
469        if self.core.state.is_busy() {
470            let (deferred, token) = defer_read();
471            self.core.stalled_io.push(StalledIo::Read {
472                address,
473                len: data.len(),
474                deferred,
475            });
476            return IoResult::Defer(token);
477        }
478        self.read_transport(offset, data);
479        IoResult::Ok
480    }
481
482    fn mmio_write(&mut self, address: u64, data: &[u8]) -> IoResult {
483        let offset = (address & 0xfff) as u16;
484        if offset >= VirtioMmioRegister::CONFIG.0 {
485            return defer_config_write(
486                &self.core.device_sender,
487                offset - VirtioMmioRegister::CONFIG.0,
488                data,
489            );
490        }
491        if self.core.state.is_busy() {
492            let (deferred, token) = defer_write();
493            let mut buf = [0u8; 8];
494            buf[..data.len()].copy_from_slice(data);
495            self.core.stalled_io.push(StalledIo::Write {
496                address,
497                data: buf,
498                len: data.len(),
499                deferred,
500            });
501            return IoResult::Defer(token);
502        }
503        self.write_transport(offset, data);
504        if self.core.state.is_busy() {
505            let (deferred, token) = defer_write();
506            self.core.pending_status_deferred = Some(deferred);
507            return IoResult::Defer(token);
508        }
509        IoResult::Ok
510    }
511
512    fn get_static_regions(&mut self) -> &[(&str, RangeInclusive<u64>)] {
513        std::slice::from_ref(&self.mmio.fixed_mmio_region)
514    }
515}
virtio/transport/mmio.rs

virtio/transport/
mmio.rs
