virtio/transport/
mmio.rs

1// Copyright (c) Microsoft Corporation.
2// Licensed under the MIT License.
3
4use crate::QUEUE_MAX_SIZE;
5use crate::QueueResources;
6use crate::Resources;
7use crate::VirtioDevice;
8use crate::VirtioDoorbells;
9use crate::queue::QueueParams;
10use crate::spec::VIRTIO_MMIO_INTERRUPT_STATUS_CONFIG_CHANGE;
11use crate::spec::VIRTIO_MMIO_INTERRUPT_STATUS_USED_BUFFER;
12use crate::spec::VirtioDeviceFeatures;
13use crate::spec::VirtioDeviceStatus;
14use crate::spec::mmio::VirtioMmioRegister;
15use chipset_device::ChipsetDevice;
16use chipset_device::io::IoResult;
17use chipset_device::mmio::MmioIntercept;
18use chipset_device::poll_device::PollDevice;
19use device_emulators::ReadWriteRequestType;
20use device_emulators::read_as_u32_chunks;
21use device_emulators::write_as_u32_chunks;
22use guestmem::DoorbellRegistration;
23use inspect::Inspect;
24use inspect::InspectMut;
25use parking_lot::Mutex;
26use std::fmt;
27use std::ops::RangeInclusive;
28use std::sync::Arc;
29use vmcore::device_state::ChangeDeviceState;
30use vmcore::interrupt::Interrupt;
31use vmcore::line_interrupt::LineInterrupt;
32use vmcore::save_restore::NoSavedState;
33use vmcore::save_restore::RestoreError;
34use vmcore::save_restore::SaveError;
35use vmcore::save_restore::SaveRestore;
36
37/// Run a virtio device over MMIO
38#[derive(InspectMut)]
39pub struct VirtioMmioDevice {
40    #[inspect(skip)]
41    fixed_mmio_region: (&'static str, RangeInclusive<u64>),
42
43    #[inspect(mut)]
44    device: Box<dyn VirtioDevice>,
45    #[inspect(hex)]
46    device_id: u32,
47    #[inspect(hex)]
48    vendor_id: u32,
49    #[inspect(skip)]
50    device_feature: VirtioDeviceFeatures,
51    device_feature_select: u32,
52    #[inspect(skip)]
53    driver_feature: VirtioDeviceFeatures,
54    driver_feature_select: u32,
55    queue_select: u32,
56    #[inspect(skip)]
57    events: Vec<pal_event::Event>,
58    #[inspect(skip)]
59    queues: Vec<QueueParams>,
60    device_status: VirtioDeviceStatus,
61    disabling: bool,
62    #[inspect(skip)]
63    poll_waker: Option<std::task::Waker>,
64    config_generation: u32,
65    #[inspect(skip)]
66    doorbells: VirtioDoorbells,
67    interrupt_state: Arc<Mutex<InterruptState>>,
68}
69
70#[derive(Inspect)]
71struct InterruptState {
72    interrupt: LineInterrupt,
73    status: u32,
74}
75
76impl InterruptState {
77    fn update(&mut self, is_set: bool, bits: u32) {
78        if is_set {
79            self.status |= bits;
80        } else {
81            self.status &= !bits;
82        }
83        self.interrupt.set_level(self.status != 0);
84    }
85}
86
87impl fmt::Debug for VirtioMmioDevice {
88    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
89        // TODO: implement debug print
90        f.debug_struct("VirtioMmioDevice").finish()
91    }
92}
93
94impl VirtioMmioDevice {
95    pub fn new(
96        device: Box<dyn VirtioDevice>,
97        interrupt: LineInterrupt,
98        doorbell_registration: Option<Arc<dyn DoorbellRegistration>>,
99        mmio_gpa: u64,
100        mmio_len: u64,
101    ) -> Self {
102        let traits = device.traits();
103        let queues = (0..traits.max_queues)
104            .map(|_| QueueParams {
105                size: QUEUE_MAX_SIZE,
106                ..Default::default()
107            })
108            .collect();
109        let events = (0..traits.max_queues)
110            .map(|_| pal_event::Event::new())
111            .collect();
112        let interrupt_state = Arc::new(Mutex::new(InterruptState {
113            interrupt,
114            status: 0,
115        }));
116
117        let mut device_feature = traits.device_features.clone();
118        device_feature.set_bank(
119            0,
120            device_feature
121                .bank0()
122                .with_ring_event_idx(true)
123                .with_ring_indirect_desc(true)
124                .into_bits(),
125        );
126        device_feature.set_bank(1, device_feature.bank1().with_version_1(true).into_bits());
127        Self {
128            fixed_mmio_region: ("virtio-chipset", mmio_gpa..=(mmio_gpa + mmio_len - 1)),
129            device,
130            device_id: traits.device_id as u32,
131            vendor_id: 0x1af4,
132            device_feature,
133            device_feature_select: 0,
134            driver_feature: VirtioDeviceFeatures::new(),
135            driver_feature_select: 0,
136            queue_select: 0,
137            events,
138            queues,
139            device_status: VirtioDeviceStatus::new(),
140            disabling: false,
141            poll_waker: None,
142            config_generation: 0,
143            doorbells: VirtioDoorbells::new(doorbell_registration),
144            interrupt_state,
145        }
146    }
147
148    fn update_config_generation(&mut self) {
149        self.config_generation = self.config_generation.wrapping_add(1);
150        if self.device_status.driver_ok() {
151            self.interrupt_state
152                .lock()
153                .update(true, VIRTIO_MMIO_INTERRUPT_STATUS_CONFIG_CHANGE);
154        }
155    }
156}
157
158impl VirtioMmioDevice {
159    pub(crate) fn read_u32(&self, address: u64) -> u32 {
160        let offset = (address & 0xfff) as u16;
161        assert!(offset & 3 == 0);
162        match VirtioMmioRegister(offset) {
163            VirtioMmioRegister::MAGIC_VALUE => u32::from_le_bytes(*b"virt"),
164            VirtioMmioRegister::VERSION => 2,
165            VirtioMmioRegister::DEVICE_ID => self.device_id,
166            VirtioMmioRegister::VENDOR_ID => self.vendor_id,
167            VirtioMmioRegister::DEVICE_FEATURES => {
168                let feature_select = self.device_feature_select as usize;
169                self.device_feature.bank(feature_select)
170            }
171            VirtioMmioRegister::DEVICE_FEATURES_SEL => self.device_feature_select,
172            VirtioMmioRegister::DRIVER_FEATURES => {
173                let feature_select = self.driver_feature_select as usize;
174                self.driver_feature.bank(feature_select)
175            }
176            VirtioMmioRegister::DRIVER_FEATURES_SEL => self.driver_feature_select,
177            VirtioMmioRegister::QUEUE_SEL => self.queue_select,
178            // A value of zero indicates the queue is not available.
179            VirtioMmioRegister::QUEUE_NUM_MAX => {
180                let queue_select = self.queue_select as usize;
181                if queue_select < self.queues.len() {
182                    QUEUE_MAX_SIZE.into()
183                } else {
184                    0
185                }
186            }
187            VirtioMmioRegister::QUEUE_NUM => {
188                let queue_select = self.queue_select as usize;
189                if queue_select < self.queues.len() {
190                    self.queues[queue_select].size as u32
191                } else {
192                    0
193                }
194            }
195            VirtioMmioRegister::QUEUE_READY => {
196                let queue_select = self.queue_select as usize;
197                if queue_select < self.queues.len() {
198                    if self.queues[queue_select].enable {
199                        1
200                    } else {
201                        0
202                    }
203                } else {
204                    0
205                }
206            }
207            VirtioMmioRegister::QUEUE_NOTIFY => 0,
208            VirtioMmioRegister::INTERRUPT_STATUS => self.interrupt_state.lock().status,
209            VirtioMmioRegister::INTERRUPT_ACK => 0,
210            VirtioMmioRegister::STATUS => self.device_status.as_u32(),
211            VirtioMmioRegister::QUEUE_DESC_LOW => {
212                let queue_select = self.queue_select as usize;
213                if queue_select < self.queues.len() {
214                    self.queues[queue_select].desc_addr as u32
215                } else {
216                    0
217                }
218            }
219            VirtioMmioRegister::QUEUE_DESC_HIGH => {
220                let queue_select = self.queue_select as usize;
221                if queue_select < self.queues.len() {
222                    (self.queues[queue_select].desc_addr >> 32) as u32
223                } else {
224                    0
225                }
226            }
227            VirtioMmioRegister::QUEUE_AVAIL_LOW => {
228                let queue_select = self.queue_select as usize;
229                if queue_select < self.queues.len() {
230                    self.queues[queue_select].avail_addr as u32
231                } else {
232                    0
233                }
234            }
235            VirtioMmioRegister::QUEUE_AVAIL_HIGH => {
236                let queue_select = self.queue_select as usize;
237                if queue_select < self.queues.len() {
238                    (self.queues[queue_select].avail_addr >> 32) as u32
239                } else {
240                    0
241                }
242            }
243            VirtioMmioRegister::QUEUE_USED_LOW => {
244                let queue_select = self.queue_select as usize;
245                if queue_select < self.queues.len() {
246                    self.queues[queue_select].used_addr as u32
247                } else {
248                    0
249                }
250            }
251            VirtioMmioRegister::QUEUE_USED_HIGH => {
252                let queue_select = self.queue_select as usize;
253                if queue_select < self.queues.len() {
254                    (self.queues[queue_select].used_addr >> 32) as u32
255                } else {
256                    0
257                }
258            }
259            VirtioMmioRegister::CONFIG_GENERATION => self.config_generation,
260            VirtioMmioRegister(offset) if offset >= VirtioMmioRegister::CONFIG.0 => self
261                .device
262                .read_registers_u32(offset - VirtioMmioRegister::CONFIG.0),
263            _ => 0xffffffff,
264        }
265    }
266
267    pub(crate) fn write_u32(&mut self, address: u64, val: u32) {
268        let offset = (address & 0xfff) as u16;
269        assert!(offset & 3 == 0);
270        let queue_select = self.queue_select as usize;
271        let queues_locked = self.device_status.driver_ok();
272        let features_locked = queues_locked || self.device_status.features_ok();
273        match VirtioMmioRegister(offset) {
274            VirtioMmioRegister::DEVICE_FEATURES_SEL => self.device_feature_select = val,
275            VirtioMmioRegister::DRIVER_FEATURES => {
276                let bank = self.driver_feature_select as usize;
277                if features_locked || bank >= self.device_feature.len() {
278                    // Update is not persisted.
279                } else {
280                    self.driver_feature
281                        .set_bank(bank, val & self.device_feature.bank(bank));
282                }
283            }
284            VirtioMmioRegister::DRIVER_FEATURES_SEL => self.driver_feature_select = val,
285            VirtioMmioRegister::QUEUE_SEL => self.queue_select = val,
286            VirtioMmioRegister::QUEUE_NUM => {
287                if !queues_locked && queue_select < self.queues.len() {
288                    let val = val as u16;
289                    let queue = &mut self.queues[queue_select];
290                    if val > QUEUE_MAX_SIZE {
291                        queue.size = QUEUE_MAX_SIZE;
292                    } else {
293                        queue.size = val;
294                    }
295                }
296            }
297            VirtioMmioRegister::QUEUE_READY => {
298                if !queues_locked && queue_select < self.queues.len() {
299                    let queue = &mut self.queues[queue_select];
300                    queue.enable = val != 0;
301                }
302            }
303            VirtioMmioRegister::QUEUE_NOTIFY => {
304                if (val as usize) < self.events.len() {
305                    self.events[val as usize].signal();
306                }
307            }
308            VirtioMmioRegister::INTERRUPT_ACK => {
309                self.interrupt_state.lock().update(false, val);
310            }
311            VirtioMmioRegister::STATUS => {
312                if val == 0 {
313                    if self.disabling {
314                        return;
315                    }
316                    let started = self.device_status.driver_ok();
317                    self.config_generation = 0;
318                    if started {
319                        self.doorbells.clear();
320                        // Try the fast path: poll with a noop waker to see if
321                        // the device can disable synchronously.
322                        let waker = std::task::Waker::noop();
323                        let mut cx = std::task::Context::from_waker(waker);
324                        if self.device.poll_disable(&mut cx).is_pending() {
325                            self.disabling = true;
326                            // Wake the real poll waker so that poll_device will
327                            // re-poll with a real waker, replacing the noop one.
328                            if let Some(waker) = self.poll_waker.take() {
329                                waker.wake();
330                            }
331                            return;
332                        }
333                    }
334                    // Fast path: disable completed synchronously.
335                    self.device_status = VirtioDeviceStatus::new();
336                    self.interrupt_state.lock().update(false, !0);
337                    return;
338                }
339
340                let new_status = VirtioDeviceStatus::from(val as u8);
341                if new_status.acknowledge() {
342                    self.device_status.set_acknowledge(true);
343                }
344                if new_status.driver() {
345                    self.device_status.set_driver(true);
346                }
347                if new_status.failed() {
348                    self.device_status.set_failed(true);
349                }
350
351                if !self.device_status.features_ok() && new_status.features_ok() {
352                    self.device_status.set_features_ok(true);
353                    self.update_config_generation();
354                }
355
356                if !self.device_status.driver_ok() && new_status.driver_ok() {
357                    let notification_address =
358                        (address & !0xfff) + VirtioMmioRegister::QUEUE_NOTIFY.0 as u64;
359                    for i in 0..self.events.len() {
360                        self.doorbells.add(
361                            notification_address,
362                            Some(i as u64),
363                            Some(4),
364                            &self.events[i],
365                        );
366                    }
367                    let queues = self
368                        .queues
369                        .iter()
370                        .zip(self.events.iter().cloned())
371                        .map(|(queue, event)| {
372                            let interrupt_state = self.interrupt_state.clone();
373                            let notify = Interrupt::from_fn(move || {
374                                interrupt_state
375                                    .lock()
376                                    .update(true, VIRTIO_MMIO_INTERRUPT_STATUS_USED_BUFFER);
377                            });
378                            QueueResources {
379                                params: *queue,
380                                notify,
381                                event,
382                            }
383                        })
384                        .collect();
385
386                    match self.device.enable(Resources {
387                        features: self.driver_feature.clone(),
388                        queues,
389                        shared_memory_region: None,
390                        shared_memory_size: 0,
391                    }) {
392                        Ok(()) => {
393                            self.device_status.set_driver_ok(true);
394                        }
395                        Err(err) => {
396                            self.doorbells.clear();
397                            // FUTURE: consider setting DEVICE_NEEDS_RESET and
398                            // delivering a config change interrupt so the guest
399                            // can detect the failure proactively instead of
400                            // waiting for IO timeouts.
401                            tracelimit::error_ratelimited!(
402                                error = &*err as &dyn std::error::Error,
403                                "virtio device enable failed"
404                            );
405                        }
406                    }
407                    self.update_config_generation();
408                }
409            }
410            VirtioMmioRegister::QUEUE_DESC_LOW => {
411                if !queues_locked && queue_select < self.queues.len() {
412                    let queue = &mut self.queues[queue_select];
413                    queue.desc_addr = queue.desc_addr & 0xffffffff00000000 | val as u64;
414                }
415            }
416            VirtioMmioRegister::QUEUE_DESC_HIGH => {
417                if !queues_locked && queue_select < self.queues.len() {
418                    let queue = &mut self.queues[queue_select];
419                    queue.desc_addr = (val as u64) << 32 | queue.desc_addr & 0xffffffff;
420                }
421            }
422            VirtioMmioRegister::QUEUE_AVAIL_LOW => {
423                if !queues_locked && queue_select < self.queues.len() {
424                    let queue = &mut self.queues[queue_select];
425                    queue.avail_addr = queue.avail_addr & 0xffffffff00000000 | val as u64;
426                }
427            }
428            VirtioMmioRegister::QUEUE_AVAIL_HIGH => {
429                if !queues_locked && queue_select < self.queues.len() {
430                    let queue = &mut self.queues[queue_select];
431                    queue.avail_addr = (val as u64) << 32 | queue.avail_addr & 0xffffffff;
432                }
433            }
434            VirtioMmioRegister::QUEUE_USED_LOW => {
435                if !queues_locked && (queue_select) < self.queues.len() {
436                    let queue = &mut self.queues[queue_select];
437                    queue.used_addr = queue.used_addr & 0xffffffff00000000 | val as u64;
438                }
439            }
440            VirtioMmioRegister::QUEUE_USED_HIGH => {
441                if !queues_locked && queue_select < self.queues.len() {
442                    let queue = &mut self.queues[queue_select];
443                    queue.used_addr = (val as u64) << 32 | queue.used_addr & 0xffffffff;
444                }
445            }
446            VirtioMmioRegister(offset) if offset >= VirtioMmioRegister::CONFIG.0 => self
447                .device
448                .write_registers_u32(offset - VirtioMmioRegister::CONFIG.0, val),
449            _ => (),
450        }
451    }
452}
453
454impl ChangeDeviceState for VirtioMmioDevice {
455    fn start(&mut self) {}
456
457    async fn stop(&mut self) {}
458
459    async fn reset(&mut self) {
460        if self.device_status.driver_ok() || self.disabling {
461            self.doorbells.clear();
462            std::future::poll_fn(|cx| self.device.poll_disable(cx)).await;
463        }
464        self.device_status = VirtioDeviceStatus::new();
465        self.disabling = false;
466        self.config_generation = 0;
467        self.interrupt_state.lock().update(false, !0);
468    }
469}
470
471impl PollDevice for VirtioMmioDevice {
472    fn poll_device(&mut self, cx: &mut std::task::Context<'_>) {
473        self.poll_waker = Some(cx.waker().clone());
474        if self.disabling {
475            if self.device.poll_disable(cx).is_ready() {
476                self.device_status = VirtioDeviceStatus::new();
477                self.disabling = false;
478                self.interrupt_state.lock().update(false, !0);
479            }
480        }
481    }
482}
483
484impl ChipsetDevice for VirtioMmioDevice {
485    fn supports_mmio(&mut self) -> Option<&mut dyn MmioIntercept> {
486        Some(self)
487    }
488
489    fn supports_poll_device(&mut self) -> Option<&mut dyn PollDevice> {
490        Some(self)
491    }
492}
493
494impl SaveRestore for VirtioMmioDevice {
495    type SavedState = NoSavedState; // TODO
496
497    fn save(&mut self) -> Result<Self::SavedState, SaveError> {
498        Ok(NoSavedState)
499    }
500
501    fn restore(&mut self, NoSavedState: Self::SavedState) -> Result<(), RestoreError> {
502        Ok(())
503    }
504}
505
506impl MmioIntercept for VirtioMmioDevice {
507    fn mmio_read(&mut self, address: u64, data: &mut [u8]) -> IoResult {
508        read_as_u32_chunks(address, data, |address| self.read_u32(address));
509        IoResult::Ok
510    }
511
512    fn mmio_write(&mut self, address: u64, data: &[u8]) -> IoResult {
513        write_as_u32_chunks(address, data, |address, request_type| match request_type {
514            ReadWriteRequestType::Write(value) => {
515                self.write_u32(address, value);
516                None
517            }
518            ReadWriteRequestType::Read => Some(self.read_u32(address)),
519        });
520        IoResult::Ok
521    }
522
523    fn get_static_regions(&mut self) -> &[(&str, RangeInclusive<u64>)] {
524        std::slice::from_ref(&self.fixed_mmio_region)
525    }
526}
virtio/transport/mmio.rs

virtio/transport/
mmio.rs
