pci_core/capabilities/
msix.rs

1// Copyright (c) Microsoft Corporation.
2// Licensed under the MIT License.
3
4//! MSI-X Capability.
5
6use super::PciCapability;
7use crate::msi::MsiTarget;
8use crate::spec::caps::CapabilityId;
9use crate::spec::caps::msix::MsixCapabilityHeader;
10use crate::spec::caps::msix::MsixTableEntryIdx;
11use inspect::Inspect;
12use inspect::InspectMut;
13use parking_lot::Mutex;
14use std::fmt::Debug;
15use std::sync::Arc;
16use vmcore::interrupt::Interrupt;
17
18#[derive(Debug, Inspect)]
19struct MsiTableLocation {
20    #[inspect(hex)]
21    offset: u32,
22    bar: u8,
23}
24
25impl MsiTableLocation {
26    fn new(bar: u8, offset: u32) -> Self {
27        assert!(bar < 6);
28        assert!(offset & 7 == 0);
29        Self { offset, bar }
30    }
31
32    fn read_u32(&self) -> u32 {
33        self.offset | self.bar as u32
34    }
35}
36
37#[derive(Inspect)]
38struct MsixCapability {
39    count: u16,
40    #[inspect(with = "|x| inspect::adhoc(|req| x.lock().inspect_mut(req))")]
41    state: Arc<Mutex<MsixState>>,
42    config_table_location: MsiTableLocation,
43    pending_bits_location: MsiTableLocation,
44}
45
46impl PciCapability for MsixCapability {
47    fn label(&self) -> &str {
48        "msi-x"
49    }
50
51    fn capability_id(&self) -> CapabilityId {
52        CapabilityId::MSIX
53    }
54
55    fn len(&self) -> usize {
56        12
57    }
58
59    fn read_u32(&self, offset: u16) -> u32 {
60        match MsixCapabilityHeader(offset) {
61            MsixCapabilityHeader::CONTROL_CAPS => {
62                CapabilityId::MSIX.0 as u32
63                    | ((self.count as u32 - 1) | if self.state.lock().enabled { 0x8000 } else { 0 })
64                        << 16
65            }
66            MsixCapabilityHeader::OFFSET_TABLE => self.config_table_location.read_u32(),
67            MsixCapabilityHeader::OFFSET_PBA => self.pending_bits_location.read_u32(),
68            _ => panic!("Unreachable read offset {}", offset),
69        }
70    }
71
72    fn write_u32(&mut self, offset: u16, val: u32) {
73        match MsixCapabilityHeader(offset) {
74            MsixCapabilityHeader::CONTROL_CAPS => {
75                let enabled = val & 0x80000000 != 0;
76                let mut state = self.state.lock();
77                let was_enabled = state.enabled;
78                state.enabled = enabled;
79                if was_enabled && !enabled {
80                    for entry in &mut state.vectors {
81                        if entry.is_enabled(true) {
82                            entry.msi.disable();
83                        }
84                    }
85                } else if enabled && !was_enabled {
86                    for entry in &mut state.vectors {
87                        if entry.is_enabled(true) {
88                            entry.msi.enable(
89                                entry.state.address,
90                                entry.state.data,
91                                entry.state.is_pending,
92                            );
93                            entry.state.is_pending = false;
94                        }
95                    }
96                }
97            }
98            MsixCapabilityHeader::OFFSET_TABLE | MsixCapabilityHeader::OFFSET_PBA => {
99                tracelimit::warn_ratelimited!(
100                    "Unexpected write offset {:?}",
101                    MsixCapabilityHeader(offset)
102                )
103            }
104            _ => panic!("Unreachable write offset {}", offset),
105        }
106    }
107
108    fn reset(&mut self) {
109        let mut state = self.state.lock();
110        state.enabled = false;
111        for vector in &mut state.vectors {
112            vector.state = EntryState::new();
113        }
114    }
115}
116
117#[derive(Clone, Inspect, Debug)]
118pub(crate) struct MsiInterrupt(#[inspect(flatten)] Arc<Mutex<MsiInterruptInner>>);
119
120#[derive(Inspect, Debug)]
121struct MsiInterruptInner {
122    target: MsiTarget,
123    pending: bool,
124    enabled: bool,
125    address: u64,
126    data: u32,
127}
128
129impl MsiInterrupt {
130    pub fn new(target: MsiTarget) -> Self {
131        Self(Arc::new(Mutex::new(MsiInterruptInner {
132            target,
133            pending: false,
134            enabled: false,
135            address: 0,
136            data: 0,
137        })))
138    }
139
140    pub fn enable(&self, address: u64, data: u32, set_pending: bool) {
141        let mut state = self.0.lock();
142        state.pending |= set_pending;
143        state.address = address;
144        state.data = data;
145        state.enabled = true;
146        if state.pending {
147            state.target.signal_msi(0, address, data);
148            state.pending = false;
149        }
150    }
151
152    pub fn disable(&self) {
153        let mut state = self.0.lock();
154        state.enabled = false;
155    }
156
157    pub fn drain_pending(&self) -> bool {
158        let mut state = self.0.lock();
159        let was_pending = state.pending;
160        state.pending = false;
161        was_pending
162    }
163
164    pub fn interrupt(&self) -> Interrupt {
165        let state = self.0.clone();
166        Interrupt::from_fn(move || {
167            let mut state = state.lock();
168            if state.enabled {
169                state.target.signal_msi(0, state.address, state.data);
170            } else {
171                state.pending = true;
172            }
173        })
174    }
175}
176
177struct MsixMessageTableEntry {
178    msi: MsiInterrupt,
179    state: EntryState,
180}
181
182impl InspectMut for MsixMessageTableEntry {
183    fn inspect_mut(&mut self, req: inspect::Request<'_>) {
184        req.respond()
185            .hex("address", self.state.address)
186            .hex("data", self.state.data)
187            .hex("control", self.state.control)
188            .field("enabled", self.state.control & 1 == 0)
189            .field("is_pending", self.check_is_pending(true));
190    }
191}
192
193#[derive(Debug)]
194struct EntryState {
195    address: u64,
196    data: u32,
197    control: u32,
198    is_pending: bool,
199}
200
201impl EntryState {
202    fn new() -> Self {
203        Self {
204            address: 0,
205            data: 0,
206            control: 1,
207            is_pending: false,
208        }
209    }
210}
211
212impl MsixMessageTableEntry {
213    fn new(msi: MsiInterrupt) -> Self {
214        Self {
215            msi,
216            state: EntryState::new(),
217        }
218    }
219
220    fn read_u32(&self, offset: u16) -> u32 {
221        match MsixTableEntryIdx(offset) {
222            MsixTableEntryIdx::MSG_ADDR_LO => self.state.address as u32,
223            MsixTableEntryIdx::MSG_ADDR_HI => (self.state.address >> 32) as u32,
224            MsixTableEntryIdx::MSG_DATA => self.state.data,
225            MsixTableEntryIdx::VECTOR_CTL => self.state.control,
226            _ => panic!("Unexpected read offset {}", offset),
227        }
228    }
229
230    fn write_u32(&mut self, offset: u16, val: u32) {
231        match MsixTableEntryIdx(offset) {
232            MsixTableEntryIdx::MSG_ADDR_LO => {
233                self.state.address = (self.state.address & 0xffffffff00000000) | val as u64
234            }
235            MsixTableEntryIdx::MSG_ADDR_HI => {
236                self.state.address = (val as u64) << 32 | self.state.address & 0xffffffff
237            }
238            MsixTableEntryIdx::MSG_DATA => self.state.data = val,
239            MsixTableEntryIdx::VECTOR_CTL => self.state.control = val,
240            _ => panic!("Unexpected write offset {}", offset),
241        }
242    }
243
244    fn is_enabled(&self, global_enabled: bool) -> bool {
245        global_enabled && self.state.control & 1 == 0
246    }
247
248    fn check_is_pending(&mut self, global_enabled: bool) -> bool {
249        if !self.state.is_pending && !self.is_enabled(global_enabled) {
250            self.state.is_pending = self.msi.drain_pending();
251        }
252        self.state.is_pending
253    }
254}
255
256#[derive(InspectMut)]
257struct MsixState {
258    enabled: bool,
259    #[inspect(mut, with = "inspect_entries")]
260    vectors: Vec<MsixMessageTableEntry>,
261}
262
263fn inspect_entries(entries: &mut [MsixMessageTableEntry]) -> impl '_ + InspectMut {
264    inspect::adhoc_mut(|req| {
265        let mut resp = req.respond();
266        for (i, entry) in entries.iter_mut().enumerate() {
267            resp.field_mut(&i.to_string(), entry);
268        }
269    })
270}
271
272/// Emulator for the hardware-level interface required to configure and trigger
273/// MSI-X interrupts on a PCI device.
274#[derive(Clone)]
275pub struct MsixEmulator {
276    state: Arc<Mutex<MsixState>>,
277    pending_bits_offset: u16,
278    pending_bits_dword_count: u16,
279}
280
281impl MsixEmulator {
282    /// Create a new [`MsixEmulator`] instance, along with with its associated
283    /// [`PciCapability`] structure.
284    ///
285    /// This implementation of MSI-X expects a dedicated BAR to store the vector
286    /// and pending tables.
287    ///
288    /// * * *
289    ///
290    /// DEVNOTE: This current implementation of MSI-X isn't particularly
291    /// "flexible" with respect to the various ways the PCI spec allows MSI-X to
292    /// be implemented. e.g: it uses a shared BAR for the table and BPA, with
293    /// fixed offsets into the BAR for both of those tables. It would be nice to
294    /// re-visit this code and make it more flexible.
295    pub fn new(bar: u8, count: u16, msi_target: &MsiTarget) -> (Self, impl PciCapability + use<>) {
296        let state = MsixState {
297            enabled: false,
298            vectors: (0..count)
299                .map(|_| MsixMessageTableEntry::new(MsiInterrupt::new(msi_target.clone())))
300                .collect(),
301        };
302        let state = Arc::new(Mutex::new(state));
303        let pending_bits_offset = count * 16;
304        (
305            Self {
306                state: state.clone(),
307                pending_bits_offset,
308                pending_bits_dword_count: count.div_ceil(32),
309            },
310            MsixCapability {
311                count,
312                state,
313                config_table_location: MsiTableLocation::new(bar, 0),
314                pending_bits_location: MsiTableLocation::new(bar, pending_bits_offset.into()),
315            },
316        )
317    }
318
319    /// Return the total length of the MSI-X BAR
320    pub fn bar_len(&self) -> u64 {
321        (self.pending_bits_offset + self.pending_bits_dword_count * 4).into()
322    }
323
324    /// Read a `u32` from the MSI-X BAR at the given offset.
325    pub fn read_u32(&self, offset: u16) -> u32 {
326        let mut state = self.state.lock();
327        let state: &mut MsixState = &mut state;
328        if offset < self.pending_bits_offset {
329            let index = offset / 16;
330            if let Some(entry) = state.vectors.get(index as usize) {
331                return entry.read_u32(offset & 0xf);
332            }
333        } else {
334            let dword = (offset - self.pending_bits_offset) / 4;
335            let start = dword as usize * 32;
336            if start < state.vectors.len() {
337                let end = (start + 32).min(state.vectors.len());
338                let mut val = 0u32;
339                for (i, entry) in state.vectors[start..end].iter_mut().enumerate() {
340                    if entry.check_is_pending(state.enabled) {
341                        val |= 1 << i;
342                    }
343                }
344                return val;
345            }
346        }
347        tracelimit::warn_ratelimited!(offset, "Unexpected read offset");
348        0
349    }
350
351    /// Write a `u32` to the MSI-X BAR at the given offset.
352    pub fn write_u32(&mut self, offset: u16, val: u32) {
353        let mut state = self.state.lock();
354        if offset < self.pending_bits_offset {
355            let index = offset / 16;
356            let global = state.enabled;
357            if let Some(entry) = state.vectors.get_mut(index as usize) {
358                let was_enabled = entry.is_enabled(global);
359                entry.write_u32(offset & 0xf, val);
360                let is_enabled = entry.is_enabled(global);
361                if is_enabled && !was_enabled {
362                    entry.msi.enable(
363                        entry.state.address,
364                        entry.state.data,
365                        entry.state.is_pending,
366                    );
367                    entry.state.is_pending = false;
368                } else if was_enabled && !is_enabled {
369                    entry.msi.disable();
370                }
371                return;
372            }
373        } else if offset - self.pending_bits_offset < self.pending_bits_dword_count * 4 {
374            return;
375        }
376        tracelimit::warn_ratelimited!(offset, "Unexpected write offset");
377    }
378
379    /// Return an [`Interrupt`] associated with the particular MSI-X vector, or
380    /// `None` if the index is out of bounds.
381    pub fn interrupt(&self, index: u16) -> Option<Interrupt> {
382        Some(
383            self.state
384                .lock()
385                .vectors
386                .get_mut(index as usize)?
387                .msi
388                .interrupt(),
389        )
390    }
391
392    #[cfg(test)]
393    fn clear_pending_bit(&self, index: u8) {
394        let mut state = self.state.lock();
395        state.vectors[index as usize].state.is_pending = false;
396    }
397
398    #[cfg(test)]
399    fn set_pending_bit(&self, index: u8) {
400        let mut state = self.state.lock();
401        state.vectors[index as usize].state.is_pending = true;
402    }
403}
404
405mod save_restore {
406    use super::*;
407    use thiserror::Error;
408    use vmcore::save_restore::RestoreError;
409    use vmcore::save_restore::SaveError;
410    use vmcore::save_restore::SaveRestore;
411
412    mod state {
413        use mesh::payload::Protobuf;
414        use vmcore::save_restore::SavedStateRoot;
415
416        #[derive(Debug, Protobuf)]
417        #[mesh(package = "pci.caps.msix")]
418        pub struct SavedMsixMessageTableEntryState {
419            #[mesh(1)]
420            pub address: u64,
421            #[mesh(2)]
422            pub data: u32,
423            #[mesh(3)]
424            pub control: u32,
425            #[mesh(4)]
426            pub is_pending: bool,
427        }
428
429        #[derive(Debug, Protobuf, SavedStateRoot)]
430        #[mesh(package = "pci.caps.msix")]
431        pub struct SavedState {
432            #[mesh(2)]
433            pub enabled: bool,
434            #[mesh(3)]
435            pub vectors: Vec<SavedMsixMessageTableEntryState>,
436        }
437    }
438
439    #[derive(Debug, Error)]
440    enum MsixRestoreError {
441        #[error("mismatched vector lengths: current:{0}, saved:{1}")]
442        MismatchedTableLengths(usize, usize),
443    }
444
445    impl SaveRestore for MsixCapability {
446        type SavedState = state::SavedState;
447
448        fn save(&mut self) -> Result<Self::SavedState, SaveError> {
449            let state = self.state.lock();
450            let saved_state = state::SavedState {
451                enabled: state.enabled,
452                vectors: {
453                    state
454                        .vectors
455                        .iter()
456                        .map(|vec| {
457                            let EntryState {
458                                address,
459                                data,
460                                control,
461                                is_pending,
462                            } = vec.state;
463
464                            state::SavedMsixMessageTableEntryState {
465                                address,
466                                data,
467                                control,
468                                is_pending,
469                            }
470                        })
471                        .collect()
472                },
473            };
474            Ok(saved_state)
475        }
476
477        fn restore(&mut self, state: Self::SavedState) -> Result<(), RestoreError> {
478            let state::SavedState { enabled, vectors } = state;
479
480            let mut state = self.state.lock();
481            state.enabled = enabled;
482
483            if vectors.len() != state.vectors.len() {
484                return Err(RestoreError::InvalidSavedState(
485                    MsixRestoreError::MismatchedTableLengths(vectors.len(), state.vectors.len())
486                        .into(),
487                ));
488            }
489
490            for (new_vec, vec) in vectors.into_iter().zip(state.vectors.iter_mut()) {
491                vec.state = EntryState {
492                    address: new_vec.address,
493                    data: new_vec.data,
494                    control: new_vec.control,
495                    is_pending: new_vec.is_pending,
496                }
497            }
498
499            Ok(())
500        }
501    }
502}
503
504#[cfg(test)]
505mod tests {
506    use super::*;
507    use crate::{msi::MsiConnection, test_helpers::TestPciInterruptController};
508
509    #[test]
510    fn msix_check() {
511        let msi_conn = MsiConnection::new();
512        let (mut msix, mut cap) = MsixEmulator::new(2, 64, msi_conn.target());
513        let msi_controller = TestPciInterruptController::new();
514        msi_conn.connect(msi_controller.signal_msi());
515        // check capabilities
516        assert_eq!(cap.read_u32(0), 0x3f0011);
517        assert_eq!(cap.read_u32(4), 2);
518        assert_eq!(cap.read_u32(8), 0x402);
519        cap.write_u32(0, 0xffffffff);
520        assert_eq!(cap.read_u32(0), 0x803f0011);
521        // check BAR
522        // Vector[0]
523        assert_eq!(msix.read_u32(0), 0);
524        assert_eq!(msix.read_u32(4), 0);
525        assert_eq!(msix.read_u32(8), 0);
526        assert_eq!(msix.read_u32(12), 1);
527        msix.write_u32(0, 0x12345678);
528        msix.write_u32(4, 0x9abcdef0);
529        msix.write_u32(8, 0x123);
530        msix.write_u32(12, 0x456);
531        assert_eq!(msix.read_u32(0), 0x12345678);
532        assert_eq!(msix.read_u32(4), 0x9abcdef0);
533        assert_eq!(msix.read_u32(8), 0x123);
534        assert_eq!(msix.read_u32(12), 0x456);
535        // Vector[63]
536        assert_eq!(msix.read_u32(0x3f0), 0);
537        assert_eq!(msix.read_u32(0x3f4), 0);
538        assert_eq!(msix.read_u32(0x3f8), 0);
539        assert_eq!(msix.read_u32(0x3fc), 1);
540        msix.write_u32(0x3f0, 0x12345678);
541        msix.write_u32(0x3f4, 0x9abcdef0);
542        msix.write_u32(0x3f8, 0x123);
543        msix.write_u32(0x3fc, 0x456);
544        assert_eq!(msix.read_u32(0x3f0), 0x12345678);
545        assert_eq!(msix.read_u32(0x3f4), 0x9abcdef0);
546        assert_eq!(msix.read_u32(0x3f8), 0x123);
547        assert_eq!(msix.read_u32(0x3fc), 0x456);
548        // Pending Bit Array
549        assert_eq!(msix.read_u32(0x400), 0);
550        assert_eq!(msix.read_u32(0x404), 0);
551        msix.set_pending_bit(1);
552        assert_eq!(msix.read_u32(0x400), 2);
553        assert_eq!(msix.read_u32(0x404), 0);
554        msix.set_pending_bit(33);
555        assert_eq!(msix.read_u32(0x400), 2);
556        assert_eq!(msix.read_u32(0x404), 2);
557        msix.set_pending_bit(63);
558        msix.set_pending_bit(31);
559        assert_eq!(msix.read_u32(0x400), 0x80000002);
560        assert_eq!(msix.read_u32(0x404), 0x80000002);
561        msix.clear_pending_bit(1);
562        assert_eq!(msix.read_u32(0x400), 0x80000000);
563        assert_eq!(msix.read_u32(0x404), 0x80000002);
564    }
565}
pci_core/capabilities/msix.rs

pci_core/capabilities/
msix.rs
