mana_driver/
queues.rs

1// Copyright (c) Microsoft Corporation.
2// Licensed under the MIT License.
3
4//! Types to access work, completion, and event queues.
5
6use crate::save_restore::CqEqSavedState;
7use crate::save_restore::DoorbellSavedState;
8use crate::save_restore::WqSavedState;
9use gdma_defs::CLIENT_OOB_8;
10use gdma_defs::CLIENT_OOB_24;
11use gdma_defs::CLIENT_OOB_32;
12use gdma_defs::CqEqDoorbellValue;
13use gdma_defs::Cqe;
14use gdma_defs::DB_CQ;
15use gdma_defs::DB_EQ;
16use gdma_defs::DB_RQ;
17use gdma_defs::DB_SQ;
18use gdma_defs::Eqe;
19use gdma_defs::GdmaQueueType;
20use gdma_defs::OWNER_BITS;
21use gdma_defs::OWNER_MASK;
22use gdma_defs::Sge;
23use gdma_defs::WQE_ALIGNMENT;
24use gdma_defs::WqDoorbellValue;
25use gdma_defs::WqeHeader;
26use gdma_defs::WqeParams;
27use inspect::Inspect;
28use std::marker::PhantomData;
29use std::sync::Arc;
30use std::sync::atomic::Ordering::Acquire;
31use user_driver::memory::MemoryBlock;
32use zerocopy::FromBytes;
33use zerocopy::Immutable;
34use zerocopy::IntoBytes;
35use zerocopy::KnownLayout;
36
37/// An interface to write a doorbell value to signal the device.
38pub trait Doorbell: Send + Sync {
39    /// Returns the maximum page number.
40    fn page_count(&self) -> u32;
41    /// Write a doorbell value at page `page`, offset `address`.
42    fn write(&self, page: u32, address: u32, value: u64);
43    /// Save the doorbell state.
44    fn save(&self, doorbell_id: Option<u64>) -> DoorbellSavedState;
45}
46
47struct NullDoorbell;
48
49impl Doorbell for NullDoorbell {
50    fn page_count(&self) -> u32 {
51        0
52    }
53
54    fn write(&self, _page: u32, _address: u32, _value: u64) {}
55
56    fn save(&self, _doorbell_id: Option<u64>) -> DoorbellSavedState {
57        DoorbellSavedState {
58            doorbell_id: 0,
59            page_count: 0,
60        }
61    }
62}
63
64/// A single GDMA doorbell page.
65#[derive(Clone)]
66pub struct DoorbellPage {
67    doorbell: Arc<dyn Doorbell>,
68    doorbell_id: u32,
69}
70
71impl DoorbellPage {
72    pub(crate) fn null() -> Self {
73        Self {
74            doorbell: Arc::new(NullDoorbell),
75            doorbell_id: 0,
76        }
77    }
78
79    /// Returns a doorbell page at `doorbell_id` the doorbell region.
80    pub fn new(doorbell: Arc<dyn Doorbell>, doorbell_id: u32) -> anyhow::Result<Self> {
81        let page_count = doorbell.page_count();
82        if doorbell_id >= page_count {
83            anyhow::bail!(
84                "doorbell id {} exceeds page count {}",
85                doorbell_id,
86                page_count
87            );
88        }
89        Ok(Self {
90            doorbell,
91            doorbell_id,
92        })
93    }
94
95    /// Writes a doorbell value.
96    pub fn write(&self, address: u32, value: u64) {
97        assert!(address < 4096);
98        self.doorbell.write(self.doorbell_id, address, value);
99    }
100}
101
102/// An event queue.
103pub struct CqEq<T> {
104    doorbell: DoorbellPage,
105    doorbell_addr: u32,
106    queue_type: GdmaQueueType,
107    mem: MemoryBlock,
108    id: u32,
109    next: u32,
110    size: u32,
111    shift: u32,
112    _phantom: PhantomData<fn() -> T>,
113}
114
115impl<T> Inspect for CqEq<T> {
116    fn inspect(&self, req: inspect::Request<'_>) {
117        req.respond()
118            .field("id", self.id)
119            .hex("size", self.size)
120            .hex("next", self.next);
121    }
122}
123
124impl CqEq<Cqe> {
125    /// Creates a new completion queue.
126    pub fn new_cq(mem: MemoryBlock, doorbell: DoorbellPage, id: u32) -> Self {
127        Self::new(GdmaQueueType::GDMA_CQ, DB_CQ, mem, doorbell, id)
128    }
129
130    /// Restores an existing completion queue.
131    pub fn restore_cq(mem: MemoryBlock, state: CqEqSavedState, doorbell: DoorbellPage) -> Self {
132        Self::restore(GdmaQueueType::GDMA_CQ, mem, doorbell, state)
133    }
134}
135
136impl CqEq<Eqe> {
137    /// Creates a new event queue.
138    pub fn new_eq(mem: MemoryBlock, doorbell: DoorbellPage, id: u32) -> Self {
139        Self::new(GdmaQueueType::GDMA_EQ, DB_EQ, mem, doorbell, id)
140    }
141
142    /// Restores an existing event queue.
143    pub fn restore_eq(mem: MemoryBlock, state: CqEqSavedState, doorbell: DoorbellPage) -> Self {
144        Self::restore(GdmaQueueType::GDMA_EQ, mem, doorbell, state)
145    }
146}
147
148impl<T: IntoBytes + FromBytes + Immutable + KnownLayout> CqEq<T> {
149    /// Creates a new queue.
150    fn new(
151        queue_type: GdmaQueueType,
152        doorbell_addr: u32,
153        mem: MemoryBlock,
154        doorbell: DoorbellPage,
155        id: u32,
156    ) -> Self {
157        let size = mem.len();
158        assert!(size.is_power_of_two());
159        Self {
160            doorbell,
161            doorbell_addr,
162            queue_type,
163            mem,
164            id,
165            next: size as u32,
166            size: size as u32,
167            shift: size.trailing_zeros(),
168            _phantom: PhantomData,
169        }
170    }
171
172    /// Save the state of the queue for restoration after servicing.
173    pub fn save(&self) -> CqEqSavedState {
174        CqEqSavedState {
175            doorbell: DoorbellSavedState {
176                doorbell_id: self.doorbell.doorbell_id as u64,
177                page_count: self.doorbell.doorbell.page_count(),
178            },
179            doorbell_addr: self.doorbell_addr,
180            id: self.id,
181            next: self.next,
182            size: self.size,
183            shift: self.shift,
184        }
185    }
186
187    /// Restore a queue from saved state.
188    pub fn restore(
189        queue_type: GdmaQueueType,
190        mem: MemoryBlock,
191        doorbell: DoorbellPage,
192        state: CqEqSavedState,
193    ) -> Self {
194        Self {
195            doorbell,
196            doorbell_addr: state.doorbell_addr,
197            queue_type,
198            mem,
199            id: state.id,
200            next: state.next,
201            size: state.size,
202            shift: state.shift,
203            _phantom: PhantomData,
204        }
205    }
206
207    /// Updates the queue ID.
208    pub(crate) fn set_id(&mut self, id: u32) {
209        self.id = id;
210    }
211
212    /// Updates the doorbell page.
213    pub(crate) fn set_doorbell(&mut self, page: DoorbellPage) {
214        self.doorbell = page;
215    }
216
217    /// Gets the queue ID.
218    pub fn id(&self) -> u32 {
219        self.id
220    }
221
222    fn read_next<U: FromBytes + Immutable + KnownLayout>(&self, offset: u32) -> U {
223        assert!((offset as usize & (size_of::<T>() - 1)) + size_of::<U>() <= size_of::<T>());
224        self.mem
225            .read_obj((self.next.wrapping_add(offset) & (self.size - 1)) as usize)
226    }
227
228    /// Pops an event queue entry.
229    pub fn pop(&mut self) -> Option<T> {
230        // Perform an acquire load to ensure that the read of the queue entry is
231        // not reordered before the read of the owner count.
232        let b = self.mem.as_slice()
233            [(self.next.wrapping_add(size_of::<T>() as u32 - 1) & (self.size - 1)) as usize]
234            .load(Acquire);
235        let owner_count = b >> 5;
236        let cur_owner_count = (self.next >> self.shift) as u8;
237        if owner_count == (cur_owner_count.wrapping_sub(1)) & OWNER_MASK as u8 {
238            None
239        } else if owner_count == cur_owner_count & OWNER_MASK as u8 {
240            let qe = self.read_next::<T>(0);
241            self.next = self.next.wrapping_add(size_of_val(&qe) as u32);
242            Some(qe)
243        } else {
244            tracing::error!(next = self.next, owner_count, queue_type = ?self.queue_type, id = self.id, "eq/cq wrapped");
245            None
246        }
247    }
248
249    fn flush(&mut self, arm: bool) {
250        let tail = self.next & ((self.size << OWNER_BITS) - 1);
251        let value = CqEqDoorbellValue::new()
252            .with_arm(arm)
253            .with_id(self.id)
254            .with_tail(tail / size_of::<T>() as u32);
255        tracing::trace!(queue_type = ?self.queue_type, id = self.id, ?value, "cq/eq doorbell write");
256        self.doorbell.write(self.doorbell_addr, value.into());
257    }
258
259    /// Arms the event queue so that an interrupt will be delivered next time an
260    /// event arrives.
261    pub fn arm(&mut self) {
262        self.flush(true);
263    }
264
265    /// Ack's the queue. Interrupt will not be delivered until it is armed.
266    pub fn ack(&mut self) {
267        self.flush(false);
268    }
269
270    /// Reports next value for diagnostics
271    pub fn get_next(&mut self) -> u32 {
272        self.next
273    }
274}
275
276/// A completion queue.
277pub type Cq = CqEq<Cqe>;
278
279/// An event queue.
280pub type Eq = CqEq<Eqe>;
281
282/// A work queue (send or receive).
283pub struct Wq {
284    doorbell: DoorbellPage,
285    queue_type: GdmaQueueType,
286    doorbell_addr: u32,
287    mem: MemoryBlock,
288    id: u32,
289    head: u32,
290    tail: u32,
291    mask: u32,
292    uncommitted_count: u32,
293}
294
295impl Inspect for Wq {
296    fn inspect(&self, req: inspect::Request<'_>) {
297        req.respond()
298            .field("id", self.id)
299            .hex("size", self.mask + 1)
300            .hex("head", self.head)
301            .hex("tail", self.tail)
302            .field("uncommited", self.uncommitted_count);
303    }
304}
305
306/// An error indicating the queue is full.
307#[derive(Debug)]
308pub struct QueueFull;
309
310impl Wq {
311    /// Creates a new send work queue.
312    pub fn new_sq(mem: MemoryBlock, doorbell: DoorbellPage, id: u32) -> Self {
313        Self::new(GdmaQueueType::GDMA_SQ, DB_SQ, mem, doorbell, id)
314    }
315
316    /// Creates a new receive work queue.
317    pub fn new_rq(mem: MemoryBlock, doorbell: DoorbellPage, id: u32) -> Self {
318        Self::new(GdmaQueueType::GDMA_RQ, DB_RQ, mem, doorbell, id)
319    }
320
321    /// Creates a new work queue.
322    fn new(
323        queue_type: GdmaQueueType,
324        doorbell_addr: u32,
325        mem: MemoryBlock,
326        doorbell: DoorbellPage,
327        id: u32,
328    ) -> Self {
329        let size = mem.len() as u32;
330        assert!(size.is_power_of_two());
331        Self {
332            doorbell,
333            queue_type,
334            doorbell_addr,
335            mem,
336            id,
337            head: size,
338            tail: 0,
339            mask: size - 1,
340            uncommitted_count: 0,
341        }
342    }
343
344    /// Save the state of the Wq for restoration after servicing
345    pub fn save(&self) -> WqSavedState {
346        WqSavedState {
347            doorbell: DoorbellSavedState {
348                doorbell_id: self.doorbell.doorbell_id as u64,
349                page_count: self.doorbell.doorbell.page_count(),
350            },
351            doorbell_addr: self.doorbell_addr,
352            id: self.id,
353            head: self.head,
354            tail: self.tail,
355            mask: self.mask,
356        }
357    }
358
359    /// Restores an existing receive work queue.
360    pub fn restore_rq(
361        mem: MemoryBlock,
362        state: WqSavedState,
363        doorbell: DoorbellPage,
364    ) -> anyhow::Result<Self> {
365        Ok(Self {
366            doorbell,
367            doorbell_addr: state.doorbell_addr,
368            queue_type: GdmaQueueType::GDMA_RQ,
369            mem,
370            id: state.id,
371            head: state.head,
372            tail: state.tail,
373            mask: state.mask,
374            uncommitted_count: 0,
375        })
376    }
377
378    /// Restores an existing send work queue.
379    pub fn restore_sq(
380        mem: MemoryBlock,
381        state: WqSavedState,
382        doorbell: DoorbellPage,
383    ) -> anyhow::Result<Self> {
384        Ok(Self {
385            doorbell,
386            doorbell_addr: state.doorbell_addr,
387            queue_type: GdmaQueueType::GDMA_SQ,
388            mem,
389            id: state.id,
390            head: state.head,
391            tail: state.tail,
392            mask: state.mask,
393            uncommitted_count: 0,
394        })
395    }
396
397    /// Returns the queue ID.
398    pub fn id(&self) -> u32 {
399        self.id
400    }
401
402    /// Advances the head, indicating that `n` more bytes are available in the ring.
403    pub fn advance_head(&mut self, n: u32) {
404        assert!(n.is_multiple_of(WQE_ALIGNMENT as u32));
405        self.head = self.head.wrapping_add(n);
406    }
407
408    fn get_offset_in_buffer_in_bytes(&self, offset: u32) -> usize {
409        (offset as usize * WQE_ALIGNMENT) & self.mask as usize
410    }
411
412    /// Reads from the offset, the first `n` bytes.
413    pub fn read(&mut self, offset: u32, n: usize) -> Vec<u8> {
414        let mut buf = vec![0; n];
415        let offset_in_buffer = self.get_offset_in_buffer_in_bytes(offset);
416        self.mem.read_at(offset_in_buffer, &mut buf);
417        buf
418    }
419
420    fn write_tail(&self, offset: u32, data: &[u8]) {
421        debug_assert!(
422            offset as usize % WQE_ALIGNMENT + data.len() <= WQE_ALIGNMENT,
423            "can't write more than one queue segment at a time to avoid wrapping"
424        );
425        self.mem
426            .write_at((self.tail.wrapping_add(offset) & self.mask) as usize, data);
427    }
428
429    /// Returns the number of bytes available in the ring.
430    pub fn available(&self) -> u32 {
431        self.head.wrapping_sub(self.tail)
432    }
433
434    /// Pushes a new work queue entry with an inline out-of-band buffer and
435    /// external data via a scatter-gather list.
436    pub fn push<I: IntoIterator<Item = Sge>>(
437        &mut self,
438        oob: impl IntoBytes + Immutable + KnownLayout,
439        sgl: I,
440    ) -> Result<u32, QueueFull> {
441        let mut builder = self.wqe_builder(oob);
442        for sge in sgl {
443            builder.push_sge(sge);
444        }
445        builder.finish()
446    }
447
448    /// Begins building a work queue entry with an inline out-of-band buffer.
449    pub fn wqe_builder(&mut self, oob: impl IntoBytes + Immutable + KnownLayout) -> WqeBuilder<'_> {
450        WqeBuilder::new(self, oob)
451    }
452
453    /// Commits all written entries by updating the doorbell value observed by
454    /// the device.
455    pub fn commit(&mut self) {
456        // N.B. the tail is not masked to the queue size.
457        let mut value = WqDoorbellValue::new().with_id(self.id).with_tail(self.tail);
458        if self.queue_type == GdmaQueueType::GDMA_RQ {
459            // If this overflows, it's probably for a device type (like bnic)
460            // that ignores it.
461            value.set_num_rwqe(self.uncommitted_count as u8);
462        }
463        tracing::trace!(queue_type = ?self.queue_type, id = self.id, ?value, "wq doorbell write");
464        self.doorbell.write(self.doorbell_addr, value.into());
465        self.uncommitted_count = 0;
466    }
467
468    /// Reports tail value for diagnostics
469    pub fn get_tail(&mut self) -> u32 {
470        self.tail
471    }
472}
473
474/// A builder for a work queue entry.
475pub struct WqeBuilder<'a> {
476    wq: &'a mut Wq,
477    len: u32,
478    max: u32,
479    hdr: WqeHeader,
480}
481
482impl<'a> WqeBuilder<'a> {
483    fn new(wq: &'a mut Wq, oob: impl IntoBytes + Immutable + KnownLayout) -> Self {
484        let oob_size = match size_of_val(&oob) {
485            0 | 8 => CLIENT_OOB_8,
486            24 => CLIENT_OOB_24,
487            32 => CLIENT_OOB_32,
488            _ => panic!("invalid oob size"),
489        };
490        let max = wq.available();
491        let len =
492            (size_of::<WqeHeader>() + size_of_val(&oob)).next_multiple_of(size_of::<Sge>()) as u32;
493
494        // Save the header to write later.
495        let hdr = WqeHeader {
496            reserved: [0; 3],
497            last_vbytes: 0,
498            params: WqeParams::new().with_inline_client_oob_size(oob_size),
499        };
500
501        // Write the out-of-band data.
502        if len <= max {
503            match size_of_val(&oob) {
504                0 => {}
505                8 | 24 => {
506                    wq.write_tail(8, oob.as_bytes());
507                }
508                32 => {
509                    wq.write_tail(8, &oob.as_bytes()[..24]);
510                    wq.write_tail(32, &oob.as_bytes()[24..]);
511                }
512                _ => unreachable!(),
513            }
514        }
515
516        Self { wq, len, max, hdr }
517    }
518
519    /// Sets that client out-of-band data is provided in the first scatter-gather
520    /// entry, with the specified `last_vbytes` value.
521    pub fn set_client_oob_in_sgl(&mut self, last_vbytes: u8) {
522        self.hdr.last_vbytes = last_vbytes;
523        self.hdr.params.set_client_oob_in_sgl(true);
524    }
525
526    /// Sets the `gd_client_unit_data` field in the header.
527    pub fn set_gd_client_unit_data(&mut self, value: u16) {
528        self.hdr.params.set_gd_client_unit_data(value);
529    }
530
531    /// Appends a scatter-gather entry to the work queue entry.
532    pub fn push_sge(&mut self, sge: Sge) {
533        let offset = self.len;
534        self.len += size_of_val(&sge) as u32;
535        if self.len <= self.max {
536            self.wq.write_tail(offset, sge.as_bytes());
537        }
538        self.hdr
539            .params
540            .set_num_sgl_entries(self.hdr.params.num_sgl_entries() + 1);
541    }
542
543    /// Returns the number of scatter-gather entries added so far.
544    pub fn sge_count(&self) -> u8 {
545        self.hdr.params.num_sgl_entries()
546    }
547
548    /// Finishes building the work queue entry and writes it to the queue,
549    /// updating the tail and returning the total length of the entry.
550    ///
551    /// Call [`Wq::commit`] to notify the device of the new entry.
552    pub fn finish(self) -> Result<u32, QueueFull> {
553        let aligned_len = self.len.next_multiple_of(WQE_ALIGNMENT as u32);
554        if aligned_len > self.max {
555            return Err(QueueFull);
556        }
557        self.wq.write_tail(0, self.hdr.as_bytes());
558        self.wq.tail = self.wq.tail.wrapping_add(aligned_len);
559        self.wq.uncommitted_count += 1;
560        Ok(aligned_len)
561    }
562}
mana_driver/queues.rs

mana_driver/
queues.rs
