nvme_driver/
queues.rs

1// Copyright (c) Microsoft Corporation.
2// Licensed under the MIT License.
3
4//! Implementation of submission and completion queues.
5
6use super::spec;
7use crate::driver::save_restore::CompletionQueueSavedState;
8use crate::driver::save_restore::SubmissionQueueSavedState;
9use crate::registers::DeviceRegisters;
10use inspect::Inspect;
11use safeatomic::AtomicSliceOps;
12use std::sync::atomic::AtomicU64;
13use std::sync::atomic::Ordering::Acquire;
14use std::sync::atomic::Ordering::Relaxed;
15use user_driver::DeviceBacking;
16use user_driver::memory::MemoryBlock;
17
18#[derive(Inspect)]
19pub(crate) struct SubmissionQueue {
20    sqid: u16,
21    head: u32,
22    tail: u32,
23    committed_tail: u32,
24    len: u32,
25    #[inspect(skip)]
26    mem: MemoryBlock,
27}
28
29#[derive(Debug)]
30pub(crate) struct QueueFull;
31
32impl SubmissionQueue {
33    pub fn new(sqid: u16, len: u16, mem: MemoryBlock) -> Self {
34        tracing::debug!(sqid, len, pfns = ?mem.pfns(), "new submission queue");
35
36        Self {
37            sqid,
38            head: 0,
39            tail: 0,
40            committed_tail: 0,
41            len: len.into(),
42            mem,
43        }
44    }
45
46    pub fn id(&self) -> u16 {
47        self.sqid
48    }
49
50    pub fn update_head(&mut self, head: u16) {
51        let head = head as u32;
52        assert!(head < self.len);
53        self.head = head;
54    }
55
56    pub fn is_full(&self) -> bool {
57        advance(self.tail, self.len) == self.head
58    }
59
60    pub fn write(&mut self, command: spec::Command) -> Result<(), QueueFull> {
61        let next_tail = advance(self.tail, self.len);
62        if next_tail == self.head {
63            return Err(QueueFull);
64        }
65        self.mem
66            .write_obj(self.tail as usize * size_of_val(&command), &command);
67        self.tail = next_tail;
68        Ok(())
69    }
70
71    pub fn commit<T: DeviceBacking>(&mut self, region: &DeviceRegisters<T>) {
72        if self.tail != self.committed_tail {
73            safe_intrinsics::store_fence();
74            region.doorbell(self.sqid, false, self.tail);
75            self.committed_tail = self.tail;
76        }
77    }
78
79    /// Saves queue data for servicing.
80    pub fn save(&self) -> SubmissionQueueSavedState {
81        SubmissionQueueSavedState {
82            sqid: self.sqid,
83            head: self.head,
84            tail: self.tail,
85            committed_tail: self.committed_tail,
86            len: self.len,
87        }
88    }
89
90    /// Restores queue data after servicing.
91    pub fn restore(
92        mem: MemoryBlock,
93        saved_state: &SubmissionQueueSavedState,
94    ) -> anyhow::Result<Self> {
95        let SubmissionQueueSavedState {
96            sqid,
97            head,
98            tail,
99            committed_tail,
100            len,
101        } = saved_state;
102        Ok(Self {
103            sqid: *sqid,
104            head: *head,
105            tail: *tail,
106            committed_tail: *committed_tail,
107            len: *len,
108            mem,
109        })
110    }
111}
112
113#[derive(Inspect)]
114pub(crate) struct CompletionQueue {
115    cqid: u16,
116    head: u32,
117    committed_head: u32,
118    /// Queue size in entries.
119    len: u32,
120    phase: bool,
121    #[inspect(skip)]
122    mem: MemoryBlock,
123}
124
125impl CompletionQueue {
126    pub fn new(cqid: u16, len: u16, mem: MemoryBlock) -> CompletionQueue {
127        tracing::debug!(cqid, len, pfns = ?mem.pfns(), "new completion queue");
128        Self {
129            cqid,
130            head: 0,
131            committed_head: 0,
132            len: len.into(),
133            phase: true,
134            mem,
135        }
136    }
137
138    pub fn _id(&self) -> u16 {
139        self.cqid
140    }
141
142    pub fn read(&mut self) -> Option<spec::Completion> {
143        let completion_mem = self.mem.as_slice()
144            [self.head as usize * size_of::<spec::Completion>()..][..size_of::<spec::Completion>()]
145            .as_atomic_slice::<AtomicU64>()
146            .unwrap();
147
148        // Check the phase bit, using an acquire read to ensure the rest of the
149        // completion is read with or after the phase bit.
150        let high = completion_mem[1].load(Acquire);
151        let status = spec::CompletionStatus::from((high >> 48) as u16);
152        if status.phase() != self.phase {
153            return None;
154        }
155        let low = completion_mem[0].load(Relaxed);
156        let completion: spec::Completion = zerocopy::transmute!([low, high]);
157        self.head += 1;
158        if self.head == self.len {
159            self.head = 0;
160            self.phase = !self.phase;
161        }
162        Some(completion)
163    }
164
165    pub fn commit<T: DeviceBacking>(&mut self, registers: &DeviceRegisters<T>) {
166        if self.head != self.committed_head {
167            safe_intrinsics::store_fence();
168            registers.doorbell(self.cqid, true, self.head);
169            self.committed_head = self.head;
170        }
171    }
172
173    /// Saves queue data for servicing.
174    pub fn save(&self) -> CompletionQueueSavedState {
175        CompletionQueueSavedState {
176            cqid: self.cqid,
177            head: self.head,
178            committed_head: self.committed_head,
179            len: self.len,
180            phase: self.phase,
181        }
182    }
183
184    /// Restores queue data after servicing.
185    pub fn restore(
186        mem: MemoryBlock,
187        saved_state: &CompletionQueueSavedState,
188    ) -> anyhow::Result<Self> {
189        let CompletionQueueSavedState {
190            cqid,
191            head,
192            committed_head,
193            len,
194            phase,
195        } = saved_state;
196
197        Ok(Self {
198            cqid: *cqid,
199            head: *head,
200            committed_head: *committed_head,
201            len: *len,
202            phase: *phase,
203            mem,
204        })
205    }
206}
207
208fn advance(n: u32, l: u32) -> u32 {
209    if n + 1 < l { n + 1 } else { 0 }
210}
nvme_driver/queues.rs

nvme_driver/
queues.rs
