Skip to main content

nvme/workers/
admin.rs

1// Copyright (c) Microsoft Corporation.
2// Licensed under the MIT License.
3
4//! Admin queue handler.
5
6use super::IoQueueEntrySizes;
7use super::MAX_DATA_TRANSFER_SIZE;
8use super::io::IoHandler;
9use super::io::IoState;
10use crate::BROADCAST_NSID;
11use crate::DOORBELL_STRIDE_BITS;
12use crate::MAX_NSID;
13use crate::MAX_QES;
14use crate::NVME_VERSION;
15use crate::PAGE_MASK;
16use crate::PAGE_SIZE;
17use crate::VENDOR_ID;
18use crate::error::CommandResult;
19use crate::error::NvmeError;
20use crate::namespace::Namespace;
21use crate::prp::PrpRange;
22use crate::queue::CompletionQueue;
23use crate::queue::DoorbellMemory;
24use crate::queue::QueueError;
25use crate::queue::SubmissionQueue;
26use crate::spec;
27use disk_backend::Disk;
28use futures::FutureExt;
29use futures::SinkExt;
30use futures::StreamExt;
31use futures_concurrency::future::Race;
32use guestmem::GuestMemory;
33use guid::Guid;
34use inspect::Inspect;
35use pal_async::task::Spawn;
36use pal_async::task::Task;
37use parking_lot::Mutex;
38use parking_lot::RwLock;
39use std::collections::BTreeMap;
40use std::collections::btree_map;
41use std::future::pending;
42use std::future::poll_fn;
43use std::io::Cursor;
44use std::io::Write;
45use std::sync::Arc;
46use task_control::AsyncRun;
47use task_control::Cancelled;
48use task_control::InspectTask;
49use task_control::StopTask;
50use task_control::TaskControl;
51use thiserror::Error;
52use vmcore::interrupt::Interrupt;
53use vmcore::vm_task::VmTaskDriver;
54use vmcore::vm_task::VmTaskDriverSource;
55use zerocopy::FromBytes;
56use zerocopy::FromZeros;
57use zerocopy::IntoBytes;
58
59const IOSQES: u8 = 6;
60const IOCQES: u8 = 4;
61const MAX_ASYNC_EVENT_REQUESTS: u8 = 4; // minimum recommended by spec
62const ERROR_LOG_PAGE_ENTRIES: u8 = 1;
63
64/// Panic message for an admin CQ write that finds the queue full.
65///
66/// Every admin CQ write has a free slot: `next_event` waits for one before it
67/// takes an event, and only the admin task writes the admin CQ. A full queue
68/// would mean dropping the completion, which NVMe Base 2.1 section 3.3.1.2.1
69/// does not allow ("the controller shall not post status to that Completion
70/// Queue until slots become available").
71const ADMIN_CQ_SLOT_RESERVED: &str =
72    "admin completion queue full despite the slot next_event reserved";
73
74#[derive(Inspect)]
75pub struct AdminConfig {
76    #[inspect(skip)]
77    pub driver_source: VmTaskDriverSource,
78    #[inspect(skip)]
79    pub mem: GuestMemory,
80    #[inspect(skip)]
81    pub interrupts: Vec<Interrupt>,
82    #[inspect(skip)]
83    pub doorbells: Arc<RwLock<DoorbellMemory>>,
84    #[inspect(display)]
85    pub subsystem_id: Guid,
86    pub max_sqs: u16,
87    pub max_cqs: u16,
88    pub qe_sizes: Arc<Mutex<IoQueueEntrySizes>>,
89}
90
91#[derive(Inspect)]
92pub struct AdminHandler {
93    driver: VmTaskDriver,
94    config: AdminConfig,
95    #[inspect(iter_by_key)]
96    namespaces: BTreeMap<u32, Arc<Namespace>>,
97}
98
99#[derive(Inspect)]
100pub struct AdminState {
101    admin_sq: SubmissionQueue,
102    admin_cq: CompletionQueue,
103    #[inspect(with = "|x| inspect::iter_by_index(x).map_key(|x| x + 1)")]
104    io_sqs: Vec<Option<IoSq>>,
105    #[inspect(with = "|x| inspect::iter_by_index(x).map_key(|x| x + 1)")]
106    io_cqs: Vec<IoCq>,
107    #[inspect(skip)]
108    sq_delete_response: mesh::Receiver<u16>,
109    #[inspect(iter_by_index)]
110    asynchronous_event_requests: Vec<u16>,
111    #[inspect(
112        rename = "namespaces",
113        with = "|x| inspect::iter_by_key(x.iter().map(|v| (v, ChangedNamespace { changed: true })))"
114    )]
115    changed_namespaces: Vec<u32>,
116    notified_changed_namespaces: bool,
117    /// Asynchronous Event Configuration (Set Features FID 0x0B / CDW11),
118    /// stored verbatim and echoed back via Get Features. The NVMe Base
119    /// specification lists this Feature as mandatory for I/O controllers
120    /// (Base 2.0c section 3.1.2.1.1 / Base 2.3 section 3.1.3.6, "Feature
121    /// Support Requirements"). Each bit in CDW11 enables a class of
122    /// asynchronous event notification (refer to
123    /// [`spec::Cdw11FeatureAsyncEventConfig`]). Initiators that strictly
124    /// follow the spec may refuse to allocate any Asynchronous Event
125    /// Request resources when the Set Features command for this Feature
126    /// is rejected, which breaks AEN delivery (including the
127    /// changed-namespace AEN that drives namespace hot-add notification).
128    ///
129    /// Defaults to all bits set so that any AEN class the controller
130    /// chooses to fire is enabled until the host explicitly narrows the
131    /// mask via Set Features.
132    async_event_config: u32,
133    #[inspect(skip)]
134    recv_changed_namespace: futures::channel::mpsc::Receiver<u32>,
135    #[inspect(skip)]
136    send_changed_namespace: futures::channel::mpsc::Sender<u32>,
137    #[inspect(skip)]
138    poll_namespace_change: BTreeMap<u32, Task<()>>,
139    features: FeatureState,
140}
141
142/// Stored configuration for the mandatory Set/Get Features that the controller
143/// accepts and echoes back but does not otherwise act upon. Storing and echoing
144/// these values satisfies the NVMe Base 2.3 §3.1.3.6 (Figure 32) requirement
145/// that an I/O controller accept every mandatory feature.
146#[derive(Inspect)]
147struct FeatureState {
148    #[inspect(hex)]
149    arbitration: u32,
150    #[inspect(hex)]
151    power_management: u32,
152    #[inspect(hex)]
153    error_recovery: u32,
154    #[inspect(hex)]
155    interrupt_coalescing: u32,
156    /// Coalescing Disable bit per interrupt vector, indexed by interrupt vector.
157    #[inspect(iter_by_index)]
158    interrupt_vector_coalescing_disable: Vec<bool>,
159}
160
161impl FeatureState {
162    fn new(num_interrupt_vectors: usize) -> Self {
163        Self {
164            arbitration: 0,
165            power_management: 0,
166            error_recovery: 0,
167            interrupt_coalescing: 0,
168            interrupt_vector_coalescing_disable: vec![false; num_interrupt_vectors],
169        }
170    }
171}
172
173/// The Temperature Threshold value reported for the requested threshold type.
174///
175/// The emulator models no temperature sensors, so thresholds never trip and no
176/// state needs to be stored: Set Features is accepted and ignored, and Get
177/// Features always reports the reset defaults — an over-temperature threshold
178/// of the maximum (effectively disabled) and an under-temperature threshold of
179/// zero.
180fn default_temperature_threshold(thsel: u8) -> u16 {
181    if thsel == 0 { 0xffff } else { 0x0000 }
182}
183
184#[derive(Inspect)]
185struct ChangedNamespace {
186    changed: bool,
187}
188
189#[derive(Inspect)]
190struct IoSq {
191    pending_delete_cid: Option<u16>,
192    sq_idx: usize,
193    cqid: u16,
194}
195
196#[derive(Inspect)]
197struct IoCq {
198    driver: VmTaskDriver,
199    #[inspect(flatten)]
200    task: TaskControl<IoHandler, IoState>,
201}
202
203impl AdminState {
204    pub fn new(handler: &AdminHandler, asq: u64, asqs: u16, acq: u64, acqs: u16) -> Self {
205        // Start polling for namespace changes. Use a bounded channel to avoid
206        // unbounded memory allocation when the queue is stuck.
207        #[expect(clippy::disallowed_methods)] // TODO
208        let (send_changed_namespace, recv_changed_namespace) = futures::channel::mpsc::channel(256);
209        let poll_namespace_change = handler
210            .namespaces
211            .iter()
212            .map(|(&nsid, namespace)| {
213                (
214                    nsid,
215                    spawn_namespace_notifier(
216                        &handler.driver,
217                        nsid,
218                        namespace.clone(),
219                        send_changed_namespace.clone(),
220                    ),
221                )
222            })
223            .collect();
224
225        let admin_cq = CompletionQueue::new(
226            handler.config.doorbells.clone(),
227            1,
228            handler.config.mem.clone(),
229            Some(handler.config.interrupts[0].clone()),
230            acq,
231            acqs,
232        );
233        let mut state = Self {
234            admin_sq: SubmissionQueue::new(&admin_cq, 0, asq, asqs),
235            admin_cq,
236            io_sqs: Vec::new(),
237            io_cqs: Vec::new(),
238            sq_delete_response: Default::default(),
239            asynchronous_event_requests: Vec::new(),
240            changed_namespaces: Vec::new(),
241            notified_changed_namespaces: false,
242            async_event_config: u32::MAX,
243            recv_changed_namespace,
244            send_changed_namespace,
245            poll_namespace_change,
246            features: FeatureState::new(handler.config.interrupts.len()),
247        };
248        state.set_max_queues(handler, handler.config.max_sqs, handler.config.max_cqs);
249        state
250    }
251
252    /// Stops all submission queues and drains them of any pending IO.
253    ///
254    /// This future may be dropped and reissued.
255    pub async fn drain(&mut self) {
256        for cq in &mut self.io_cqs {
257            cq.task.stop().await;
258            if let Some(state) = cq.task.state_mut() {
259                state.drain().await;
260                cq.task.remove();
261            }
262        }
263    }
264
265    /// Caller must ensure that no queues are active.
266    fn set_max_queues(&mut self, handler: &AdminHandler, num_sqs: u16, num_cqs: u16) {
267        self.io_sqs.truncate(num_sqs.into());
268        self.io_sqs.resize_with(num_sqs.into(), || None);
269        self.io_cqs.resize_with(num_cqs.into(), || {
270            // This driver doesn't explicitly do any IO (that's handled by
271            // the storage backends), so the target VP doesn't matter. But
272            // set it anyway as a hint to the backend that this queue needs
273            // its own thread.
274            let driver = handler
275                .config
276                .driver_source
277                .builder()
278                .run_on_target(false)
279                .target_vp(0)
280                .build("nvme");
281
282            IoCq {
283                driver,
284                task: TaskControl::new(IoHandler::new(
285                    handler.config.mem.clone(),
286                    self.sq_delete_response.sender(),
287                )),
288            }
289        });
290    }
291
292    fn add_changed_namespace(&mut self, nsid: u32) {
293        if let Err(i) = self.changed_namespaces.binary_search(&nsid) {
294            self.changed_namespaces.insert(i, nsid);
295        }
296    }
297
298    async fn add_namespace(
299        &mut self,
300        driver: &VmTaskDriver,
301        nsid: u32,
302        namespace: &Arc<Namespace>,
303    ) {
304        // Update the IO queues.
305        for cq in &mut self.io_cqs {
306            let io_running = cq.task.stop().await;
307            if let Some(io_state) = cq.task.state_mut() {
308                io_state.add_namespace(nsid, namespace.clone());
309            }
310            if io_running {
311                cq.task.start();
312            }
313        }
314
315        // Start polling.
316        let old = self.poll_namespace_change.insert(
317            nsid,
318            spawn_namespace_notifier(
319                driver,
320                nsid,
321                namespace.clone(),
322                self.send_changed_namespace.clone(),
323            ),
324        );
325        assert!(old.is_none());
326
327        // Notify the guest driver of the change.
328        self.add_changed_namespace(nsid);
329    }
330
331    async fn remove_namespace(&mut self, nsid: u32) {
332        // Update the IO queues.
333        for cq in &mut self.io_cqs {
334            let io_running = cq.task.stop().await;
335            if let Some(io_state) = cq.task.state_mut() {
336                io_state.remove_namespace(nsid);
337            }
338            if io_running {
339                cq.task.start();
340            }
341        }
342
343        // Stop polling.
344        self.poll_namespace_change
345            .remove(&nsid)
346            .unwrap()
347            .cancel()
348            .await;
349
350        // Notify the guest driver of the change.
351        self.add_changed_namespace(nsid);
352    }
353}
354
355fn spawn_namespace_notifier(
356    driver: &VmTaskDriver,
357    nsid: u32,
358    namespace: Arc<Namespace>,
359    mut send_changed_namespace: futures::channel::mpsc::Sender<u32>,
360) -> Task<()> {
361    driver.spawn("wait_resize", async move {
362        let mut counter = None;
363        loop {
364            counter = Some(namespace.wait_change(counter).await);
365            tracing::info!(nsid, "namespace changed");
366            if send_changed_namespace.send(nsid).await.is_err() {
367                break;
368            }
369        }
370    })
371}
372
373#[derive(Debug, Error)]
374#[error("invalid queue identifier {qid}")]
375struct InvalidQueueIdentifier {
376    qid: u16,
377    #[source]
378    reason: InvalidQueueIdentifierReason,
379}
380
381#[derive(Debug, Error)]
382enum InvalidQueueIdentifierReason {
383    #[error("queue id is out of bounds")]
384    Oob,
385    #[error("queue id is in use")]
386    InUse,
387    #[error("queue id is not in use")]
388    NotInUse,
389}
390
391impl From<InvalidQueueIdentifier> for NvmeError {
392    fn from(err: InvalidQueueIdentifier) -> Self {
393        Self::new(spec::Status::INVALID_QUEUE_IDENTIFIER, err)
394    }
395}
396
397enum Event {
398    Command(Result<spec::Command, QueueError>),
399    SqDeleteComplete(u16),
400    NamespaceChange(u32),
401}
402
403/// Error returned when a namespace cannot be added.
404#[derive(Debug, Error)]
405pub enum AddNamespaceError {
406    /// A namespace with this ID already exists.
407    #[error("namespace id conflict for {0}")]
408    Conflict(u32),
409    /// The namespace ID is outside the valid range supported by the
410    /// subsystem (see the `NN` field of Identify Controller).
411    #[error("namespace id {0} is out of range (must be 1..={MAX_NSID})")]
412    OutOfRange(u32),
413}
414
415impl AdminHandler {
416    pub fn new(driver: VmTaskDriver, config: AdminConfig) -> Self {
417        Self {
418            driver,
419            config,
420            namespaces: Default::default(),
421        }
422    }
423
424    pub async fn add_namespace(
425        &mut self,
426        state: Option<&mut AdminState>,
427        nsid: u32,
428        disk: Disk,
429    ) -> Result<(), AddNamespaceError> {
430        if nsid == 0 || nsid > MAX_NSID {
431            return Err(AddNamespaceError::OutOfRange(nsid));
432        }
433        let namespace = &*match self.namespaces.entry(nsid) {
434            btree_map::Entry::Vacant(entry) => entry.insert(Arc::new(Namespace::new(
435                self.config.mem.clone(),
436                nsid,
437                disk,
438            ))),
439            btree_map::Entry::Occupied(_) => return Err(AddNamespaceError::Conflict(nsid)),
440        };
441
442        if let Some(state) = state {
443            state.add_namespace(&self.driver, nsid, namespace).await;
444        }
445
446        Ok(())
447    }
448
449    pub async fn remove_namespace(&mut self, state: Option<&mut AdminState>, nsid: u32) -> bool {
450        if self.namespaces.remove(&nsid).is_none() {
451            return false;
452        }
453
454        if let Some(state) = state {
455            state.remove_namespace(nsid).await;
456        }
457
458        true
459    }
460
461    async fn next_event(&mut self, state: &mut AdminState) -> Result<Event, QueueError> {
462        let event = loop {
463            // Wait for there to be room for a completion for the next
464            // command or the completed sq deletion.
465            poll_fn(|cx| state.admin_cq.poll_ready(cx)).await?;
466
467            // Fire the changed-namespace AEN only when the host has
468            // enabled the Attached Namespace Attribute Notices class via
469            // Set Features 0Bh (NVMe Base 2.0c section 5.21.1.11 /
470            // Base 2.3 section 5.2.26.1.5, CDW11 bit 8). Per spec,
471            // "If this bit is cleared to '0', then the controller shall
472            // not send the Attached Namespace Attribute Changed
473            // asynchronous event to the host." The mask defaults to all
474            // bits set, so this only suppresses delivery when the host
475            // has explicitly opted out via Set Features.
476            let ns_aen_enabled = spec::Cdw11FeatureAsyncEventConfig::from(state.async_event_config)
477                .namespace_attribute_notices();
478
479            if !state.changed_namespaces.is_empty()
480                && !state.notified_changed_namespaces
481                && ns_aen_enabled
482            {
483                if let Some(cid) = state.asynchronous_event_requests.pop() {
484                    let posted = state.admin_cq.write(
485                        spec::Completion {
486                            dw0: spec::AsynchronousEventRequestDw0::new()
487                                .with_event_type(spec::AsynchronousEventType::NOTICE.0)
488                                .with_log_page_identifier(spec::LogPageIdentifier::CHANGED_NAMESPACE_LIST.0)
489                                .with_information(spec::AsynchronousEventInformationNotice::NAMESPACE_ATTRIBUTE_CHANGED.0)
490                                .into(),
491                            dw1: 0,
492                            sqhd: state.admin_sq.sqhd(),
493                            sqid: 0,
494                            cid,
495                            status: spec::CompletionStatus::new(),
496                        },
497                    )?;
498                    assert!(posted, "{ADMIN_CQ_SLOT_RESERVED}");
499
500                    state.notified_changed_namespaces = true;
501                    continue;
502                }
503            }
504
505            let next_command = poll_fn(|cx| state.admin_sq.poll_next(cx)).map(Event::Command);
506            let sq_delete_complete = async {
507                let Some(sqid) = state.sq_delete_response.next().await else {
508                    pending().await
509                };
510                Event::SqDeleteComplete(sqid)
511            };
512            let changed_namespace = async {
513                let Some(nsid) = state.recv_changed_namespace.next().await else {
514                    pending().await
515                };
516                Event::NamespaceChange(nsid)
517            };
518
519            break (next_command, sq_delete_complete, changed_namespace)
520                .race()
521                .await;
522        };
523        Ok(event)
524    }
525
526    async fn process_event(
527        &mut self,
528        state: &mut AdminState,
529        event: Result<Event, QueueError>,
530    ) -> Result<(), QueueError> {
531        let (cid, result) = match event? {
532            Event::Command(command) => {
533                let command = command?;
534                let opcode = spec::AdminOpcode(command.cdw0.opcode());
535
536                tracing::trace!(?opcode, ?command, "command");
537
538                let result = match opcode {
539                    // Identify Controller reports no fused operations (FUSES is
540                    // 0), and admin commands over PCIe always use PRPs (NVMe
541                    // Base 2.1 Figure 91), so any other FUSE or PSDT value is
542                    // unsupported.
543                    _ if command.cdw0.fuse() != 0 || command.cdw0.psdt() != 0 => Err(
544                        NvmeError::do_not_retry(spec::Status::INVALID_FIELD_IN_COMMAND),
545                    ),
546                    spec::AdminOpcode::IDENTIFY => self
547                        .handle_identify(state, &command)
548                        .map(|()| Some(Default::default())),
549                    spec::AdminOpcode::GET_FEATURES => {
550                        self.handle_get_features(state, &command).await.map(Some)
551                    }
552                    spec::AdminOpcode::SET_FEATURES => {
553                        self.handle_set_features(state, &command).map(Some)
554                    }
555                    spec::AdminOpcode::CREATE_IO_COMPLETION_QUEUE => self
556                        .handle_create_io_completion_queue(state, &command)
557                        .map(|()| Some(Default::default())),
558                    spec::AdminOpcode::CREATE_IO_SUBMISSION_QUEUE => self
559                        .handle_create_io_submission_queue(state, &command)
560                        .await
561                        .map(|()| Some(Default::default())),
562                    spec::AdminOpcode::DELETE_IO_COMPLETION_QUEUE => self
563                        .handle_delete_io_completion_queue(state, &command)
564                        .await
565                        .map(|()| Some(Default::default())),
566                    spec::AdminOpcode::DELETE_IO_SUBMISSION_QUEUE => {
567                        self.handle_delete_io_submission_queue(state, &command)
568                            .await
569                    }
570                    spec::AdminOpcode::ASYNCHRONOUS_EVENT_REQUEST => {
571                        self.handle_asynchronous_event_request(state, &command)
572                    }
573                    spec::AdminOpcode::ABORT => self.handle_abort(),
574                    spec::AdminOpcode::GET_LOG_PAGE => self
575                        .handle_get_log_page(state, &command)
576                        .map(|()| Some(Default::default())),
577                    spec::AdminOpcode::DOORBELL_BUFFER_CONFIG
578                        if self.supports_shadow_doorbells(state) =>
579                    {
580                        self.handle_doorbell_buffer_config(state, &command)
581                            .await
582                            .map(|()| Some(Default::default()))
583                    }
584                    opcode => {
585                        tracelimit::warn_ratelimited!(?opcode, "unsupported opcode");
586                        Err(spec::Status::INVALID_COMMAND_OPCODE.into())
587                    }
588                };
589
590                let result = match result {
591                    Ok(Some(cr)) => cr,
592                    Ok(None) => return Ok(()),
593                    Err(err) => {
594                        tracelimit::warn_ratelimited!(
595                            error = &err as &dyn std::error::Error,
596                            cid = command.cdw0.cid(),
597                            ?opcode,
598                            "command error"
599                        );
600                        err.into()
601                    }
602                };
603
604                (command.cdw0.cid(), result)
605            }
606            Event::SqDeleteComplete(sqid) => {
607                let sq = state.io_sqs[sqid as usize - 1].take().unwrap();
608                let cid = sq.pending_delete_cid.unwrap();
609                (cid, Default::default())
610            }
611            Event::NamespaceChange(nsid) => {
612                state.add_changed_namespace(nsid);
613                return Ok(());
614            }
615        };
616
617        let completion = spec::Completion {
618            dw0: result.dw[0],
619            dw1: result.dw[1],
620            sqid: 0,
621            sqhd: state.admin_sq.sqhd(),
622            status: result.completion_status(),
623            cid,
624        };
625
626        let posted = state.admin_cq.write(completion)?;
627        assert!(posted, "{ADMIN_CQ_SLOT_RESERVED}");
628        Ok(())
629    }
630
631    fn handle_identify(
632        &mut self,
633        state: &AdminState,
634        command: &spec::Command,
635    ) -> Result<(), NvmeError> {
636        let cdw10: spec::Cdw10Identify = command.cdw10.into();
637        // All identify results are 4096 bytes.
638        let mut buf = [0u64; 512];
639        let buf = buf.as_mut_bytes();
640        match spec::Cns(cdw10.cns()) {
641            spec::Cns::CONTROLLER => {
642                let id = spec::IdentifyController::mut_from_prefix(buf).unwrap().0; // TODO: zerocopy: from-prefix (mut_from_prefix): use-rest-of-range (https://github.com/microsoft/openvmm/issues/759)
643                *id = self.identify_controller(state);
644
645                write!(
646                    Cursor::new(&mut id.subnqn[..]),
647                    "nqn.2014-08.org.nvmexpress:uuid:{}",
648                    self.config.subsystem_id
649                )
650                .unwrap();
651            }
652            spec::Cns::ACTIVE_NAMESPACES => {
653                if command.nsid >= 0xfffffffe {
654                    return Err(spec::Status::INVALID_NAMESPACE_OR_FORMAT.into());
655                }
656                let nsids = <[u32]>::mut_from_bytes(buf).unwrap();
657                for (ns, nsid) in self
658                    .namespaces
659                    .keys()
660                    .filter(|&ns| *ns > command.nsid)
661                    .zip(nsids)
662                {
663                    *nsid = *ns;
664                }
665            }
666            spec::Cns::NAMESPACE => {
667                if command.nsid == 0 || command.nsid > MAX_NSID {
668                    return Err(spec::Status::INVALID_NAMESPACE_OR_FORMAT.into());
669                }
670                if let Some(ns) = self.namespaces.get(&command.nsid) {
671                    ns.identify(buf);
672                } else {
673                    // Valid but inactive namespace: return a zero-filled
674                    // structure (the buffer is already zeroed).
675                    tracing::trace!(nsid = command.nsid, "inactive namespace id");
676                }
677            }
678            spec::Cns::DESCRIPTOR_NAMESPACE => {
679                if command.nsid == 0 || command.nsid > MAX_NSID {
680                    return Err(spec::Status::INVALID_NAMESPACE_OR_FORMAT.into());
681                }
682                if let Some(ns) = self.namespaces.get(&command.nsid) {
683                    ns.namespace_id_descriptor(buf);
684                } else {
685                    // Valid but inactive namespace: return a zero-filled
686                    // structure (the buffer is already zeroed).
687                    tracing::trace!(nsid = command.nsid, "inactive namespace id");
688                }
689            }
690            spec::Cns::SPECIFIC_CONTROLLER_IO_COMMAND_SET => {
691                // CSI is in Command Dword 11, bits 31:24. Only the NVM command
692                // set (CSI 0h) is supported, and it defines no I/O Command Set
693                // specific Identify Controller data structure, so return a
694                // zero-filled structure per NVMe Base 2.3 section 5.2.13.2.6.
695                // Any other command set is unsupported.
696                let csi = (command.cdw11 >> 24) as u8;
697                if csi != 0 {
698                    return Err(spec::Status::INVALID_FIELD_IN_COMMAND.into());
699                }
700                // The buffer is already zero-filled; fall through to write it
701                // back to the host.
702            }
703            cns => {
704                tracelimit::warn_ratelimited!(?cns, "unsupported cns");
705                return Err(spec::Status::INVALID_FIELD_IN_COMMAND.into());
706            }
707        };
708        PrpRange::parse(&self.config.mem, buf.len(), command.dptr)?.write(&self.config.mem, buf)?;
709        Ok(())
710    }
711
712    fn identify_controller(&self, state: &AdminState) -> spec::IdentifyController {
713        spec::IdentifyController {
714            vid: VENDOR_ID,
715            ssvid: VENDOR_ID,
716            mdts: (MAX_DATA_TRANSFER_SIZE / PAGE_SIZE).trailing_zeros() as u8,
717            ver: NVME_VERSION,
718            rtd3r: 400000,
719            rtd3e: 400000,
720            sqes: spec::QueueEntrySize::new()
721                .with_min(IOSQES)
722                .with_max(IOSQES),
723            cqes: spec::QueueEntrySize::new()
724                .with_min(IOCQES)
725                .with_max(IOCQES),
726            frmw: spec::FirmwareUpdates::new().with_ffsro(true).with_nofs(1),
727            nn: MAX_NSID,
728            ieee: [0x74, 0xe2, 0x8c], // Microsoft
729            fr: (*b"v1.00000").into(),
730            mn: (*b"MSFT NVMe Accelerator v1.0              ").into(),
731            sn: (*b"SN: 000001          ").into(),
732            aerl: MAX_ASYNC_EVENT_REQUESTS - 1,
733            elpe: ERROR_LOG_PAGE_ENTRIES - 1,
734            oaes: spec::Oaes::new().with_namespace_attribute(true),
735            oncs: spec::Oncs::new()
736                .with_dataset_management(true)
737                // Namespaces still have to opt in individually via `rescap`.
738                .with_reservations(true),
739            vwc: spec::VolatileWriteCache::new()
740                .with_present(true)
741                .with_broadcast_flush_behavior(spec::BroadcastFlushBehavior::NOT_SUPPORTED.0),
742            cntrltype: spec::ControllerType::IO_CONTROLLER,
743            oacs: spec::OptionalAdminCommandSupport::new()
744                .with_doorbell_buffer_config(self.supports_shadow_doorbells(state)),
745            ..FromZeros::new_zeroed()
746        }
747    }
748
749    fn handle_set_features(
750        &mut self,
751        state: &mut AdminState,
752        command: &spec::Command,
753    ) -> Result<CommandResult, NvmeError> {
754        let cdw10: spec::Cdw10SetFeatures = command.cdw10.into();
755        let mut dw = [0; 2];
756        // This controller does not support saving feature values across power
757        // cycles or resets (Identify Controller ONCS.SSFS is 0), so a request
758        // to save a feature value must be rejected rather than silently applied
759        // for the current power cycle only.
760        if cdw10.save() {
761            return Err(spec::Status::FEATURE_IDENTIFIER_NOT_SAVEABLE.into());
762        }
763        match spec::Feature(cdw10.fid()) {
764            spec::Feature::NUMBER_OF_QUEUES => {
765                if state.io_sqs.iter().any(|sq| sq.is_some())
766                    || state.io_cqs.iter().any(|cq| cq.task.has_state())
767                {
768                    return Err(spec::Status::COMMAND_SEQUENCE_ERROR.into());
769                }
770                let cdw11: spec::Cdw11FeatureNumberOfQueues = command.cdw11.into();
771                if cdw11.ncq_z() == u16::MAX || cdw11.nsq_z() == u16::MAX {
772                    return Err(spec::Status::INVALID_FIELD_IN_COMMAND.into());
773                }
774                let num_sqs = (cdw11.nsq_z() + 1).min(self.config.max_sqs);
775                let num_cqs = (cdw11.ncq_z() + 1).min(self.config.max_cqs);
776                state.set_max_queues(self, num_sqs, num_cqs);
777
778                dw[0] = spec::Cdw11FeatureNumberOfQueues::new()
779                    .with_ncq_z(num_cqs - 1)
780                    .with_nsq_z(num_sqs - 1)
781                    .into();
782            }
783            spec::Feature::VOLATILE_WRITE_CACHE => {
784                let cdw11 = spec::Cdw11FeatureVolatileWriteCache::from(command.cdw11);
785                if !cdw11.wce() {
786                    tracelimit::warn_ratelimited!(
787                        "ignoring unsupported attempt to disable write cache"
788                    );
789                }
790            }
791            spec::Feature::ASYNC_EVENT_CONFIG => {
792                // The Asynchronous Event Configuration feature is mandatory
793                // for I/O controllers per the NVMe Base specification's
794                // Feature Support Requirements table (Base 2.0c section
795                // 3.1.2.1.1 / Base 2.3 section 3.1.3.6). The host sets bits
796                // in CDW11 to enable each class of asynchronous event
797                // notification. We store the value verbatim; Get Features
798                // echoes it back, and the AEN dispatch loop consults the
799                // relevant bits before firing each notification class.
800                state.async_event_config = command.cdw11;
801            }
802            spec::Feature::ARBITRATION => {
803                state.features.arbitration = command.cdw11;
804            }
805            spec::Feature::POWER_MANAGEMENT => {
806                let cdw11 = spec::Cdw11FeaturePowerManagement::from(command.cdw11);
807                // Only power state 0 is supported (NPSS is 0 in the Identify
808                // Controller data structure, i.e. one power state).
809                if cdw11.ps() != 0 {
810                    return Err(spec::Status::INVALID_FIELD_IN_COMMAND.into());
811                }
812                state.features.power_management = command.cdw11;
813            }
814            spec::Feature::TEMPERATURE_THRESHOLD => {
815                let cdw11 = spec::Cdw11FeatureTemperatureThreshold::from(command.cdw11);
816                // Only over- and under-temperature thresholds are defined.
817                if cdw11.thsel() > 1 {
818                    return Err(spec::Status::INVALID_FIELD_IN_COMMAND.into());
819                }
820                // Accepted and ignored: the emulator models no temperature.
821            }
822            spec::Feature::ERROR_RECOVERY => {
823                state.features.error_recovery = command.cdw11;
824            }
825            spec::Feature::INTERRUPT_COALESCING => {
826                state.features.interrupt_coalescing = command.cdw11;
827            }
828            spec::Feature::INTERRUPT_VECTOR_CONFIG => {
829                let cdw11 = spec::Cdw11FeatureInterruptVectorConfig::from(command.cdw11);
830                let cd = state
831                    .features
832                    .interrupt_vector_coalescing_disable
833                    .get_mut(cdw11.iv() as usize)
834                    .ok_or(spec::Status::INVALID_FIELD_IN_COMMAND)?;
835                *cd = cdw11.cd();
836            }
837            feature => {
838                tracelimit::warn_ratelimited!(?feature, "unsupported feature");
839                return Err(spec::Status::INVALID_FIELD_IN_COMMAND.into());
840            }
841        }
842        Ok(CommandResult::new(spec::Status::SUCCESS, dw))
843    }
844
845    async fn handle_get_features(
846        &mut self,
847        state: &mut AdminState,
848        command: &spec::Command,
849    ) -> Result<CommandResult, NvmeError> {
850        let cdw10: spec::Cdw10GetFeatures = command.cdw10.into();
851        let mut dw = [0; 2];
852
853        // Only the current value (Select 000b) is supported; the controller
854        // does not support default/saved/supported-capabilities selects
855        // (Identify Controller ONCS.SSFS is 0).
856        if cdw10.sel() != 0 {
857            return Err(spec::Status::INVALID_FIELD_IN_COMMAND.into());
858        }
859        match spec::Feature(cdw10.fid()) {
860            spec::Feature::NUMBER_OF_QUEUES => {
861                let num_cqs = state.io_cqs.len();
862                let num_sqs = state.io_sqs.len();
863                dw[0] = spec::Cdw11FeatureNumberOfQueues::new()
864                    .with_ncq_z((num_cqs - 1) as u16)
865                    .with_nsq_z((num_sqs - 1) as u16)
866                    .into();
867            }
868            spec::Feature::VOLATILE_WRITE_CACHE => {
869                // Write cache is always enabled.
870                dw[0] = spec::Cdw11FeatureVolatileWriteCache::new()
871                    .with_wce(true)
872                    .into();
873            }
874            spec::Feature::ASYNC_EVENT_CONFIG => {
875                // Echo back the most recently configured mask. The cache
876                // is initialized to all bits set (refer to
877                // [`AdminState::new`]) so that a host which never issues
878                // Set Features 0Bh still sees every notification class
879                // reported as enabled, preserving the pre-existing
880                // behavior of unconditional AEN delivery.
881                dw[0] = state.async_event_config;
882            }
883            spec::Feature::NVM_RESERVATION_PERSISTENCE => {
884                // FFFFFFFFh gets Invalid Field in Command (NVMe Base 2.1
885                // section 5.1.25.1.30), which overrides the Invalid Namespace
886                // or Format section 4.4 gives other namespace-scope features.
887                // Other NSIDs follow Figure 92 and section 3.2.1.5.
888                if command.nsid == BROADCAST_NSID {
889                    return Err(spec::Status::INVALID_FIELD_IN_COMMAND.into());
890                }
891                if command.nsid == 0 || command.nsid > MAX_NSID {
892                    return Err(spec::Status::INVALID_NAMESPACE_OR_FORMAT.into());
893                }
894                let namespace = self
895                    .namespaces
896                    .get(&command.nsid)
897                    .ok_or(spec::Status::INVALID_FIELD_IN_COMMAND)?;
898
899                return namespace.get_feature(command).await;
900            }
901            spec::Feature::ARBITRATION => {
902                dw[0] = state.features.arbitration;
903            }
904            spec::Feature::POWER_MANAGEMENT => {
905                dw[0] = state.features.power_management;
906            }
907            spec::Feature::TEMPERATURE_THRESHOLD => {
908                let cdw11 = spec::Cdw11FeatureTemperatureThreshold::from(command.cdw11);
909                if cdw11.thsel() > 1 {
910                    return Err(spec::Status::INVALID_FIELD_IN_COMMAND.into());
911                }
912                dw[0] = spec::Cdw11FeatureTemperatureThreshold::new()
913                    .with_tmpth(default_temperature_threshold(cdw11.thsel()))
914                    .with_tmpsel(cdw11.tmpsel())
915                    .with_thsel(cdw11.thsel())
916                    .into();
917            }
918            spec::Feature::ERROR_RECOVERY => {
919                dw[0] = state.features.error_recovery;
920            }
921            spec::Feature::INTERRUPT_COALESCING => {
922                dw[0] = state.features.interrupt_coalescing;
923            }
924            spec::Feature::INTERRUPT_VECTOR_CONFIG => {
925                let cdw11 = spec::Cdw11FeatureInterruptVectorConfig::from(command.cdw11);
926                let cd = *state
927                    .features
928                    .interrupt_vector_coalescing_disable
929                    .get(cdw11.iv() as usize)
930                    .ok_or(spec::Status::INVALID_FIELD_IN_COMMAND)?;
931                dw[0] = spec::Cdw11FeatureInterruptVectorConfig::new()
932                    .with_iv(cdw11.iv())
933                    .with_cd(cd)
934                    .into();
935            }
936            feature => {
937                tracelimit::warn_ratelimited!(?feature, "unsupported feature");
938                return Err(spec::Status::INVALID_FIELD_IN_COMMAND.into());
939            }
940        }
941        Ok(CommandResult::new(spec::Status::SUCCESS, dw))
942    }
943
944    fn handle_create_io_completion_queue(
945        &mut self,
946        state: &mut AdminState,
947        command: &spec::Command,
948    ) -> Result<(), NvmeError> {
949        let cdw10: spec::Cdw10CreateIoQueue = command.cdw10.into();
950        let cdw11: spec::Cdw11CreateIoCompletionQueue = command.cdw11.into();
951        if !cdw11.pc() {
952            return Err(spec::Status::INVALID_FIELD_IN_COMMAND.into());
953        }
954        let cqid = cdw10.qid();
955        let cq = state
956            .io_cqs
957            .get_mut((cqid as usize).wrapping_sub(1))
958            .ok_or(InvalidQueueIdentifier {
959                qid: cqid,
960                reason: InvalidQueueIdentifierReason::Oob,
961            })?;
962
963        if cq.task.has_state() {
964            return Err(InvalidQueueIdentifier {
965                qid: cqid,
966                reason: InvalidQueueIdentifierReason::InUse,
967            }
968            .into());
969        }
970
971        let interrupt = if cdw11.ien() {
972            let iv = cdw11.iv();
973            if iv as usize >= self.config.interrupts.len() {
974                return Err(spec::Status::INVALID_INTERRUPT_VECTOR.into());
975            };
976            Some(iv)
977        } else {
978            None
979        };
980        let gpa = command.dptr[0] & PAGE_MASK;
981        let len0 = cdw10.qsize_z();
982        if len0 == 0 || len0 >= MAX_QES || self.config.qe_sizes.lock().cqe_bits != IOCQES {
983            return Err(spec::Status::INVALID_QUEUE_SIZE.into());
984        }
985
986        let interrupt = interrupt.map(|iv| self.config.interrupts[iv as usize].clone());
987        let namespaces = self.namespaces.clone();
988
989        let state = IoState::new(
990            &self.config.mem,
991            self.config.doorbells.clone(),
992            gpa,
993            len0 + 1,
994            cqid,
995            interrupt,
996            namespaces,
997        );
998
999        cq.task.insert(&cq.driver, "nvme-io", state);
1000        cq.task.start();
1001        Ok(())
1002    }
1003
1004    async fn handle_create_io_submission_queue(
1005        &mut self,
1006        state: &mut AdminState,
1007        command: &spec::Command,
1008    ) -> Result<(), NvmeError> {
1009        let cdw10: spec::Cdw10CreateIoQueue = command.cdw10.into();
1010        let cdw11: spec::Cdw11CreateIoSubmissionQueue = command.cdw11.into();
1011        if !cdw11.pc() {
1012            return Err(spec::Status::INVALID_FIELD_IN_COMMAND.into());
1013        }
1014        let sqid = cdw10.qid();
1015        let sq = state
1016            .io_sqs
1017            .get_mut((sqid as usize).wrapping_sub(1))
1018            .ok_or(InvalidQueueIdentifier {
1019                qid: sqid,
1020                reason: InvalidQueueIdentifierReason::Oob,
1021            })?;
1022
1023        if sq.is_some() {
1024            return Err(InvalidQueueIdentifier {
1025                qid: sqid,
1026                reason: InvalidQueueIdentifierReason::InUse,
1027            }
1028            .into());
1029        }
1030
1031        let cqid = cdw11.cqid();
1032        let cq = state
1033            .io_cqs
1034            .get_mut((cqid as usize).wrapping_sub(1))
1035            .ok_or(spec::Status::COMPLETION_QUEUE_INVALID)?;
1036
1037        if !cq.task.has_state() {
1038            return Err(spec::Status::COMPLETION_QUEUE_INVALID.into());
1039        }
1040
1041        let sq_gpa = command.dptr[0] & PAGE_MASK;
1042        let len0 = cdw10.qsize_z();
1043        if len0 == 0 || len0 >= MAX_QES || self.config.qe_sizes.lock().sqe_bits != IOSQES {
1044            return Err(spec::Status::INVALID_QUEUE_SIZE.into());
1045        }
1046
1047        let running = cq.task.stop().await;
1048        let sq_idx = cq
1049            .task
1050            .state_mut()
1051            .unwrap()
1052            .create_sq(sqid, sq_gpa, len0 + 1);
1053        if running {
1054            cq.task.start();
1055        }
1056        *sq = Some(IoSq {
1057            sq_idx,
1058            pending_delete_cid: None,
1059            cqid,
1060        });
1061        Ok(())
1062    }
1063
1064    async fn handle_delete_io_submission_queue(
1065        &self,
1066        state: &mut AdminState,
1067        command: &spec::Command,
1068    ) -> Result<Option<CommandResult>, NvmeError> {
1069        let cdw10: spec::Cdw10DeleteIoQueue = command.cdw10.into();
1070        let sqid = cdw10.qid();
1071        let sq = state
1072            .io_sqs
1073            .get_mut((sqid as usize).wrapping_sub(1))
1074            .ok_or(InvalidQueueIdentifier {
1075                qid: sqid,
1076                reason: InvalidQueueIdentifierReason::Oob,
1077            })?
1078            .as_mut()
1079            .ok_or(InvalidQueueIdentifier {
1080                qid: sqid,
1081                reason: InvalidQueueIdentifierReason::NotInUse,
1082            })?;
1083
1084        if sq.pending_delete_cid.is_some() {
1085            return Err(InvalidQueueIdentifier {
1086                qid: sqid,
1087                reason: InvalidQueueIdentifierReason::NotInUse,
1088            }
1089            .into());
1090        }
1091
1092        let cq = &mut state.io_cqs[(sq.cqid as usize).wrapping_sub(1)];
1093        let running = cq.task.stop().await;
1094        cq.task.state_mut().unwrap().delete_sq(sq.sq_idx);
1095        if running {
1096            cq.task.start();
1097        }
1098        sq.pending_delete_cid = Some(command.cdw0.cid());
1099        Ok(None)
1100    }
1101
1102    async fn handle_delete_io_completion_queue(
1103        &self,
1104        state: &mut AdminState,
1105        command: &spec::Command,
1106    ) -> Result<(), NvmeError> {
1107        let cdw10: spec::Cdw10DeleteIoQueue = command.cdw10.into();
1108        let cqid = cdw10.qid();
1109        let cq = state
1110            .io_cqs
1111            .get_mut((cqid as usize).wrapping_sub(1))
1112            .ok_or(InvalidQueueIdentifier {
1113                qid: cqid,
1114                reason: InvalidQueueIdentifierReason::Oob,
1115            })?;
1116
1117        if !cq.task.has_state() {
1118            return Err(InvalidQueueIdentifier {
1119                qid: cqid,
1120                reason: InvalidQueueIdentifierReason::NotInUse,
1121            }
1122            .into());
1123        }
1124        let running = cq.task.stop().await;
1125        if cq.task.state().unwrap().has_sqs() {
1126            if running {
1127                cq.task.start();
1128            }
1129            return Err(spec::Status::INVALID_QUEUE_DELETION.into());
1130        }
1131        cq.task.remove();
1132        Ok(())
1133    }
1134
1135    fn handle_asynchronous_event_request(
1136        &self,
1137        state: &mut AdminState,
1138        command: &spec::Command,
1139    ) -> Result<Option<CommandResult>, NvmeError> {
1140        if state.asynchronous_event_requests.len() >= MAX_ASYNC_EVENT_REQUESTS as usize {
1141            return Err(spec::Status::ASYNCHRONOUS_EVENT_REQUEST_LIMIT_EXCEEDED.into());
1142        }
1143        state.asynchronous_event_requests.push(command.cdw0.cid());
1144        Ok(None)
1145    }
1146
1147    /// Abort is a required command, but a legal implementation is to just
1148    /// complete it with a status that means "I'm sorry, that command couldn't
1149    /// be aborted."
1150    fn handle_abort(&self) -> Result<Option<CommandResult>, NvmeError> {
1151        Ok(Some(CommandResult {
1152            status: spec::Status::SUCCESS,
1153            dw: [1, 0],
1154            dnr: false,
1155        }))
1156    }
1157
1158    fn handle_get_log_page(
1159        &self,
1160        state: &mut AdminState,
1161        command: &spec::Command,
1162    ) -> Result<(), NvmeError> {
1163        let cdw10 = spec::Cdw10GetLogPage::from(command.cdw10);
1164        let cdw11 = spec::Cdw11GetLogPage::from(command.cdw11);
1165        let numd =
1166            ((cdw10.numdl_z() as u32) | ((cdw11.numdu() as u32) << 16)).saturating_add(1) as usize;
1167        let len = numd * 4;
1168        let prp = PrpRange::parse(&self.config.mem, len, command.dptr)?;
1169
1170        match spec::LogPageIdentifier(cdw10.lid()) {
1171            spec::LogPageIdentifier::SUPPORTED_LOG_PAGES => {
1172                // Figure 207: one 4-byte LID Supported and Effects entry per
1173                // LID (0h..=FFh), 1024 bytes total. Mark each log page this
1174                // controller supports with LSUPP set.
1175                let mut page = [0u32; 256];
1176                let supported = spec::LidSupportedAndEffects::new().with_lsupp(true);
1177                for lid in [
1178                    spec::LogPageIdentifier::SUPPORTED_LOG_PAGES,
1179                    spec::LogPageIdentifier::ERROR_INFORMATION,
1180                    spec::LogPageIdentifier::HEALTH_INFORMATION,
1181                    spec::LogPageIdentifier::FIRMWARE_SLOT_INFORMATION,
1182                    spec::LogPageIdentifier::CHANGED_NAMESPACE_LIST,
1183                ] {
1184                    page[lid.0 as usize] = supported.into();
1185                }
1186                let bytes = page.as_bytes();
1187                prp.write(&self.config.mem, &bytes[..len.min(bytes.len())])?;
1188            }
1189            spec::LogPageIdentifier::ERROR_INFORMATION => {
1190                // Write empty log entries.
1191                prp.zero(
1192                    &self.config.mem,
1193                    len.min(ERROR_LOG_PAGE_ENTRIES as usize * 64),
1194                )?;
1195            }
1196            spec::LogPageIdentifier::HEALTH_INFORMATION => {
1197                // Not supported per namespace (LPA.SMARTS is 0): NSID 0h and
1198                // FFFFFFFFh both name the controller log page.
1199                if command.nsid != 0 && command.nsid != !0 {
1200                    return Err(spec::Status::INVALID_FIELD_IN_COMMAND.into());
1201                }
1202                // Write an empty page.
1203                prp.zero(&self.config.mem, len.min(512))?;
1204            }
1205            spec::LogPageIdentifier::FIRMWARE_SLOT_INFORMATION => {
1206                // Write an empty page.
1207                prp.zero(&self.config.mem, len.min(512))?;
1208            }
1209            spec::LogPageIdentifier::CHANGED_NAMESPACE_LIST => {
1210                // Zero the whole list.
1211                prp.zero(&self.config.mem, len.min(4096))?;
1212                // Now write in the changed namespaces.
1213                if state.changed_namespaces.len() > 1024 {
1214                    // Too many to fit, write !0 so the driver scans everything.
1215                    prp.write(&self.config.mem, (!0u32).as_bytes())?;
1216                } else {
1217                    let count = state.changed_namespaces.len().min(numd);
1218                    prp.write(
1219                        &self.config.mem,
1220                        state.changed_namespaces[..count].as_bytes(),
1221                    )?;
1222                }
1223                state.changed_namespaces.clear();
1224                if !cdw10.rae() {
1225                    state.notified_changed_namespaces = false;
1226                }
1227            }
1228            lid => {
1229                tracelimit::warn_ratelimited!(?lid, "unsupported log page");
1230                return Err(spec::Status::INVALID_LOG_PAGE.into());
1231            }
1232        }
1233
1234        Ok(())
1235    }
1236
1237    fn supports_shadow_doorbells(&self, state: &AdminState) -> bool {
1238        let num_queues = state.io_sqs.len().max(state.io_cqs.len()) + 1;
1239        let len = num_queues * (2 << DOORBELL_STRIDE_BITS);
1240        // The spec only allows a single shadow doorbell page.
1241        len <= PAGE_SIZE
1242    }
1243
1244    async fn handle_doorbell_buffer_config(
1245        &self,
1246        state: &mut AdminState,
1247        command: &spec::Command,
1248    ) -> Result<(), NvmeError> {
1249        // Validated by caller.
1250        assert!(self.supports_shadow_doorbells(state));
1251
1252        let shadow_db_gpa = command.dptr[0];
1253        let event_idx_gpa = command.dptr[1];
1254        if (shadow_db_gpa | event_idx_gpa) & !PAGE_MASK != 0 {
1255            return Err(NvmeError::from(spec::Status::INVALID_FIELD_IN_COMMAND));
1256        }
1257
1258        self.config
1259            .doorbells
1260            .write()
1261            .replace_mem(self.config.mem.clone(), shadow_db_gpa, Some(event_idx_gpa))
1262            .map_err(|err| NvmeError::new(spec::Status::DATA_TRANSFER_ERROR, err))?;
1263
1264        Ok(())
1265    }
1266}
1267
1268impl AsyncRun<AdminState> for AdminHandler {
1269    async fn run(
1270        &mut self,
1271        stop: &mut StopTask<'_>,
1272        state: &mut AdminState,
1273    ) -> Result<(), Cancelled> {
1274        loop {
1275            let event = stop.until_stopped(self.next_event(state)).await?;
1276            if let Err(err) = self.process_event(state, event).await {
1277                tracing::error!(
1278                    error = &err as &dyn std::error::Error,
1279                    "admin queue failure"
1280                );
1281                break;
1282            }
1283        }
1284        Ok(())
1285    }
1286}
1287
1288impl InspectTask<AdminState> for AdminHandler {
1289    fn inspect(&self, req: inspect::Request<'_>, state: Option<&AdminState>) {
1290        req.respond().merge(self).merge(state);
1291    }
1292}