1use super::IoQueueEntrySizes;
7use super::MAX_DATA_TRANSFER_SIZE;
8use super::io::IoHandler;
9use super::io::IoState;
10use crate::BROADCAST_NSID;
11use crate::DOORBELL_STRIDE_BITS;
12use crate::MAX_NSID;
13use crate::MAX_QES;
14use crate::NVME_VERSION;
15use crate::PAGE_MASK;
16use crate::PAGE_SIZE;
17use crate::VENDOR_ID;
18use crate::error::CommandResult;
19use crate::error::NvmeError;
20use crate::namespace::Namespace;
21use crate::prp::PrpRange;
22use crate::queue::CompletionQueue;
23use crate::queue::DoorbellMemory;
24use crate::queue::QueueError;
25use crate::queue::SubmissionQueue;
26use crate::spec;
27use disk_backend::Disk;
28use futures::FutureExt;
29use futures::SinkExt;
30use futures::StreamExt;
31use futures_concurrency::future::Race;
32use guestmem::GuestMemory;
33use guid::Guid;
34use inspect::Inspect;
35use pal_async::task::Spawn;
36use pal_async::task::Task;
37use parking_lot::Mutex;
38use parking_lot::RwLock;
39use std::collections::BTreeMap;
40use std::collections::btree_map;
41use std::future::pending;
42use std::future::poll_fn;
43use std::io::Cursor;
44use std::io::Write;
45use std::sync::Arc;
46use task_control::AsyncRun;
47use task_control::Cancelled;
48use task_control::InspectTask;
49use task_control::StopTask;
50use task_control::TaskControl;
51use thiserror::Error;
52use vmcore::interrupt::Interrupt;
53use vmcore::vm_task::VmTaskDriver;
54use vmcore::vm_task::VmTaskDriverSource;
55use zerocopy::FromBytes;
56use zerocopy::FromZeros;
57use zerocopy::IntoBytes;
58
59const IOSQES: u8 = 6;
60const IOCQES: u8 = 4;
61const MAX_ASYNC_EVENT_REQUESTS: u8 = 4; const ERROR_LOG_PAGE_ENTRIES: u8 = 1;
63
64const ADMIN_CQ_SLOT_RESERVED: &str =
72 "admin completion queue full despite the slot next_event reserved";
73
74#[derive(Inspect)]
75pub struct AdminConfig {
76 #[inspect(skip)]
77 pub driver_source: VmTaskDriverSource,
78 #[inspect(skip)]
79 pub mem: GuestMemory,
80 #[inspect(skip)]
81 pub interrupts: Vec<Interrupt>,
82 #[inspect(skip)]
83 pub doorbells: Arc<RwLock<DoorbellMemory>>,
84 #[inspect(display)]
85 pub subsystem_id: Guid,
86 pub max_sqs: u16,
87 pub max_cqs: u16,
88 pub qe_sizes: Arc<Mutex<IoQueueEntrySizes>>,
89}
90
91#[derive(Inspect)]
92pub struct AdminHandler {
93 driver: VmTaskDriver,
94 config: AdminConfig,
95 #[inspect(iter_by_key)]
96 namespaces: BTreeMap<u32, Arc<Namespace>>,
97}
98
99#[derive(Inspect)]
100pub struct AdminState {
101 admin_sq: SubmissionQueue,
102 admin_cq: CompletionQueue,
103 #[inspect(with = "|x| inspect::iter_by_index(x).map_key(|x| x + 1)")]
104 io_sqs: Vec<Option<IoSq>>,
105 #[inspect(with = "|x| inspect::iter_by_index(x).map_key(|x| x + 1)")]
106 io_cqs: Vec<IoCq>,
107 #[inspect(skip)]
108 sq_delete_response: mesh::Receiver<u16>,
109 #[inspect(iter_by_index)]
110 asynchronous_event_requests: Vec<u16>,
111 #[inspect(
112 rename = "namespaces",
113 with = "|x| inspect::iter_by_key(x.iter().map(|v| (v, ChangedNamespace { changed: true })))"
114 )]
115 changed_namespaces: Vec<u32>,
116 notified_changed_namespaces: bool,
117 async_event_config: u32,
133 #[inspect(skip)]
134 recv_changed_namespace: futures::channel::mpsc::Receiver<u32>,
135 #[inspect(skip)]
136 send_changed_namespace: futures::channel::mpsc::Sender<u32>,
137 #[inspect(skip)]
138 poll_namespace_change: BTreeMap<u32, Task<()>>,
139 features: FeatureState,
140}
141
142#[derive(Inspect)]
147struct FeatureState {
148 #[inspect(hex)]
149 arbitration: u32,
150 #[inspect(hex)]
151 power_management: u32,
152 #[inspect(hex)]
153 error_recovery: u32,
154 #[inspect(hex)]
155 interrupt_coalescing: u32,
156 #[inspect(iter_by_index)]
158 interrupt_vector_coalescing_disable: Vec<bool>,
159}
160
161impl FeatureState {
162 fn new(num_interrupt_vectors: usize) -> Self {
163 Self {
164 arbitration: 0,
165 power_management: 0,
166 error_recovery: 0,
167 interrupt_coalescing: 0,
168 interrupt_vector_coalescing_disable: vec![false; num_interrupt_vectors],
169 }
170 }
171}
172
173fn default_temperature_threshold(thsel: u8) -> u16 {
181 if thsel == 0 { 0xffff } else { 0x0000 }
182}
183
184#[derive(Inspect)]
185struct ChangedNamespace {
186 changed: bool,
187}
188
189#[derive(Inspect)]
190struct IoSq {
191 pending_delete_cid: Option<u16>,
192 sq_idx: usize,
193 cqid: u16,
194}
195
196#[derive(Inspect)]
197struct IoCq {
198 driver: VmTaskDriver,
199 #[inspect(flatten)]
200 task: TaskControl<IoHandler, IoState>,
201}
202
203impl AdminState {
204 pub fn new(handler: &AdminHandler, asq: u64, asqs: u16, acq: u64, acqs: u16) -> Self {
205 #[expect(clippy::disallowed_methods)] let (send_changed_namespace, recv_changed_namespace) = futures::channel::mpsc::channel(256);
209 let poll_namespace_change = handler
210 .namespaces
211 .iter()
212 .map(|(&nsid, namespace)| {
213 (
214 nsid,
215 spawn_namespace_notifier(
216 &handler.driver,
217 nsid,
218 namespace.clone(),
219 send_changed_namespace.clone(),
220 ),
221 )
222 })
223 .collect();
224
225 let admin_cq = CompletionQueue::new(
226 handler.config.doorbells.clone(),
227 1,
228 handler.config.mem.clone(),
229 Some(handler.config.interrupts[0].clone()),
230 acq,
231 acqs,
232 );
233 let mut state = Self {
234 admin_sq: SubmissionQueue::new(&admin_cq, 0, asq, asqs),
235 admin_cq,
236 io_sqs: Vec::new(),
237 io_cqs: Vec::new(),
238 sq_delete_response: Default::default(),
239 asynchronous_event_requests: Vec::new(),
240 changed_namespaces: Vec::new(),
241 notified_changed_namespaces: false,
242 async_event_config: u32::MAX,
243 recv_changed_namespace,
244 send_changed_namespace,
245 poll_namespace_change,
246 features: FeatureState::new(handler.config.interrupts.len()),
247 };
248 state.set_max_queues(handler, handler.config.max_sqs, handler.config.max_cqs);
249 state
250 }
251
252 pub async fn drain(&mut self) {
256 for cq in &mut self.io_cqs {
257 cq.task.stop().await;
258 if let Some(state) = cq.task.state_mut() {
259 state.drain().await;
260 cq.task.remove();
261 }
262 }
263 }
264
265 fn set_max_queues(&mut self, handler: &AdminHandler, num_sqs: u16, num_cqs: u16) {
267 self.io_sqs.truncate(num_sqs.into());
268 self.io_sqs.resize_with(num_sqs.into(), || None);
269 self.io_cqs.resize_with(num_cqs.into(), || {
270 let driver = handler
275 .config
276 .driver_source
277 .builder()
278 .run_on_target(false)
279 .target_vp(0)
280 .build("nvme");
281
282 IoCq {
283 driver,
284 task: TaskControl::new(IoHandler::new(
285 handler.config.mem.clone(),
286 self.sq_delete_response.sender(),
287 )),
288 }
289 });
290 }
291
292 fn add_changed_namespace(&mut self, nsid: u32) {
293 if let Err(i) = self.changed_namespaces.binary_search(&nsid) {
294 self.changed_namespaces.insert(i, nsid);
295 }
296 }
297
298 async fn add_namespace(
299 &mut self,
300 driver: &VmTaskDriver,
301 nsid: u32,
302 namespace: &Arc<Namespace>,
303 ) {
304 for cq in &mut self.io_cqs {
306 let io_running = cq.task.stop().await;
307 if let Some(io_state) = cq.task.state_mut() {
308 io_state.add_namespace(nsid, namespace.clone());
309 }
310 if io_running {
311 cq.task.start();
312 }
313 }
314
315 let old = self.poll_namespace_change.insert(
317 nsid,
318 spawn_namespace_notifier(
319 driver,
320 nsid,
321 namespace.clone(),
322 self.send_changed_namespace.clone(),
323 ),
324 );
325 assert!(old.is_none());
326
327 self.add_changed_namespace(nsid);
329 }
330
331 async fn remove_namespace(&mut self, nsid: u32) {
332 for cq in &mut self.io_cqs {
334 let io_running = cq.task.stop().await;
335 if let Some(io_state) = cq.task.state_mut() {
336 io_state.remove_namespace(nsid);
337 }
338 if io_running {
339 cq.task.start();
340 }
341 }
342
343 self.poll_namespace_change
345 .remove(&nsid)
346 .unwrap()
347 .cancel()
348 .await;
349
350 self.add_changed_namespace(nsid);
352 }
353}
354
355fn spawn_namespace_notifier(
356 driver: &VmTaskDriver,
357 nsid: u32,
358 namespace: Arc<Namespace>,
359 mut send_changed_namespace: futures::channel::mpsc::Sender<u32>,
360) -> Task<()> {
361 driver.spawn("wait_resize", async move {
362 let mut counter = None;
363 loop {
364 counter = Some(namespace.wait_change(counter).await);
365 tracing::info!(nsid, "namespace changed");
366 if send_changed_namespace.send(nsid).await.is_err() {
367 break;
368 }
369 }
370 })
371}
372
373#[derive(Debug, Error)]
374#[error("invalid queue identifier {qid}")]
375struct InvalidQueueIdentifier {
376 qid: u16,
377 #[source]
378 reason: InvalidQueueIdentifierReason,
379}
380
381#[derive(Debug, Error)]
382enum InvalidQueueIdentifierReason {
383 #[error("queue id is out of bounds")]
384 Oob,
385 #[error("queue id is in use")]
386 InUse,
387 #[error("queue id is not in use")]
388 NotInUse,
389}
390
391impl From<InvalidQueueIdentifier> for NvmeError {
392 fn from(err: InvalidQueueIdentifier) -> Self {
393 Self::new(spec::Status::INVALID_QUEUE_IDENTIFIER, err)
394 }
395}
396
397enum Event {
398 Command(Result<spec::Command, QueueError>),
399 SqDeleteComplete(u16),
400 NamespaceChange(u32),
401}
402
403#[derive(Debug, Error)]
405pub enum AddNamespaceError {
406 #[error("namespace id conflict for {0}")]
408 Conflict(u32),
409 #[error("namespace id {0} is out of range (must be 1..={MAX_NSID})")]
412 OutOfRange(u32),
413}
414
415impl AdminHandler {
416 pub fn new(driver: VmTaskDriver, config: AdminConfig) -> Self {
417 Self {
418 driver,
419 config,
420 namespaces: Default::default(),
421 }
422 }
423
424 pub async fn add_namespace(
425 &mut self,
426 state: Option<&mut AdminState>,
427 nsid: u32,
428 disk: Disk,
429 ) -> Result<(), AddNamespaceError> {
430 if nsid == 0 || nsid > MAX_NSID {
431 return Err(AddNamespaceError::OutOfRange(nsid));
432 }
433 let namespace = &*match self.namespaces.entry(nsid) {
434 btree_map::Entry::Vacant(entry) => entry.insert(Arc::new(Namespace::new(
435 self.config.mem.clone(),
436 nsid,
437 disk,
438 ))),
439 btree_map::Entry::Occupied(_) => return Err(AddNamespaceError::Conflict(nsid)),
440 };
441
442 if let Some(state) = state {
443 state.add_namespace(&self.driver, nsid, namespace).await;
444 }
445
446 Ok(())
447 }
448
449 pub async fn remove_namespace(&mut self, state: Option<&mut AdminState>, nsid: u32) -> bool {
450 if self.namespaces.remove(&nsid).is_none() {
451 return false;
452 }
453
454 if let Some(state) = state {
455 state.remove_namespace(nsid).await;
456 }
457
458 true
459 }
460
461 async fn next_event(&mut self, state: &mut AdminState) -> Result<Event, QueueError> {
462 let event = loop {
463 poll_fn(|cx| state.admin_cq.poll_ready(cx)).await?;
466
467 let ns_aen_enabled = spec::Cdw11FeatureAsyncEventConfig::from(state.async_event_config)
477 .namespace_attribute_notices();
478
479 if !state.changed_namespaces.is_empty()
480 && !state.notified_changed_namespaces
481 && ns_aen_enabled
482 {
483 if let Some(cid) = state.asynchronous_event_requests.pop() {
484 let posted = state.admin_cq.write(
485 spec::Completion {
486 dw0: spec::AsynchronousEventRequestDw0::new()
487 .with_event_type(spec::AsynchronousEventType::NOTICE.0)
488 .with_log_page_identifier(spec::LogPageIdentifier::CHANGED_NAMESPACE_LIST.0)
489 .with_information(spec::AsynchronousEventInformationNotice::NAMESPACE_ATTRIBUTE_CHANGED.0)
490 .into(),
491 dw1: 0,
492 sqhd: state.admin_sq.sqhd(),
493 sqid: 0,
494 cid,
495 status: spec::CompletionStatus::new(),
496 },
497 )?;
498 assert!(posted, "{ADMIN_CQ_SLOT_RESERVED}");
499
500 state.notified_changed_namespaces = true;
501 continue;
502 }
503 }
504
505 let next_command = poll_fn(|cx| state.admin_sq.poll_next(cx)).map(Event::Command);
506 let sq_delete_complete = async {
507 let Some(sqid) = state.sq_delete_response.next().await else {
508 pending().await
509 };
510 Event::SqDeleteComplete(sqid)
511 };
512 let changed_namespace = async {
513 let Some(nsid) = state.recv_changed_namespace.next().await else {
514 pending().await
515 };
516 Event::NamespaceChange(nsid)
517 };
518
519 break (next_command, sq_delete_complete, changed_namespace)
520 .race()
521 .await;
522 };
523 Ok(event)
524 }
525
526 async fn process_event(
527 &mut self,
528 state: &mut AdminState,
529 event: Result<Event, QueueError>,
530 ) -> Result<(), QueueError> {
531 let (cid, result) = match event? {
532 Event::Command(command) => {
533 let command = command?;
534 let opcode = spec::AdminOpcode(command.cdw0.opcode());
535
536 tracing::trace!(?opcode, ?command, "command");
537
538 let result = match opcode {
539 _ if command.cdw0.fuse() != 0 || command.cdw0.psdt() != 0 => Err(
544 NvmeError::do_not_retry(spec::Status::INVALID_FIELD_IN_COMMAND),
545 ),
546 spec::AdminOpcode::IDENTIFY => self
547 .handle_identify(state, &command)
548 .map(|()| Some(Default::default())),
549 spec::AdminOpcode::GET_FEATURES => {
550 self.handle_get_features(state, &command).await.map(Some)
551 }
552 spec::AdminOpcode::SET_FEATURES => {
553 self.handle_set_features(state, &command).map(Some)
554 }
555 spec::AdminOpcode::CREATE_IO_COMPLETION_QUEUE => self
556 .handle_create_io_completion_queue(state, &command)
557 .map(|()| Some(Default::default())),
558 spec::AdminOpcode::CREATE_IO_SUBMISSION_QUEUE => self
559 .handle_create_io_submission_queue(state, &command)
560 .await
561 .map(|()| Some(Default::default())),
562 spec::AdminOpcode::DELETE_IO_COMPLETION_QUEUE => self
563 .handle_delete_io_completion_queue(state, &command)
564 .await
565 .map(|()| Some(Default::default())),
566 spec::AdminOpcode::DELETE_IO_SUBMISSION_QUEUE => {
567 self.handle_delete_io_submission_queue(state, &command)
568 .await
569 }
570 spec::AdminOpcode::ASYNCHRONOUS_EVENT_REQUEST => {
571 self.handle_asynchronous_event_request(state, &command)
572 }
573 spec::AdminOpcode::ABORT => self.handle_abort(),
574 spec::AdminOpcode::GET_LOG_PAGE => self
575 .handle_get_log_page(state, &command)
576 .map(|()| Some(Default::default())),
577 spec::AdminOpcode::DOORBELL_BUFFER_CONFIG
578 if self.supports_shadow_doorbells(state) =>
579 {
580 self.handle_doorbell_buffer_config(state, &command)
581 .await
582 .map(|()| Some(Default::default()))
583 }
584 opcode => {
585 tracelimit::warn_ratelimited!(?opcode, "unsupported opcode");
586 Err(spec::Status::INVALID_COMMAND_OPCODE.into())
587 }
588 };
589
590 let result = match result {
591 Ok(Some(cr)) => cr,
592 Ok(None) => return Ok(()),
593 Err(err) => {
594 tracelimit::warn_ratelimited!(
595 error = &err as &dyn std::error::Error,
596 cid = command.cdw0.cid(),
597 ?opcode,
598 "command error"
599 );
600 err.into()
601 }
602 };
603
604 (command.cdw0.cid(), result)
605 }
606 Event::SqDeleteComplete(sqid) => {
607 let sq = state.io_sqs[sqid as usize - 1].take().unwrap();
608 let cid = sq.pending_delete_cid.unwrap();
609 (cid, Default::default())
610 }
611 Event::NamespaceChange(nsid) => {
612 state.add_changed_namespace(nsid);
613 return Ok(());
614 }
615 };
616
617 let completion = spec::Completion {
618 dw0: result.dw[0],
619 dw1: result.dw[1],
620 sqid: 0,
621 sqhd: state.admin_sq.sqhd(),
622 status: result.completion_status(),
623 cid,
624 };
625
626 let posted = state.admin_cq.write(completion)?;
627 assert!(posted, "{ADMIN_CQ_SLOT_RESERVED}");
628 Ok(())
629 }
630
631 fn handle_identify(
632 &mut self,
633 state: &AdminState,
634 command: &spec::Command,
635 ) -> Result<(), NvmeError> {
636 let cdw10: spec::Cdw10Identify = command.cdw10.into();
637 let mut buf = [0u64; 512];
639 let buf = buf.as_mut_bytes();
640 match spec::Cns(cdw10.cns()) {
641 spec::Cns::CONTROLLER => {
642 let id = spec::IdentifyController::mut_from_prefix(buf).unwrap().0; *id = self.identify_controller(state);
644
645 write!(
646 Cursor::new(&mut id.subnqn[..]),
647 "nqn.2014-08.org.nvmexpress:uuid:{}",
648 self.config.subsystem_id
649 )
650 .unwrap();
651 }
652 spec::Cns::ACTIVE_NAMESPACES => {
653 if command.nsid >= 0xfffffffe {
654 return Err(spec::Status::INVALID_NAMESPACE_OR_FORMAT.into());
655 }
656 let nsids = <[u32]>::mut_from_bytes(buf).unwrap();
657 for (ns, nsid) in self
658 .namespaces
659 .keys()
660 .filter(|&ns| *ns > command.nsid)
661 .zip(nsids)
662 {
663 *nsid = *ns;
664 }
665 }
666 spec::Cns::NAMESPACE => {
667 if command.nsid == 0 || command.nsid > MAX_NSID {
668 return Err(spec::Status::INVALID_NAMESPACE_OR_FORMAT.into());
669 }
670 if let Some(ns) = self.namespaces.get(&command.nsid) {
671 ns.identify(buf);
672 } else {
673 tracing::trace!(nsid = command.nsid, "inactive namespace id");
676 }
677 }
678 spec::Cns::DESCRIPTOR_NAMESPACE => {
679 if command.nsid == 0 || command.nsid > MAX_NSID {
680 return Err(spec::Status::INVALID_NAMESPACE_OR_FORMAT.into());
681 }
682 if let Some(ns) = self.namespaces.get(&command.nsid) {
683 ns.namespace_id_descriptor(buf);
684 } else {
685 tracing::trace!(nsid = command.nsid, "inactive namespace id");
688 }
689 }
690 spec::Cns::SPECIFIC_CONTROLLER_IO_COMMAND_SET => {
691 let csi = (command.cdw11 >> 24) as u8;
697 if csi != 0 {
698 return Err(spec::Status::INVALID_FIELD_IN_COMMAND.into());
699 }
700 }
703 cns => {
704 tracelimit::warn_ratelimited!(?cns, "unsupported cns");
705 return Err(spec::Status::INVALID_FIELD_IN_COMMAND.into());
706 }
707 };
708 PrpRange::parse(&self.config.mem, buf.len(), command.dptr)?.write(&self.config.mem, buf)?;
709 Ok(())
710 }
711
712 fn identify_controller(&self, state: &AdminState) -> spec::IdentifyController {
713 spec::IdentifyController {
714 vid: VENDOR_ID,
715 ssvid: VENDOR_ID,
716 mdts: (MAX_DATA_TRANSFER_SIZE / PAGE_SIZE).trailing_zeros() as u8,
717 ver: NVME_VERSION,
718 rtd3r: 400000,
719 rtd3e: 400000,
720 sqes: spec::QueueEntrySize::new()
721 .with_min(IOSQES)
722 .with_max(IOSQES),
723 cqes: spec::QueueEntrySize::new()
724 .with_min(IOCQES)
725 .with_max(IOCQES),
726 frmw: spec::FirmwareUpdates::new().with_ffsro(true).with_nofs(1),
727 nn: MAX_NSID,
728 ieee: [0x74, 0xe2, 0x8c], fr: (*b"v1.00000").into(),
730 mn: (*b"MSFT NVMe Accelerator v1.0 ").into(),
731 sn: (*b"SN: 000001 ").into(),
732 aerl: MAX_ASYNC_EVENT_REQUESTS - 1,
733 elpe: ERROR_LOG_PAGE_ENTRIES - 1,
734 oaes: spec::Oaes::new().with_namespace_attribute(true),
735 oncs: spec::Oncs::new()
736 .with_dataset_management(true)
737 .with_reservations(true),
739 vwc: spec::VolatileWriteCache::new()
740 .with_present(true)
741 .with_broadcast_flush_behavior(spec::BroadcastFlushBehavior::NOT_SUPPORTED.0),
742 cntrltype: spec::ControllerType::IO_CONTROLLER,
743 oacs: spec::OptionalAdminCommandSupport::new()
744 .with_doorbell_buffer_config(self.supports_shadow_doorbells(state)),
745 ..FromZeros::new_zeroed()
746 }
747 }
748
749 fn handle_set_features(
750 &mut self,
751 state: &mut AdminState,
752 command: &spec::Command,
753 ) -> Result<CommandResult, NvmeError> {
754 let cdw10: spec::Cdw10SetFeatures = command.cdw10.into();
755 let mut dw = [0; 2];
756 if cdw10.save() {
761 return Err(spec::Status::FEATURE_IDENTIFIER_NOT_SAVEABLE.into());
762 }
763 match spec::Feature(cdw10.fid()) {
764 spec::Feature::NUMBER_OF_QUEUES => {
765 if state.io_sqs.iter().any(|sq| sq.is_some())
766 || state.io_cqs.iter().any(|cq| cq.task.has_state())
767 {
768 return Err(spec::Status::COMMAND_SEQUENCE_ERROR.into());
769 }
770 let cdw11: spec::Cdw11FeatureNumberOfQueues = command.cdw11.into();
771 if cdw11.ncq_z() == u16::MAX || cdw11.nsq_z() == u16::MAX {
772 return Err(spec::Status::INVALID_FIELD_IN_COMMAND.into());
773 }
774 let num_sqs = (cdw11.nsq_z() + 1).min(self.config.max_sqs);
775 let num_cqs = (cdw11.ncq_z() + 1).min(self.config.max_cqs);
776 state.set_max_queues(self, num_sqs, num_cqs);
777
778 dw[0] = spec::Cdw11FeatureNumberOfQueues::new()
779 .with_ncq_z(num_cqs - 1)
780 .with_nsq_z(num_sqs - 1)
781 .into();
782 }
783 spec::Feature::VOLATILE_WRITE_CACHE => {
784 let cdw11 = spec::Cdw11FeatureVolatileWriteCache::from(command.cdw11);
785 if !cdw11.wce() {
786 tracelimit::warn_ratelimited!(
787 "ignoring unsupported attempt to disable write cache"
788 );
789 }
790 }
791 spec::Feature::ASYNC_EVENT_CONFIG => {
792 state.async_event_config = command.cdw11;
801 }
802 spec::Feature::ARBITRATION => {
803 state.features.arbitration = command.cdw11;
804 }
805 spec::Feature::POWER_MANAGEMENT => {
806 let cdw11 = spec::Cdw11FeaturePowerManagement::from(command.cdw11);
807 if cdw11.ps() != 0 {
810 return Err(spec::Status::INVALID_FIELD_IN_COMMAND.into());
811 }
812 state.features.power_management = command.cdw11;
813 }
814 spec::Feature::TEMPERATURE_THRESHOLD => {
815 let cdw11 = spec::Cdw11FeatureTemperatureThreshold::from(command.cdw11);
816 if cdw11.thsel() > 1 {
818 return Err(spec::Status::INVALID_FIELD_IN_COMMAND.into());
819 }
820 }
822 spec::Feature::ERROR_RECOVERY => {
823 state.features.error_recovery = command.cdw11;
824 }
825 spec::Feature::INTERRUPT_COALESCING => {
826 state.features.interrupt_coalescing = command.cdw11;
827 }
828 spec::Feature::INTERRUPT_VECTOR_CONFIG => {
829 let cdw11 = spec::Cdw11FeatureInterruptVectorConfig::from(command.cdw11);
830 let cd = state
831 .features
832 .interrupt_vector_coalescing_disable
833 .get_mut(cdw11.iv() as usize)
834 .ok_or(spec::Status::INVALID_FIELD_IN_COMMAND)?;
835 *cd = cdw11.cd();
836 }
837 feature => {
838 tracelimit::warn_ratelimited!(?feature, "unsupported feature");
839 return Err(spec::Status::INVALID_FIELD_IN_COMMAND.into());
840 }
841 }
842 Ok(CommandResult::new(spec::Status::SUCCESS, dw))
843 }
844
845 async fn handle_get_features(
846 &mut self,
847 state: &mut AdminState,
848 command: &spec::Command,
849 ) -> Result<CommandResult, NvmeError> {
850 let cdw10: spec::Cdw10GetFeatures = command.cdw10.into();
851 let mut dw = [0; 2];
852
853 if cdw10.sel() != 0 {
857 return Err(spec::Status::INVALID_FIELD_IN_COMMAND.into());
858 }
859 match spec::Feature(cdw10.fid()) {
860 spec::Feature::NUMBER_OF_QUEUES => {
861 let num_cqs = state.io_cqs.len();
862 let num_sqs = state.io_sqs.len();
863 dw[0] = spec::Cdw11FeatureNumberOfQueues::new()
864 .with_ncq_z((num_cqs - 1) as u16)
865 .with_nsq_z((num_sqs - 1) as u16)
866 .into();
867 }
868 spec::Feature::VOLATILE_WRITE_CACHE => {
869 dw[0] = spec::Cdw11FeatureVolatileWriteCache::new()
871 .with_wce(true)
872 .into();
873 }
874 spec::Feature::ASYNC_EVENT_CONFIG => {
875 dw[0] = state.async_event_config;
882 }
883 spec::Feature::NVM_RESERVATION_PERSISTENCE => {
884 if command.nsid == BROADCAST_NSID {
889 return Err(spec::Status::INVALID_FIELD_IN_COMMAND.into());
890 }
891 if command.nsid == 0 || command.nsid > MAX_NSID {
892 return Err(spec::Status::INVALID_NAMESPACE_OR_FORMAT.into());
893 }
894 let namespace = self
895 .namespaces
896 .get(&command.nsid)
897 .ok_or(spec::Status::INVALID_FIELD_IN_COMMAND)?;
898
899 return namespace.get_feature(command).await;
900 }
901 spec::Feature::ARBITRATION => {
902 dw[0] = state.features.arbitration;
903 }
904 spec::Feature::POWER_MANAGEMENT => {
905 dw[0] = state.features.power_management;
906 }
907 spec::Feature::TEMPERATURE_THRESHOLD => {
908 let cdw11 = spec::Cdw11FeatureTemperatureThreshold::from(command.cdw11);
909 if cdw11.thsel() > 1 {
910 return Err(spec::Status::INVALID_FIELD_IN_COMMAND.into());
911 }
912 dw[0] = spec::Cdw11FeatureTemperatureThreshold::new()
913 .with_tmpth(default_temperature_threshold(cdw11.thsel()))
914 .with_tmpsel(cdw11.tmpsel())
915 .with_thsel(cdw11.thsel())
916 .into();
917 }
918 spec::Feature::ERROR_RECOVERY => {
919 dw[0] = state.features.error_recovery;
920 }
921 spec::Feature::INTERRUPT_COALESCING => {
922 dw[0] = state.features.interrupt_coalescing;
923 }
924 spec::Feature::INTERRUPT_VECTOR_CONFIG => {
925 let cdw11 = spec::Cdw11FeatureInterruptVectorConfig::from(command.cdw11);
926 let cd = *state
927 .features
928 .interrupt_vector_coalescing_disable
929 .get(cdw11.iv() as usize)
930 .ok_or(spec::Status::INVALID_FIELD_IN_COMMAND)?;
931 dw[0] = spec::Cdw11FeatureInterruptVectorConfig::new()
932 .with_iv(cdw11.iv())
933 .with_cd(cd)
934 .into();
935 }
936 feature => {
937 tracelimit::warn_ratelimited!(?feature, "unsupported feature");
938 return Err(spec::Status::INVALID_FIELD_IN_COMMAND.into());
939 }
940 }
941 Ok(CommandResult::new(spec::Status::SUCCESS, dw))
942 }
943
944 fn handle_create_io_completion_queue(
945 &mut self,
946 state: &mut AdminState,
947 command: &spec::Command,
948 ) -> Result<(), NvmeError> {
949 let cdw10: spec::Cdw10CreateIoQueue = command.cdw10.into();
950 let cdw11: spec::Cdw11CreateIoCompletionQueue = command.cdw11.into();
951 if !cdw11.pc() {
952 return Err(spec::Status::INVALID_FIELD_IN_COMMAND.into());
953 }
954 let cqid = cdw10.qid();
955 let cq = state
956 .io_cqs
957 .get_mut((cqid as usize).wrapping_sub(1))
958 .ok_or(InvalidQueueIdentifier {
959 qid: cqid,
960 reason: InvalidQueueIdentifierReason::Oob,
961 })?;
962
963 if cq.task.has_state() {
964 return Err(InvalidQueueIdentifier {
965 qid: cqid,
966 reason: InvalidQueueIdentifierReason::InUse,
967 }
968 .into());
969 }
970
971 let interrupt = if cdw11.ien() {
972 let iv = cdw11.iv();
973 if iv as usize >= self.config.interrupts.len() {
974 return Err(spec::Status::INVALID_INTERRUPT_VECTOR.into());
975 };
976 Some(iv)
977 } else {
978 None
979 };
980 let gpa = command.dptr[0] & PAGE_MASK;
981 let len0 = cdw10.qsize_z();
982 if len0 == 0 || len0 >= MAX_QES || self.config.qe_sizes.lock().cqe_bits != IOCQES {
983 return Err(spec::Status::INVALID_QUEUE_SIZE.into());
984 }
985
986 let interrupt = interrupt.map(|iv| self.config.interrupts[iv as usize].clone());
987 let namespaces = self.namespaces.clone();
988
989 let state = IoState::new(
990 &self.config.mem,
991 self.config.doorbells.clone(),
992 gpa,
993 len0 + 1,
994 cqid,
995 interrupt,
996 namespaces,
997 );
998
999 cq.task.insert(&cq.driver, "nvme-io", state);
1000 cq.task.start();
1001 Ok(())
1002 }
1003
1004 async fn handle_create_io_submission_queue(
1005 &mut self,
1006 state: &mut AdminState,
1007 command: &spec::Command,
1008 ) -> Result<(), NvmeError> {
1009 let cdw10: spec::Cdw10CreateIoQueue = command.cdw10.into();
1010 let cdw11: spec::Cdw11CreateIoSubmissionQueue = command.cdw11.into();
1011 if !cdw11.pc() {
1012 return Err(spec::Status::INVALID_FIELD_IN_COMMAND.into());
1013 }
1014 let sqid = cdw10.qid();
1015 let sq = state
1016 .io_sqs
1017 .get_mut((sqid as usize).wrapping_sub(1))
1018 .ok_or(InvalidQueueIdentifier {
1019 qid: sqid,
1020 reason: InvalidQueueIdentifierReason::Oob,
1021 })?;
1022
1023 if sq.is_some() {
1024 return Err(InvalidQueueIdentifier {
1025 qid: sqid,
1026 reason: InvalidQueueIdentifierReason::InUse,
1027 }
1028 .into());
1029 }
1030
1031 let cqid = cdw11.cqid();
1032 let cq = state
1033 .io_cqs
1034 .get_mut((cqid as usize).wrapping_sub(1))
1035 .ok_or(spec::Status::COMPLETION_QUEUE_INVALID)?;
1036
1037 if !cq.task.has_state() {
1038 return Err(spec::Status::COMPLETION_QUEUE_INVALID.into());
1039 }
1040
1041 let sq_gpa = command.dptr[0] & PAGE_MASK;
1042 let len0 = cdw10.qsize_z();
1043 if len0 == 0 || len0 >= MAX_QES || self.config.qe_sizes.lock().sqe_bits != IOSQES {
1044 return Err(spec::Status::INVALID_QUEUE_SIZE.into());
1045 }
1046
1047 let running = cq.task.stop().await;
1048 let sq_idx = cq
1049 .task
1050 .state_mut()
1051 .unwrap()
1052 .create_sq(sqid, sq_gpa, len0 + 1);
1053 if running {
1054 cq.task.start();
1055 }
1056 *sq = Some(IoSq {
1057 sq_idx,
1058 pending_delete_cid: None,
1059 cqid,
1060 });
1061 Ok(())
1062 }
1063
1064 async fn handle_delete_io_submission_queue(
1065 &self,
1066 state: &mut AdminState,
1067 command: &spec::Command,
1068 ) -> Result<Option<CommandResult>, NvmeError> {
1069 let cdw10: spec::Cdw10DeleteIoQueue = command.cdw10.into();
1070 let sqid = cdw10.qid();
1071 let sq = state
1072 .io_sqs
1073 .get_mut((sqid as usize).wrapping_sub(1))
1074 .ok_or(InvalidQueueIdentifier {
1075 qid: sqid,
1076 reason: InvalidQueueIdentifierReason::Oob,
1077 })?
1078 .as_mut()
1079 .ok_or(InvalidQueueIdentifier {
1080 qid: sqid,
1081 reason: InvalidQueueIdentifierReason::NotInUse,
1082 })?;
1083
1084 if sq.pending_delete_cid.is_some() {
1085 return Err(InvalidQueueIdentifier {
1086 qid: sqid,
1087 reason: InvalidQueueIdentifierReason::NotInUse,
1088 }
1089 .into());
1090 }
1091
1092 let cq = &mut state.io_cqs[(sq.cqid as usize).wrapping_sub(1)];
1093 let running = cq.task.stop().await;
1094 cq.task.state_mut().unwrap().delete_sq(sq.sq_idx);
1095 if running {
1096 cq.task.start();
1097 }
1098 sq.pending_delete_cid = Some(command.cdw0.cid());
1099 Ok(None)
1100 }
1101
1102 async fn handle_delete_io_completion_queue(
1103 &self,
1104 state: &mut AdminState,
1105 command: &spec::Command,
1106 ) -> Result<(), NvmeError> {
1107 let cdw10: spec::Cdw10DeleteIoQueue = command.cdw10.into();
1108 let cqid = cdw10.qid();
1109 let cq = state
1110 .io_cqs
1111 .get_mut((cqid as usize).wrapping_sub(1))
1112 .ok_or(InvalidQueueIdentifier {
1113 qid: cqid,
1114 reason: InvalidQueueIdentifierReason::Oob,
1115 })?;
1116
1117 if !cq.task.has_state() {
1118 return Err(InvalidQueueIdentifier {
1119 qid: cqid,
1120 reason: InvalidQueueIdentifierReason::NotInUse,
1121 }
1122 .into());
1123 }
1124 let running = cq.task.stop().await;
1125 if cq.task.state().unwrap().has_sqs() {
1126 if running {
1127 cq.task.start();
1128 }
1129 return Err(spec::Status::INVALID_QUEUE_DELETION.into());
1130 }
1131 cq.task.remove();
1132 Ok(())
1133 }
1134
1135 fn handle_asynchronous_event_request(
1136 &self,
1137 state: &mut AdminState,
1138 command: &spec::Command,
1139 ) -> Result<Option<CommandResult>, NvmeError> {
1140 if state.asynchronous_event_requests.len() >= MAX_ASYNC_EVENT_REQUESTS as usize {
1141 return Err(spec::Status::ASYNCHRONOUS_EVENT_REQUEST_LIMIT_EXCEEDED.into());
1142 }
1143 state.asynchronous_event_requests.push(command.cdw0.cid());
1144 Ok(None)
1145 }
1146
1147 fn handle_abort(&self) -> Result<Option<CommandResult>, NvmeError> {
1151 Ok(Some(CommandResult {
1152 status: spec::Status::SUCCESS,
1153 dw: [1, 0],
1154 dnr: false,
1155 }))
1156 }
1157
1158 fn handle_get_log_page(
1159 &self,
1160 state: &mut AdminState,
1161 command: &spec::Command,
1162 ) -> Result<(), NvmeError> {
1163 let cdw10 = spec::Cdw10GetLogPage::from(command.cdw10);
1164 let cdw11 = spec::Cdw11GetLogPage::from(command.cdw11);
1165 let numd =
1166 ((cdw10.numdl_z() as u32) | ((cdw11.numdu() as u32) << 16)).saturating_add(1) as usize;
1167 let len = numd * 4;
1168 let prp = PrpRange::parse(&self.config.mem, len, command.dptr)?;
1169
1170 match spec::LogPageIdentifier(cdw10.lid()) {
1171 spec::LogPageIdentifier::SUPPORTED_LOG_PAGES => {
1172 let mut page = [0u32; 256];
1176 let supported = spec::LidSupportedAndEffects::new().with_lsupp(true);
1177 for lid in [
1178 spec::LogPageIdentifier::SUPPORTED_LOG_PAGES,
1179 spec::LogPageIdentifier::ERROR_INFORMATION,
1180 spec::LogPageIdentifier::HEALTH_INFORMATION,
1181 spec::LogPageIdentifier::FIRMWARE_SLOT_INFORMATION,
1182 spec::LogPageIdentifier::CHANGED_NAMESPACE_LIST,
1183 ] {
1184 page[lid.0 as usize] = supported.into();
1185 }
1186 let bytes = page.as_bytes();
1187 prp.write(&self.config.mem, &bytes[..len.min(bytes.len())])?;
1188 }
1189 spec::LogPageIdentifier::ERROR_INFORMATION => {
1190 prp.zero(
1192 &self.config.mem,
1193 len.min(ERROR_LOG_PAGE_ENTRIES as usize * 64),
1194 )?;
1195 }
1196 spec::LogPageIdentifier::HEALTH_INFORMATION => {
1197 if command.nsid != 0 && command.nsid != !0 {
1200 return Err(spec::Status::INVALID_FIELD_IN_COMMAND.into());
1201 }
1202 prp.zero(&self.config.mem, len.min(512))?;
1204 }
1205 spec::LogPageIdentifier::FIRMWARE_SLOT_INFORMATION => {
1206 prp.zero(&self.config.mem, len.min(512))?;
1208 }
1209 spec::LogPageIdentifier::CHANGED_NAMESPACE_LIST => {
1210 prp.zero(&self.config.mem, len.min(4096))?;
1212 if state.changed_namespaces.len() > 1024 {
1214 prp.write(&self.config.mem, (!0u32).as_bytes())?;
1216 } else {
1217 let count = state.changed_namespaces.len().min(numd);
1218 prp.write(
1219 &self.config.mem,
1220 state.changed_namespaces[..count].as_bytes(),
1221 )?;
1222 }
1223 state.changed_namespaces.clear();
1224 if !cdw10.rae() {
1225 state.notified_changed_namespaces = false;
1226 }
1227 }
1228 lid => {
1229 tracelimit::warn_ratelimited!(?lid, "unsupported log page");
1230 return Err(spec::Status::INVALID_LOG_PAGE.into());
1231 }
1232 }
1233
1234 Ok(())
1235 }
1236
1237 fn supports_shadow_doorbells(&self, state: &AdminState) -> bool {
1238 let num_queues = state.io_sqs.len().max(state.io_cqs.len()) + 1;
1239 let len = num_queues * (2 << DOORBELL_STRIDE_BITS);
1240 len <= PAGE_SIZE
1242 }
1243
1244 async fn handle_doorbell_buffer_config(
1245 &self,
1246 state: &mut AdminState,
1247 command: &spec::Command,
1248 ) -> Result<(), NvmeError> {
1249 assert!(self.supports_shadow_doorbells(state));
1251
1252 let shadow_db_gpa = command.dptr[0];
1253 let event_idx_gpa = command.dptr[1];
1254 if (shadow_db_gpa | event_idx_gpa) & !PAGE_MASK != 0 {
1255 return Err(NvmeError::from(spec::Status::INVALID_FIELD_IN_COMMAND));
1256 }
1257
1258 self.config
1259 .doorbells
1260 .write()
1261 .replace_mem(self.config.mem.clone(), shadow_db_gpa, Some(event_idx_gpa))
1262 .map_err(|err| NvmeError::new(spec::Status::DATA_TRANSFER_ERROR, err))?;
1263
1264 Ok(())
1265 }
1266}
1267
1268impl AsyncRun<AdminState> for AdminHandler {
1269 async fn run(
1270 &mut self,
1271 stop: &mut StopTask<'_>,
1272 state: &mut AdminState,
1273 ) -> Result<(), Cancelled> {
1274 loop {
1275 let event = stop.until_stopped(self.next_event(state)).await?;
1276 if let Err(err) = self.process_event(state, event).await {
1277 tracing::error!(
1278 error = &err as &dyn std::error::Error,
1279 "admin queue failure"
1280 );
1281 break;
1282 }
1283 }
1284 Ok(())
1285 }
1286}
1287
1288impl InspectTask<AdminState> for AdminHandler {
1289 fn inspect(&self, req: inspect::Request<'_>, state: Option<&AdminState>) {
1290 req.respond().merge(self).merge(state);
1291 }
1292}