nvme_resources/fault.rs
1// Copyright (c) Microsoft Corporation.
2// Licensed under the MIT License.
3
4//! Provides an interface to programmatically and deterministically inject faults in the NVMe fault controller.
5
6use mesh::Cell;
7use mesh::MeshPayload;
8use mesh::OneshotSender;
9use mesh::rpc::Rpc;
10use nvme_spec::Command;
11use nvme_spec::Completion;
12use std::sync::Arc;
13use std::time::Duration;
14
15/// Supported fault behaviour for NVMe admin queues
16#[derive(Debug, MeshPayload)]
17pub enum AdminQueueFaultBehavior<T> {
18 /// Update the queue entry with the returned data
19 Update(T),
20 /// Drop the queue entry
21 Drop,
22 /// Delay. Note: This delay is not asynchronously applied. i.e. Subsequent
23 /// commands will be processed until the delay is over.
24 Delay(Duration),
25 /// Panic
26 Panic(String),
27 /// Writes the given payload to the PRP range. The test should ensure
28 /// that the payload is of valid size. If the size is too large, the fault
29 /// controller will panic. This behavior is not yet supported by the submission
30 /// queue fault.
31 CustomPayload(Vec<u8>),
32 /// Verify that a command was seen.
33 Verify(Option<OneshotSender<()>>),
34}
35
36/// Supported fault behaviour for NVMe IO queues
37#[derive(Debug, MeshPayload, Clone)]
38pub enum IoQueueFaultBehavior {
39 /// Writes the given payload to the PRP range. The test should ensure
40 /// that the payload is of valid size. If the size is too large, the fault
41 /// controller will panic. This behavior is not yet supported by the submission
42 /// queue fault.
43 CustomPayload(Vec<u8>),
44 /// Panic
45 Panic(String),
46 /// Delay. Note: This delay is not asynchronously applied. i.e. Subsequent
47 /// commands will be processed until the delay is over.
48 Delay(Duration),
49}
50
51/// Supported fault behaviour for PCI faults
52#[derive(MeshPayload)]
53pub enum PciFaultBehavior {
54 /// Introduce a delay to the PCI operation. This WILL block the processing
55 /// thread for the delay duration.
56 Delay(Duration),
57 /// Do nothing
58 Default,
59 /// Verify that the fault was triggered.
60 Verify(Option<OneshotSender<()>>),
61}
62
63/// A notification to the test confirming namespace change processing.
64#[derive(MeshPayload)]
65pub enum NamespaceChange {
66 /// Input: Namespace ID to notify, Output: Empty confirmation.
67 ChangeNotification(Rpc<u32, ()>),
68}
69
70/// A fault configuration to apply [`PciFaultBehavior`] to the controller management layer.
71///
72/// Currently the only supported fault is to delay enabling the controller via
73/// cc.en().
74///
75/// # Example
76/// Delay enabling the controller by 500ms.
77///
78/// ```no_run
79/// use mesh::CellUpdater;
80/// use nvme_resources::fault::FaultConfiguration;
81/// use nvme_resources::fault::PciFaultBehavior;
82/// use nvme_resources::fault::PciFaultConfig;
83/// use std::time::Duration;
84///
85/// pub fn pci_enable_delay_fault() -> FaultConfiguration{
86/// let mut fault_start_updater = CellUpdater::new(false);
87/// FaultConfiguration::new(fault_start_updater.cell())
88/// .with_pci_fault(
89/// PciFaultConfig::new().with_cc_enable_fault(
90/// PciFaultBehavior::Delay(Duration::from_millis(500)),
91/// )
92/// )
93/// }
94/// ```
95#[derive(MeshPayload)]
96pub struct PciFaultConfig {
97 /// Fault to apply to cc.en() bit during enablement
98 pub controller_management_fault_enable: PciFaultBehavior,
99 /// Custom MQES value to return in CAP register reads. 1 based value.
100 pub max_queue_size: Option<u16>,
101}
102
103/// A fault config to trigger spurious namespace change notifications from the controller.
104///
105/// The fault controller listens on the provided channel for notifications containing
106/// a `u32` value representing the NSID (Namespace Identifier) that has changed.
107/// This does not actually modify the namespace; instead, it triggers the controller
108/// to process a namespace change notification. The fault is modeled as an
109/// RPC, which the controller completes once it has processed the change and sent
110/// the corresponding Asynchronous Event Notification (AEN).
111/// As per NVMe spec: If multiple namespace changes are notified, only the first triggers an AEN.
112/// Subsequent changes do not trigger additional AENs until the driver issues a
113/// GET_LOG_PAGE command. For implementation simplicity, namespace fault is not
114/// gated by the `fault_active` flag. Since only test code can send
115/// notifications on the fault channel, it is safe to bypass this check.
116///
117/// # Example
118/// Send a namespace change notification for NSID 1 and wait for it to be processed.
119/// ```no_run
120/// use mesh::CellUpdater;
121/// use nvme_resources::fault::NamespaceChange;
122/// use nvme_resources::fault::FaultConfiguration;
123/// use nvme_resources::fault::NamespaceFaultConfig;
124/// use nvme_resources::NvmeFaultControllerHandle;
125/// use guid::Guid;
126/// use mesh::rpc::RpcSend;
127///
128/// pub async fn send_namespace_change_fault() {
129/// let mut fault_start_updater = CellUpdater::new(false);
130/// let (ns_change_send, ns_change_recv) = mesh::channel::<NamespaceChange>();
131/// let fault_configuration = FaultConfiguration::new(fault_start_updater.cell())
132/// .with_namespace_fault(
133/// NamespaceFaultConfig::new(ns_change_recv),
134/// );
135/// // Complete setup
136/// let fault_controller_handle = NvmeFaultControllerHandle {
137/// subsystem_id: Guid::new_random(),
138/// msix_count: 10,
139/// max_io_queues: 10,
140/// namespaces: vec![
141/// // Define `NamespaceDefinitions` here
142/// ],
143/// fault_config: fault_configuration,
144/// enable_tdisp_tests: false,
145/// };
146///
147/// // Send the namespace change notification and await processing.
148/// ns_change_send.call(NamespaceChange::ChangeNotification, 1).await.unwrap();
149/// }
150/// ```
151#[derive(MeshPayload)]
152pub struct NamespaceFaultConfig {
153 /// Receiver for changed namespace notifications
154 pub recv_changed_namespace: mesh::Receiver<NamespaceChange>,
155}
156
157/// A fault configuration to inject faults into the admin submission and completion queues.
158///
159/// This struct maintains a mapping from [`CommandMatch`] to [`AdminQueueFaultBehavior`] for
160/// submission and completion queues. When a command match is found, (and `fault_active == true`)
161/// the associated fault is applied.
162/// Both submission and completion queue faults match on commands
163/// because completions do not contain enough identifying information to
164/// match against. If there is more than one match for a given command, the
165/// match defined first is prioritized. Faults are added via the
166/// `with_submission_queue_fault` and `with_completion_queue_fault` methods and
167/// can be chained. AdminQueueFaultConfig::new() creates an empty fault.
168///
169/// # Panics
170/// Panics if a duplicate `CommandMatch` is added for either submission or
171/// completion queues
172///
173/// # Example
174/// Panic on CREATE_IO_COMPLETION_QUEUE and delay before sending completion for 500ms after
175/// GET_LOG_PAGE command is processed.
176/// ```no_run
177/// use mesh::CellUpdater;
178/// use nvme_resources::fault::AdminQueueFaultConfig;
179/// use nvme_resources::fault::CommandMatch;
180/// use nvme_resources::fault::FaultConfiguration;
181/// use nvme_resources::fault::AdminQueueFaultBehavior;
182/// use nvme_spec::Command;
183/// use std::time::Duration;
184/// use zerocopy::FromZeros;
185/// use zerocopy::IntoBytes;
186///
187/// pub fn build_admin_queue_fault() -> FaultConfiguration {
188/// let mut fault_start_updater = CellUpdater::new(false);
189///
190/// // Setup command matches
191/// let mut command_io_queue = Command::new_zeroed();
192/// let mut command_log_page = Command::new_zeroed();
193/// let mut mask = Command::new_zeroed();
194///
195/// command_io_queue.cdw0 = command_io_queue.cdw0.with_opcode(nvme_spec::AdminOpcode::CREATE_IO_COMPLETION_QUEUE.0);
196/// command_log_page.cdw0 = command_log_page.cdw0.with_opcode(nvme_spec::AdminOpcode::GET_LOG_PAGE.0);
197/// mask.cdw0 = mask.cdw0.with_opcode(u8::MAX);
198///
199/// return FaultConfiguration::new(fault_start_updater.cell())
200/// .with_admin_queue_fault(
201/// AdminQueueFaultConfig::new().with_submission_queue_fault(
202/// CommandMatch {
203/// command: command_io_queue,
204/// mask: mask.as_bytes().try_into().expect("mask should be 64 bytes"),
205/// },
206/// AdminQueueFaultBehavior::Panic("Received a CREATE_IO_COMPLETION_QUEUE command".to_string()),
207/// ).with_completion_queue_fault(
208/// CommandMatch {
209/// command: command_log_page,
210/// mask: mask.as_bytes().try_into().expect("mask should be 64 bytes"),
211/// },
212/// AdminQueueFaultBehavior::Delay(Duration::from_millis(500)),
213/// )
214/// );
215/// }
216/// ```
217#[derive(MeshPayload)]
218pub struct AdminQueueFaultConfig {
219 /// A map of NVME opcodes to the submission fault behavior for each. (This
220 /// would ideally be a `HashMap`, but `mesh` doesn't support that type.
221 /// Given that this is not performance sensitive, the lookup is okay)
222 pub admin_submission_queue_faults: Vec<(CommandMatch, AdminQueueFaultBehavior<Command>)>,
223 /// A map of NVME opcodes to the completion fault behavior for each.
224 pub admin_completion_queue_faults: Vec<(CommandMatch, AdminQueueFaultBehavior<Completion>)>,
225}
226
227/// A fault configuration to inject faults into the io completions.
228///
229/// This struct maintains a mapping from [`CommandMatch`] to [`IoQueueFaultBehavior`] for
230/// completions. When a command match is found, (and `fault_active == true`)
231/// the associated fault is applied.
232/// If there is more than one match for a given command, the
233/// match defined first is prioritized. Faults are added via the
234/// `with_completion_queue_fault` method and calls
235/// can be chained. IoQueueFaultConfig::new() creates an empty fault.
236///
237/// # Panics
238/// Panics if a duplicate `CommandMatch` is added
239///
240/// # Example
241/// Panic when RESERVATION_REPORT command is seen.
242/// ```no_run
243/// use mesh::CellUpdater;
244/// use nvme_resources::fault::IoQueueFaultConfig;
245/// use nvme_resources::fault::CommandMatch;
246/// use nvme_resources::fault::FaultConfiguration;
247/// use nvme_resources::fault::IoQueueFaultBehavior;
248/// use nvme_spec::Command;
249/// use nvme_spec::nvm;
250/// use zerocopy::FromZeros;
251/// use zerocopy::IntoBytes;
252///
253/// pub fn build_admin_queue_fault() -> FaultConfiguration {
254/// let mut fault_start_updater = CellUpdater::new(false);
255///
256/// // Setup command matches
257/// let mut command_io_queue = Command::new_zeroed();
258/// let mut command_log_page = Command::new_zeroed();
259/// let mut mask = Command::new_zeroed();
260///
261/// command_io_queue.cdw0 = command_io_queue.cdw0.with_opcode(nvm::NvmOpcode::RESERVATION_REPORT.0);
262/// mask.cdw0 = mask.cdw0.with_opcode(u8::MAX);
263///
264/// return FaultConfiguration::new(fault_start_updater.cell())
265/// .with_io_queue_fault(
266/// IoQueueFaultConfig::new(fault_start_updater.cell()).with_completion_queue_fault(
267/// CommandMatch {
268/// command: command_io_queue,
269/// mask: mask.as_bytes().try_into().expect("mask should be 64 bytes"),
270/// },
271/// IoQueueFaultBehavior::Panic("Received a RESERVATION_REPORT command".to_string()),
272/// )
273/// );
274/// }
275/// ```
276#[derive(MeshPayload, Clone)]
277pub struct IoQueueFaultConfig {
278 /// A map of NVME opcodes to the completion fault behavior for each.
279 pub io_completion_queue_faults: Vec<(CommandMatch, IoQueueFaultBehavior)>,
280 /// Fault active state. (Repeated here because FaultConfiguration is not Cloneable).
281 pub fault_active: Cell<bool>,
282}
283
284/// A versatile definition to command match [`NVMe commands`](nvme_spec::Command)
285///
286/// Matches NVMe commands using a 512-bit mask: (command_bytes & mask) == (pattern_bytes & mask).
287/// A convenient way to build the patterns is to treat both the command and the mask as
288/// `nvme_spec::Command` and max out the fields in the mask that should be
289/// matched.
290///
291/// # Example
292/// Builds a command match that matches on all CREATE_IO_COMPLETION_QUEUE admin commands.
293/// ```no_run
294/// use nvme_resources::fault::CommandMatch;
295/// use nvme_spec::Command;
296/// use zerocopy::FromZeros;
297/// use zerocopy::IntoBytes;
298///
299/// pub fn build_command_match() -> CommandMatch {
300/// let mut command = Command::new_zeroed();
301/// let mut mask = Command::new_zeroed();
302/// command.cdw0 = command.cdw0.with_opcode(nvme_spec::AdminOpcode::CREATE_IO_COMPLETION_QUEUE.0);
303/// mask.cdw0 = mask.cdw0.with_opcode(u8::MAX);
304/// CommandMatch {
305/// command,
306/// mask: mask.as_bytes().try_into().expect("mask should be 64 bytes"),
307/// }
308/// }
309/// ```
310#[derive(Clone, MeshPayload, PartialEq)]
311pub struct CommandMatch {
312 /// Command to match against
313 pub command: Command,
314 /// Bitmask that defines the bits to match against
315 pub mask: [u8; 64],
316}
317
318/// A fault configuration for overriding the device's hardware configuration
319/// (i.e. the values reported in PCI configuration space).
320///
321/// When a field is `Some`, its value replaces the corresponding field reported
322/// by the NVMe fault controller's PCI hardware IDs. When `None`, the real
323/// hardware ID value is used unchanged. Unlike most other faults in this
324/// module, hardware config faults are applied at device construction time
325/// and are therefore not gated by the `fault_active` flag.
326///
327/// # Example
328/// Override the Vendor ID and Device ID reported in PCI config space.
329/// ```no_run
330/// use mesh::CellUpdater;
331/// use nvme_resources::fault::FaultConfiguration;
332/// use nvme_resources::fault::HardwareConfigFaultConfig;
333///
334/// pub fn hardware_config_fault() -> FaultConfiguration {
335/// let mut fault_start_updater = CellUpdater::new(false);
336/// FaultConfiguration::new(fault_start_updater.cell())
337/// .with_hardware_config_fault(
338/// HardwareConfigFaultConfig::new()
339/// .with_vendor_id(0x1414)
340/// .with_device_id(0xc03e),
341/// )
342/// }
343/// ```
344#[derive(Debug, Copy, Clone, Default, MeshPayload)]
345pub struct HardwareConfigFaultConfig {
346 /// Override for the device's PCI Vendor ID. When `None`, the real Vendor
347 /// ID is used.
348 pub vendor_id: Option<u16>,
349 /// Override for the device's PCI Device ID. When `None`, the real Device
350 /// ID is used.
351 pub device_id: Option<u16>,
352}
353
354/// Fault configuration for the NVMe fault controller.
355///
356/// This struct defines behaviors that inject faults into the NVMe fault controller logic,
357/// such as delaying or dropping commands, triggering namespace change notifications,
358/// or customizing completion payloads. Fault injection is controlled by the
359/// `fault_active` flag, unless specified otherwise in the fault description.
360/// `fault_active` is managed by the test via [`mesh::CellUpdater`]. An
361/// exception to the `fault_active` check is the [`NamespaceFaultConfig`] which
362/// is processed regardless of `fault_active` state. (See `nvme_test` crate for
363/// details on how the faults are applied.)
364///
365/// # Example
366/// Panic when a command that matches CREATE_IO_COMPLETION_QUEUE is seen in the
367/// admin queue:
368/// ```no_run
369/// use mesh::CellUpdater;
370/// use nvme_resources::fault::FaultConfiguration;
371/// use nvme_resources::fault::AdminQueueFaultConfig;
372/// use nvme_resources::fault::CommandMatch;
373/// use nvme_spec::Command;
374/// use nvme_resources::fault::AdminQueueFaultBehavior;
375/// use nvme_resources::NvmeFaultControllerHandle;
376/// use guid::Guid;
377/// use zerocopy::FromZeros;
378/// use zerocopy::IntoBytes;
379///
380/// pub fn example_fault() {
381/// let mut fault_start_updater = CellUpdater::new(false);
382///
383/// // Setup command matches
384/// let mut command = Command::new_zeroed();
385/// let mut mask = Command::new_zeroed();
386///
387/// command.cdw0 = command.cdw0.with_opcode(nvme_spec::AdminOpcode::CREATE_IO_COMPLETION_QUEUE.0);
388/// mask.cdw0 = mask.cdw0.with_opcode(u8::MAX);
389///
390/// let fault_configuration = FaultConfiguration::new(fault_start_updater.cell())
391/// .with_admin_queue_fault(
392/// AdminQueueFaultConfig::new().with_submission_queue_fault(
393/// CommandMatch {
394/// command: command,
395/// mask: mask.as_bytes().try_into().expect("mask should be 64 bytes"),
396/// },
397/// AdminQueueFaultBehavior::Panic("Received a CREATE_IO_COMPLETION_QUEUE command".to_string()),
398/// )
399/// );
400/// let fault_controller_handle = NvmeFaultControllerHandle {
401/// subsystem_id: Guid::new_random(),
402/// msix_count: 10,
403/// max_io_queues: 10,
404/// namespaces: vec![
405/// // Define NamespaceDefinitions here
406/// ],
407/// fault_config: fault_configuration,
408/// enable_tdisp_tests: false,
409/// };
410/// // Pass the controller handle in to the vm config to create and attach the fault controller. At this point the fault is inactive.
411/// fault_start_updater.set(true); // Activate the fault injection.
412/// // ... run test ...
413/// fault_start_updater.set(false); // Deactivate the fault injection.
414/// }
415/// ```
416#[derive(MeshPayload)]
417pub struct FaultConfiguration {
418 /// Fault active state
419 pub fault_active: Cell<bool>,
420 /// Fault to apply to the admin queues
421 pub admin_fault: AdminQueueFaultConfig,
422 /// Fault to apply to management layer of the controller. Option because it
423 /// needs to be extracted by the PCI layer during initialization.
424 pub pci_fault: Option<PciFaultConfig>,
425 /// Fault for test triggered namespace change notifications
426 pub namespace_fault: NamespaceFaultConfig,
427 /// Fault to apply to all IO queues
428 pub io_fault: Arc<IoQueueFaultConfig>,
429 /// Fault to apply to the Hardware Configuration
430 pub hardware_config_fault: Option<HardwareConfigFaultConfig>,
431}
432
433impl FaultConfiguration {
434 /// Create a new empty fault configuration
435 pub fn new(fault_active: Cell<bool>) -> Self {
436 // Ideally the faults should begin life as Option::None.
437 // For now, use a dummy mesh channel for namespace fault to avoid
438 // test setup complexity & special cases in the AdminHandler run loop.
439 Self {
440 fault_active: fault_active.clone(),
441 admin_fault: AdminQueueFaultConfig::new(),
442 pci_fault: Some(PciFaultConfig::new()),
443 namespace_fault: NamespaceFaultConfig::new(mesh::channel().1),
444 io_fault: Arc::new(IoQueueFaultConfig::new(fault_active)),
445 hardware_config_fault: None,
446 }
447 }
448
449 /// Add a PCI fault configuration to the fault configuration
450 pub fn with_pci_fault(mut self, pci_fault: PciFaultConfig) -> Self {
451 self.pci_fault = Some(pci_fault);
452 self
453 }
454
455 /// Add an admin queue fault configuration to the fault configuration
456 pub fn with_admin_queue_fault(mut self, admin_fault: AdminQueueFaultConfig) -> Self {
457 self.admin_fault = admin_fault;
458 self
459 }
460
461 /// Add an IO queue fault configuration to the fault configuration
462 pub fn with_io_queue_fault(mut self, io_fault: IoQueueFaultConfig) -> Self {
463 self.io_fault = Arc::new(io_fault);
464 self
465 }
466
467 /// Add a namespace fault configuration to the fault configuration
468 pub fn with_namespace_fault(mut self, namespace_fault: NamespaceFaultConfig) -> Self {
469 self.namespace_fault = namespace_fault;
470 self
471 }
472
473 /// Add a hardware config fault configuration to the fault configuration
474 pub fn with_hardware_config_fault(
475 mut self,
476 hardware_config_fault: HardwareConfigFaultConfig,
477 ) -> Self {
478 self.hardware_config_fault = Some(hardware_config_fault);
479 self
480 }
481}
482
483impl HardwareConfigFaultConfig {
484 /// Create a new no-op hardware config fault configuration. All fields
485 /// default to `None`, meaning the real hardware values are used.
486 pub fn new() -> Self {
487 Self::default()
488 }
489
490 /// Override the PCI Vendor ID reported by the device.
491 pub fn with_vendor_id(mut self, vendor_id: u16) -> Self {
492 self.vendor_id = Some(vendor_id);
493 self
494 }
495
496 /// Override the PCI Device ID reported by the device.
497 pub fn with_device_id(mut self, device_id: u16) -> Self {
498 self.device_id = Some(device_id);
499 self
500 }
501}
502
503impl PciFaultConfig {
504 /// Create a new no-op fault configuration
505 pub fn new() -> Self {
506 Self {
507 controller_management_fault_enable: PciFaultBehavior::Default,
508 max_queue_size: None,
509 }
510 }
511
512 /// Add a cc.en() fault
513 pub fn with_cc_enable_fault(mut self, behaviour: PciFaultBehavior) -> Self {
514 self.controller_management_fault_enable = behaviour;
515 self
516 }
517
518 /// Add a custom CAP.MQES value to return on register reads
519 pub fn with_max_queue_size(mut self, max_queue_size: u16) -> Self {
520 self.max_queue_size = Some(max_queue_size);
521 self
522 }
523}
524
525impl AdminQueueFaultConfig {
526 /// Create an empty fault configuration
527 pub fn new() -> Self {
528 Self {
529 admin_submission_queue_faults: vec![],
530 admin_completion_queue_faults: vec![],
531 }
532 }
533
534 /// Add a [`CommandMatch`] -> [`AdminQueueFaultBehavior`] mapping for the submission queue.
535 ///
536 /// # Panics
537 /// Panics if an identical [`CommandMatch`] has already been configured.
538 pub fn with_submission_queue_fault(
539 mut self,
540 pattern: CommandMatch,
541 behaviour: AdminQueueFaultBehavior<Command>,
542 ) -> Self {
543 if self
544 .admin_submission_queue_faults
545 .iter()
546 .any(|(c, _)| pattern == *c)
547 {
548 panic!(
549 "Duplicate submission queue fault for Compare {:?} and Mask {:?}",
550 pattern.command, pattern.mask
551 );
552 }
553
554 self.admin_submission_queue_faults
555 .push((pattern, behaviour));
556 self
557 }
558
559 /// Add a [`CommandMatch`] -> [`AdminQueueFaultBehavior`] mapping for the completion queue.
560 ///
561 /// # Panics
562 /// Panics if an identical [`CommandMatch`] has already been configured.
563 pub fn with_completion_queue_fault(
564 mut self,
565 pattern: CommandMatch,
566 behaviour: AdminQueueFaultBehavior<Completion>,
567 ) -> Self {
568 if self
569 .admin_completion_queue_faults
570 .iter()
571 .any(|(c, _)| pattern == *c)
572 {
573 panic!(
574 "Duplicate completion queue fault for Compare {:?} and Mask {:?}",
575 pattern.command, pattern.mask
576 );
577 }
578
579 self.admin_completion_queue_faults
580 .push((pattern, behaviour));
581 self
582 }
583}
584
585impl NamespaceFaultConfig {
586 /// Creates a new NamespaceFaultConfig with a fresh channel.
587 pub fn new(recv_changed_namespace: mesh::Receiver<NamespaceChange>) -> Self {
588 Self {
589 recv_changed_namespace,
590 }
591 }
592}
593
594impl IoQueueFaultConfig {
595 /// Create an empty IO queue fault configuration
596 pub fn new(fault_active: Cell<bool>) -> Self {
597 Self {
598 io_completion_queue_faults: vec![],
599 fault_active,
600 }
601 }
602
603 /// Add a [`CommandMatch`] -> [`IoQueueFaultBehavior`] mapping for the completion queue.
604 ///
605 /// # Panics
606 /// Panics if an identical [`CommandMatch`] has already been configured.
607 pub fn with_completion_queue_fault(
608 mut self,
609 pattern: CommandMatch,
610 behaviour: IoQueueFaultBehavior,
611 ) -> Self {
612 if self
613 .io_completion_queue_faults
614 .iter()
615 .any(|(c, _)| pattern == *c)
616 {
617 panic!(
618 "Duplicate completion queue fault for Compare {:?} and Mask {:?}",
619 pattern.command, pattern.mask
620 );
621 }
622
623 self.io_completion_queue_faults.push((pattern, behaviour));
624 self
625 }
626}