Skip to main content

storvsp/
lib.rs

1// Copyright (c) Microsoft Corporation.
2// Licensed under the MIT License.
3
4//! VMBus SCSI controller emulator (StorVSP).
5//!
6//! StorVSP implements the Hyper-V synthetic SCSI protocol — a VMBus-based
7//! transport that carries SCSI CDBs between the guest's `storvsc` driver and
8//! the VMM. This is not a standard SCSI transport (like iSCSI or SAS); it's a
9//! Hyper-V-specific wire format defined in [`storvsp_protocol`].
10//!
11//! # Architecture
12//!
13//! The crate uses a multi-worker model. The primary VMBus channel handles
14//! protocol version negotiation (Win6 through Blue); sub-channels process I/O
15//! in parallel. Each worker owns a VMBus ring and processes packets
16//! concurrently via `FuturesUnordered`.
17//!
18//! StorVSP handles the transport (ring buffer management, GPADL setup, packet
19//! framing, sub-channel lifecycle) and a few SCSI control commands directly
20//! (`REPORT_LUNS`, `INQUIRY` for absent targets). All actual I/O is delegated
21//! to [`AsyncScsiDisk`] implementations — StorVSP
22//! never interprets SCSI data CDBs itself.
23//!
24//! For the channel/sub-channel model, CPU affinity, and performance
25//! characteristics, see the
26//! [StorVSP Channels & Subchannels](https://openvmm.dev/reference/devices/vmbus/storvsp_channels.html)
27//! page in the OpenVMM Guide.
28//!
29//! # Key types
30//!
31//! - [`StorageDevice`] — the VMBus device. Implements `VmbusDevice` and
32//!   `SaveRestoreVmbusDevice`.
33//! - [`ScsiController`] — manages attached disks by [`ScsiPath`]. Supports
34//!   runtime attach/remove.
35//! - [`ScsiControllerDisk`] — wraps `Arc<dyn AsyncScsiDisk>`.
36//!
37//! # Performance
38//!
39//! Poll-mode optimization: when pending I/O count exceeds
40//! `poll_mode_queue_depth`, the worker switches from interrupt-driven to
41//! busy-poll for new requests, reducing guest exit frequency. Future storage
42//! for SCSI request processing is pooled to avoid allocation on the hot path.
43
44#![expect(missing_docs)]
45#![forbid(unsafe_code)]
46
47#[cfg(feature = "ioperf")]
48pub mod ioperf;
49
50#[cfg(feature = "test")]
51pub mod test_helpers;
52
53#[cfg(not(feature = "test"))]
54mod test_helpers;
55
56pub mod resolver;
57mod save_restore;
58
59use crate::ring::gparange::MultiPagedRangeBuf;
60use anyhow::Context as _;
61use async_trait::async_trait;
62use futures::StreamExt;
63use guestmem::AccessError;
64use guestmem::GuestMemory;
65use guestmem::MemoryRead;
66use guestmem::MemoryWrite;
67use guestmem::ranges::PagedRange;
68use guid::Guid;
69use inspect::Inspect;
70use inspect::InspectMut;
71use inspect_counters::Counter;
72use inspect_counters::Histogram;
73use oversized_box::OversizedBox;
74use parking_lot::Mutex;
75use parking_lot::RwLock;
76use ring::OutgoingPacketType;
77use scsi::AdditionalSenseCode;
78use scsi::ScsiOp;
79use scsi::ScsiStatus;
80use scsi::srb::SrbStatus;
81use scsi::srb::SrbStatusAndFlags;
82use scsi_buffers::RequestBuffers;
83use scsi_core::AsyncScsiDisk;
84use scsi_core::Request;
85use scsi_core::ScsiResult;
86use scsi_defs as scsi;
87use scsidisk::illegal_request_sense;
88use slab::Slab;
89use std::collections::hash_map::Entry;
90use std::collections::hash_map::HashMap;
91use std::fmt::Debug;
92use std::future::Future;
93use std::future::poll_fn;
94use std::pin::Pin;
95use std::sync::Arc;
96use std::sync::atomic::AtomicBool;
97use std::sync::atomic::AtomicU32;
98use std::sync::atomic::Ordering::Acquire;
99use std::sync::atomic::Ordering::Relaxed;
100use std::sync::atomic::Ordering::Release;
101use std::task::Context;
102use std::task::Poll;
103use std::task::Waker;
104use storvsp_resources::ScsiPath;
105use task_control::AsyncRun;
106use task_control::InspectTask;
107use task_control::StopTask;
108use task_control::TaskControl;
109use thiserror::Error;
110use tracing_helpers::ErrorValueExt;
111use unicycle::FuturesUnordered;
112use vmbus_async::queue;
113use vmbus_async::queue::ExternalDataError;
114use vmbus_async::queue::IncomingPacket;
115use vmbus_async::queue::OutgoingPacket;
116use vmbus_async::queue::Queue;
117use vmbus_channel::RawAsyncChannel;
118use vmbus_channel::bus::ChannelType;
119use vmbus_channel::bus::OfferParams;
120use vmbus_channel::bus::OpenRequest;
121use vmbus_channel::channel::ChannelControl;
122use vmbus_channel::channel::ChannelOpenError;
123use vmbus_channel::channel::DeviceResources;
124use vmbus_channel::channel::RestoreControl;
125use vmbus_channel::channel::SaveRestoreVmbusDevice;
126use vmbus_channel::channel::VmbusDevice;
127use vmbus_channel::gpadl_ring::GpadlRingMem;
128use vmbus_channel::gpadl_ring::gpadl_channel;
129use vmbus_core::protocol::UserDefinedData;
130use vmbus_ring as ring;
131use vmbus_ring::RingMem;
132use vmcore::save_restore::RestoreError;
133use vmcore::save_restore::SaveError;
134use vmcore::save_restore::SavedStateBlob;
135use vmcore::vm_task::VmTaskDriver;
136use vmcore::vm_task::VmTaskDriverSource;
137use zerocopy::FromBytes;
138use zerocopy::FromZeros;
139use zerocopy::Immutable;
140use zerocopy::IntoBytes;
141use zerocopy::KnownLayout;
142
143/// The IO queue depth at which the controller switches from guest-signal-driven
144/// to poll-mode operation. This optimization reduces the guest exit rate by
145/// relying on (typically-interrupt-driven) IO completions to drive polling for
146/// new IO requests.
147const DEFAULT_POLL_MODE_QUEUE_DEPTH: u32 = 1;
148
149pub struct StorageDevice {
150    instance_id: Guid,
151    ide_path: Option<ScsiPath>,
152    workers: Vec<WorkerAndDriver>,
153    controller: Arc<ScsiControllerState>,
154    resources: DeviceResources,
155    driver_source: VmTaskDriverSource,
156    max_sub_channel_count: u16,
157    protocol: Arc<Protocol>,
158    io_queue_depth: u32,
159}
160
161#[derive(Inspect)]
162struct WorkerAndDriver {
163    #[inspect(flatten)]
164    worker: TaskControl<WorkerState, Worker>,
165    driver: VmTaskDriver,
166}
167
168struct WorkerState;
169
170impl InspectMut for StorageDevice {
171    fn inspect_mut(&mut self, req: inspect::Request<'_>) {
172        let mut resp = req.respond();
173
174        let disks = self.controller.disks.read();
175        for (path, controller_disk) in disks.iter() {
176            resp.child(&format!("disks/{}", path), |req| {
177                controller_disk.disk.inspect(req);
178            });
179        }
180
181        resp.fields(
182            "channels",
183            self.workers
184                .iter()
185                .filter(|task| task.worker.has_state())
186                .enumerate(),
187        )
188        .field(
189            "poll_mode_queue_depth",
190            inspect::AtomicMut(&self.controller.poll_mode_queue_depth),
191        );
192    }
193}
194
195struct Worker<T: RingMem = GpadlRingMem> {
196    inner: WorkerInner,
197    rescan_notification: Arc<RescanNotification>,
198    queue: Queue<T>,
199}
200
201/// A coalescing notification with one waiter per worker.
202#[derive(Default)]
203struct RescanNotification {
204    pending: AtomicBool,
205    waker: Mutex<Option<Waker>>,
206}
207
208impl RescanNotification {
209    fn notify(&self) {
210        self.pending.store(true, Release);
211        // Keep registration across notifications, including a wake delayed
212        // until after the pending flag has already been consumed.
213        let waker = self.waker.lock().clone();
214        if let Some(waker) = waker {
215            waker.wake();
216        }
217    }
218
219    fn reset(&self) {
220        self.pending.store(false, Relaxed);
221    }
222
223    fn receiver(&self) -> RescanReceiver<'_> {
224        RescanReceiver {
225            notification: self,
226            last_waker: None,
227        }
228    }
229}
230
231struct RescanReceiver<'a> {
232    notification: &'a RescanNotification,
233    last_waker: Option<Waker>,
234}
235
236impl RescanReceiver<'_> {
237    fn poll(&mut self, cx: &mut Context<'_>) -> Poll<()> {
238        // Register once unless the task's waker changes. Unrelated
239        // I/O wakeups only compare the waker and load the pending flag.
240        if self
241            .last_waker
242            .as_ref()
243            .is_none_or(|waker| !cx.waker().will_wake(waker))
244        {
245            let waker = cx.waker().clone();
246            let old_waker = self.notification.waker.lock().replace(waker.clone());
247            self.last_waker = Some(waker);
248            drop(old_waker);
249        }
250        // Check after registration so a concurrent notification cannot
251        // leave the worker asleep.
252        if self.notification.pending.load(Relaxed) {
253            self.notification.pending.swap(false, Acquire);
254            Poll::Ready(())
255        } else {
256            Poll::Pending
257        }
258    }
259}
260
261struct Protocol {
262    state: RwLock<ProtocolState>,
263    /// Signaled when `state` transitions to `ProtocolState::Ready`.
264    ready: event_listener::Event,
265}
266
267struct WorkerInner {
268    protocol: Arc<Protocol>,
269    request_size: usize,
270    controller: Arc<ScsiControllerState>,
271    channel_index: u16,
272    scsi_queue: Arc<ScsiCommandQueue>,
273    scsi_requests: FuturesUnordered<ScsiRequest>,
274    scsi_requests_states: Slab<ScsiRequestState>,
275    full_request_pool: Vec<Arc<ScsiRequestAndRange>>,
276    future_pool: Vec<OversizedBox<(), ScsiOpStorage>>,
277    channel_control: ChannelControl,
278    max_io_queue_depth: usize,
279    stats: WorkerStats,
280}
281
282#[derive(Debug, Default, Inspect)]
283struct WorkerStats {
284    ios_submitted: Counter,
285    ios_completed: Counter,
286    wakes: Counter,
287    wakes_spurious: Counter,
288    per_wake_submissions: Histogram<10>,
289    per_wake_completions: Histogram<10>,
290}
291
292#[repr(u16)]
293#[derive(Copy, Clone, Debug, Inspect, PartialEq, Eq, PartialOrd, Ord)]
294enum Version {
295    Win6 = storvsp_protocol::VERSION_WIN6,
296    Win7 = storvsp_protocol::VERSION_WIN7,
297    Win8 = storvsp_protocol::VERSION_WIN8,
298    Blue = storvsp_protocol::VERSION_BLUE,
299}
300
301#[derive(Debug, Error)]
302#[error("protocol version {0:#x} not supported")]
303struct UnsupportedVersion(u16);
304
305impl Version {
306    fn parse(major_minor: u16) -> Result<Self, UnsupportedVersion> {
307        let version = match major_minor {
308            storvsp_protocol::VERSION_WIN6 => Self::Win6,
309            storvsp_protocol::VERSION_WIN7 => Self::Win7,
310            storvsp_protocol::VERSION_WIN8 => Self::Win8,
311            storvsp_protocol::VERSION_BLUE => Self::Blue,
312            version => return Err(UnsupportedVersion(version)),
313        };
314        assert_eq!(version as u16, major_minor);
315        Ok(version)
316    }
317
318    fn max_request_size(&self) -> usize {
319        match self {
320            Version::Win8 | Version::Blue => storvsp_protocol::SCSI_REQUEST_LEN_V2,
321            Version::Win6 | Version::Win7 => storvsp_protocol::SCSI_REQUEST_LEN_V1,
322        }
323    }
324}
325
326#[derive(Copy, Clone)]
327enum ProtocolState {
328    Init(InitState),
329    Ready {
330        version: Version,
331        subchannel_count: u16,
332    },
333}
334
335#[derive(Copy, Clone, Debug)]
336enum InitState {
337    Begin,
338    QueryVersion,
339    QueryProperties {
340        version: Version,
341    },
342    EndInitialization {
343        version: Version,
344        subchannel_count: Option<u16>,
345    },
346}
347
348/// The internal SCSI operation future type.
349///
350/// This is a boxed future of a large pre-determined size. The box is reused
351/// after a SCSI request completes to avoid allocations in the hot path.
352///
353/// An Option type is used so that the future can be efficiently dropped (via
354/// `Pin::set(x, None)`) before it is stashed away for reuse.
355type ScsiOpStorage = [u64; SCSI_REQUEST_STACK_SIZE / 8];
356type ScsiOpFuture = Pin<OversizedBox<dyn Future<Output = ScsiResult> + Send, ScsiOpStorage>>;
357
358/// The amount of space reserved for a ScsiOpFuture.
359///
360/// This was chosen by running `cargo test -p storvsp -- --no-capture` and looking at the required
361/// size that was given in the failure message
362const SCSI_REQUEST_STACK_SIZE: usize = scsi_core::ASYNC_SCSI_DISK_STACK_SIZE + 272;
363
364struct ScsiRequest {
365    request_id: usize,
366    future: Option<ScsiOpFuture>,
367}
368
369impl ScsiRequest {
370    fn new(
371        request_id: usize,
372        future: OversizedBox<dyn Future<Output = ScsiResult> + Send, ScsiOpStorage>,
373    ) -> Self {
374        Self {
375            request_id,
376            future: Some(future.into()),
377        }
378    }
379}
380
381impl Future for ScsiRequest {
382    type Output = (usize, ScsiResult, OversizedBox<(), ScsiOpStorage>);
383
384    fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Self::Output> {
385        let this = self.get_mut();
386        let future = this.future.as_mut().unwrap().as_mut();
387        let result = std::task::ready!(future.poll(cx));
388        // Return the future so that its storage can be reused.
389        let future = this.future.take().unwrap();
390        Poll::Ready((this.request_id, result, OversizedBox::empty_pinned(future)))
391    }
392}
393
394#[derive(Debug, Error)]
395enum WorkerError {
396    #[error("cancelled")]
397    Cancelled(task_control::Cancelled),
398    #[error("packet error")]
399    PacketError(#[source] PacketError),
400    #[error("queue error")]
401    Queue(#[source] queue::Error),
402    #[error("queue should have enough space but no longer does")]
403    NotEnoughSpace,
404}
405
406impl From<task_control::Cancelled> for WorkerError {
407    fn from(value: task_control::Cancelled) -> Self {
408        Self::Cancelled(value)
409    }
410}
411
412#[derive(Debug, Error)]
413enum PacketError {
414    #[error("Not transactional")]
415    NotTransactional,
416    #[error("Unrecognized operation {0:?}")]
417    UnrecognizedOperation(storvsp_protocol::Operation),
418    #[error("Invalid packet type")]
419    InvalidPacketType,
420    #[error("Invalid data transfer length")]
421    InvalidDataTransferLength,
422    #[error("Access error: {0}")]
423    Access(#[source] AccessError),
424    #[error("Range error")]
425    Range(#[source] ExternalDataError),
426}
427
428#[derive(Debug, Default, Clone)]
429struct Range {
430    len: usize,
431    is_write: bool,
432}
433
434impl Range {
435    fn new(buf: &MultiPagedRangeBuf, request: &storvsp_protocol::ScsiRequest) -> Option<Self> {
436        let len = request.data_transfer_length as usize;
437        let is_write = request.data_in != 0;
438        // Ensure there is exactly one range and it's large enough, or there are
439        // zero ranges and there is no associated SCSI buffer.
440        if buf.range_count() > 1 || (len > 0 && buf.first()?.len() < len) {
441            return None;
442        }
443        Some(Self { len, is_write })
444    }
445
446    fn buffer<'a>(
447        &'a self,
448        buf: &'a MultiPagedRangeBuf,
449        guest_memory: &'a GuestMemory,
450    ) -> RequestBuffers<'a> {
451        let mut range = buf.first().unwrap_or_else(PagedRange::empty);
452        range.truncate(self.len);
453        RequestBuffers::new(guest_memory, range, self.is_write)
454    }
455}
456
457#[derive(Debug)]
458struct Packet {
459    data: PacketData,
460    transaction_id: u64,
461    request_size: usize,
462}
463
464#[derive(Debug)]
465enum PacketData {
466    BeginInitialization,
467    EndInitialization,
468    QueryProtocolVersion(u16),
469    QueryProperties,
470    CreateSubChannels(u16),
471    ExecuteScsi(Arc<ScsiRequestAndRange>),
472    ResetBus,
473    ResetAdapter,
474    ResetLun,
475}
476
477fn parse_packet<T: RingMem>(
478    packet: &IncomingPacket<'_, T>,
479    pool: &mut Vec<Arc<ScsiRequestAndRange>>,
480) -> Result<Packet, PacketError> {
481    let packet = match packet {
482        IncomingPacket::Completion(_) => return Err(PacketError::InvalidPacketType),
483        IncomingPacket::Data(packet) => packet,
484    };
485    let transaction_id = packet
486        .transaction_id()
487        .ok_or(PacketError::NotTransactional)?;
488
489    let mut reader = packet.reader();
490    let header: storvsp_protocol::Packet = reader.read_plain().map_err(PacketError::Access)?;
491    // You would expect that this should be limited to the current protocol
492    // version's maximum packet size, but this is not what Hyper-V does, and
493    // Linux 6.1 relies on this behavior during protocol initialization.
494    let request_size = reader.len().min(storvsp_protocol::SCSI_REQUEST_LEN_MAX);
495    let data = match header.operation {
496        storvsp_protocol::Operation::BEGIN_INITIALIZATION => PacketData::BeginInitialization,
497        storvsp_protocol::Operation::END_INITIALIZATION => PacketData::EndInitialization,
498        storvsp_protocol::Operation::QUERY_PROTOCOL_VERSION => {
499            let mut version = storvsp_protocol::ProtocolVersion::new_zeroed();
500            reader
501                .read(version.as_mut_bytes())
502                .map_err(PacketError::Access)?;
503            PacketData::QueryProtocolVersion(version.major_minor)
504        }
505        storvsp_protocol::Operation::QUERY_PROPERTIES => PacketData::QueryProperties,
506        storvsp_protocol::Operation::EXECUTE_SRB => {
507            let mut full_request = pool.pop().unwrap_or_else(|| {
508                Arc::new(ScsiRequestAndRange {
509                    external_data: Range::default(),
510                    external_data_buf: MultiPagedRangeBuf::new(),
511                    request: storvsp_protocol::ScsiRequest::new_zeroed(),
512                    request_size,
513                })
514            });
515
516            {
517                let full_request = Arc::get_mut(&mut full_request).unwrap();
518                let request_buf = &mut full_request.request.as_mut_bytes()[..request_size];
519                reader.read(request_buf).map_err(PacketError::Access)?;
520
521                full_request.external_data_buf.clear();
522                packet
523                    .read_external_ranges(&mut full_request.external_data_buf)
524                    .map_err(PacketError::Range)?;
525
526                full_request.external_data =
527                    Range::new(&full_request.external_data_buf, &full_request.request)
528                        .ok_or(PacketError::InvalidDataTransferLength)?;
529            }
530
531            PacketData::ExecuteScsi(full_request)
532        }
533        storvsp_protocol::Operation::RESET_LUN => PacketData::ResetLun,
534        storvsp_protocol::Operation::RESET_ADAPTER => PacketData::ResetAdapter,
535        storvsp_protocol::Operation::RESET_BUS => PacketData::ResetBus,
536        storvsp_protocol::Operation::CREATE_SUB_CHANNELS => {
537            let mut sub_channel_count: u16 = 0;
538            reader
539                .read(sub_channel_count.as_mut_bytes())
540                .map_err(PacketError::Access)?;
541            PacketData::CreateSubChannels(sub_channel_count)
542        }
543        _ => return Err(PacketError::UnrecognizedOperation(header.operation)),
544    };
545
546    if let PacketData::ExecuteScsi(_) = data {
547        tracing::trace!(transaction_id, ?data, "parse_packet");
548    } else {
549        tracing::debug!(transaction_id, ?data, "parse_packet");
550    }
551
552    Ok(Packet {
553        data,
554        request_size,
555        transaction_id,
556    })
557}
558
559impl WorkerInner {
560    fn send_vmbus_packet<M: RingMem>(
561        &mut self,
562        writer: &mut queue::WriteBatch<'_, M>,
563        packet_type: OutgoingPacketType<'_>,
564        request_size: usize,
565        transaction_id: u64,
566        operation: storvsp_protocol::Operation,
567        status: storvsp_protocol::NtStatus,
568        payload: &[u8],
569    ) -> Result<(), WorkerError> {
570        let header = storvsp_protocol::Packet {
571            operation,
572            flags: 0,
573            status,
574        };
575
576        let packet_size = size_of_val(&header) + request_size;
577
578        // Zero pad or truncate the payload to the queue's packet size. This is
579        // necessary because Windows guests check that each packet's size is
580        // exactly the largest possible packet size for the negotiated protocol
581        // version.
582        let len = size_of_val(&header) + size_of_val(payload);
583        let padding = [0; storvsp_protocol::SCSI_REQUEST_LEN_MAX];
584        let (payload_bytes, padding_bytes) = if len > packet_size {
585            (&payload[..packet_size - size_of_val(&header)], &[][..])
586        } else {
587            (payload, &padding[..packet_size - len])
588        };
589        assert_eq!(
590            size_of_val(&header) + payload_bytes.len() + padding_bytes.len(),
591            packet_size
592        );
593        writer
594            .try_write(&OutgoingPacket {
595                transaction_id,
596                packet_type,
597                payload: &[header.as_bytes(), payload_bytes, padding_bytes],
598            })
599            .map_err(|err| match err {
600                queue::TryWriteError::Full(_) => WorkerError::NotEnoughSpace,
601                queue::TryWriteError::Queue(err) => WorkerError::Queue(err),
602            })
603    }
604
605    fn send_packet<M: RingMem, P: IntoBytes + Immutable + KnownLayout>(
606        &mut self,
607        writer: &mut queue::WriteHalf<'_, M>,
608        operation: storvsp_protocol::Operation,
609        status: storvsp_protocol::NtStatus,
610        payload: &P,
611    ) -> Result<(), WorkerError> {
612        self.send_vmbus_packet(
613            &mut writer.batched(),
614            OutgoingPacketType::InBandNoCompletion,
615            self.request_size,
616            0,
617            operation,
618            status,
619            payload.as_bytes(),
620        )
621    }
622
623    fn send_completion<M: RingMem, P: IntoBytes + Immutable + KnownLayout>(
624        &mut self,
625        writer: &mut queue::WriteHalf<'_, M>,
626        packet: &Packet,
627        status: storvsp_protocol::NtStatus,
628        payload: &P,
629    ) -> Result<(), WorkerError> {
630        self.send_vmbus_packet(
631            &mut writer.batched(),
632            OutgoingPacketType::Completion,
633            packet.request_size,
634            packet.transaction_id,
635            storvsp_protocol::Operation::COMPLETE_IO,
636            status,
637            payload.as_bytes(),
638        )
639    }
640}
641
642struct ScsiCommandQueue {
643    controller: Arc<ScsiControllerState>,
644    mem: GuestMemory,
645    force_path_id: Option<u8>,
646}
647
648impl ScsiCommandQueue {
649    async fn execute_scsi(&self, full_request: &ScsiRequestAndRange) -> ScsiResult {
650        let request = &full_request.request;
651        let op = ScsiOp(request.payload[0]);
652        let external_data = full_request
653            .external_data
654            .buffer(&full_request.external_data_buf, &self.mem);
655
656        tracing::trace!(
657            path_id = request.path_id,
658            target_id = request.target_id,
659            lun = request.lun,
660            op = ?op,
661            "execute_scsi start...",
662        );
663
664        let path_id = self.force_path_id.unwrap_or(request.path_id);
665
666        let controller_disk = self
667            .controller
668            .disks
669            .read()
670            .get(&ScsiPath {
671                path: path_id,
672                target: request.target_id,
673                lun: request.lun,
674            })
675            .cloned();
676
677        let result = match op {
678            ScsiOp::REPORT_LUNS => {
679                const HEADER_SIZE: usize = size_of::<scsi::LunList>();
680                let mut luns: Vec<u8> = self
681                    .controller
682                    .disks
683                    .read()
684                    .keys()
685                    .flat_map(|path| {
686                        // Use the original path ID and not the forced one to
687                        // match Hyper-V storvsp behavior.
688                        if request.path_id == path.path && request.target_id == path.target {
689                            Some(path.lun)
690                        } else {
691                            None
692                        }
693                    })
694                    .collect();
695                luns.sort_unstable();
696                let mut data: Vec<u64> = vec![0; luns.len() + 1];
697                let header = scsi::LunList {
698                    length: (luns.len() as u32 * 8).into(),
699                    reserved: [0; 4],
700                };
701                data.as_mut_bytes()[..HEADER_SIZE].copy_from_slice(header.as_bytes());
702                for (i, lun) in luns.iter().enumerate() {
703                    data[i + 1].as_mut_bytes()[..2].copy_from_slice(&(*lun as u16).to_be_bytes());
704                }
705                if external_data.len() >= HEADER_SIZE {
706                    let tx = std::cmp::min(external_data.len(), data.as_bytes().len());
707                    external_data.writer().write(&data.as_bytes()[..tx]).map_or(
708                        ScsiResult {
709                            scsi_status: ScsiStatus::CHECK_CONDITION,
710                            srb_status: SrbStatus::INVALID_REQUEST,
711                            tx: 0,
712                            sense_data: Some(illegal_request_sense(
713                                AdditionalSenseCode::INVALID_CDB,
714                            )),
715                        },
716                        |_| ScsiResult {
717                            scsi_status: ScsiStatus::GOOD,
718                            srb_status: SrbStatus::SUCCESS,
719                            tx,
720                            sense_data: None,
721                        },
722                    )
723                } else {
724                    ScsiResult {
725                        scsi_status: ScsiStatus::GOOD,
726                        srb_status: SrbStatus::SUCCESS,
727                        tx: 0,
728                        sense_data: None,
729                    }
730                }
731            }
732            _ if let Some(controller_disk) = controller_disk => {
733                let mut cdb = [0; 16];
734                cdb.copy_from_slice(&request.payload[0..storvsp_protocol::CDB16GENERIC_LENGTH]);
735                controller_disk
736                    .disk
737                    .execute_scsi(
738                        &external_data,
739                        &Request {
740                            cdb,
741                            srb_flags: request.srb_flags,
742                        },
743                    )
744                    .await
745            }
746            ScsiOp::INQUIRY => {
747                let cdb = scsi::CdbInquiry::ref_from_prefix(&request.payload)
748                    .unwrap()
749                    .0; // TODO: zerocopy: ref-from-prefix: use-rest-of-range (https://github.com/microsoft/openvmm/issues/759)
750                if external_data.len() < cdb.allocation_length.get() as usize
751                    || request.data_in != storvsp_protocol::SCSI_IOCTL_DATA_IN
752                    || (cdb.allocation_length.get() as usize) < size_of::<scsi::InquiryDataHeader>()
753                {
754                    ScsiResult {
755                        scsi_status: ScsiStatus::CHECK_CONDITION,
756                        srb_status: SrbStatus::INVALID_REQUEST,
757                        tx: 0,
758                        sense_data: Some(illegal_request_sense(AdditionalSenseCode::INVALID_CDB)),
759                    }
760                } else {
761                    let enable_vpd = cdb.flags.vpd();
762                    if enable_vpd || cdb.page_code != 0 {
763                        // cannot support VPD inquiry for non-existing device (lun).
764                        ScsiResult {
765                            scsi_status: ScsiStatus::CHECK_CONDITION,
766                            srb_status: SrbStatus::INVALID_REQUEST,
767                            tx: 0,
768                            sense_data: Some(illegal_request_sense(
769                                AdditionalSenseCode::INVALID_CDB,
770                            )),
771                        }
772                    } else {
773                        const LOGICAL_UNIT_NOT_PRESENT_DEVICE: u8 = 0x7F;
774                        let mut data = scsidisk::INQUIRY_DATA_TEMPLATE;
775                        data.header.device_type = LOGICAL_UNIT_NOT_PRESENT_DEVICE;
776
777                        if request.lun != 0 {
778                            // Below fields are only set for lun0 inquiry so zero out here.
779                            data.vendor_id = [0; 8];
780                            data.product_id = [0; 16];
781                            data.product_revision_level = [0; 4];
782                        }
783
784                        let datab = data.as_bytes();
785                        let tx = std::cmp::min(
786                            cdb.allocation_length.get() as usize,
787                            size_of::<scsi::InquiryData>(),
788                        );
789                        external_data.writer().write(&datab[..tx]).map_or(
790                            ScsiResult {
791                                scsi_status: ScsiStatus::CHECK_CONDITION,
792                                srb_status: SrbStatus::INVALID_REQUEST,
793                                tx: 0,
794                                sense_data: Some(illegal_request_sense(
795                                    AdditionalSenseCode::INVALID_CDB,
796                                )),
797                            },
798                            |_| ScsiResult {
799                                scsi_status: ScsiStatus::GOOD,
800                                srb_status: SrbStatus::SUCCESS,
801                                tx,
802                                sense_data: None,
803                            },
804                        )
805                    }
806                }
807            }
808            _ => ScsiResult {
809                scsi_status: ScsiStatus::CHECK_CONDITION,
810                srb_status: SrbStatus::INVALID_LUN,
811                tx: 0,
812                sense_data: None,
813            },
814        };
815
816        tracing::trace!(
817            path_id = request.path_id,
818            target_id = request.target_id,
819            lun = request.lun,
820            op = ?op,
821            result = ?result,
822            "execute_scsi completed.",
823        );
824        result
825    }
826}
827
828impl<T: RingMem + 'static> Worker<T> {
829    fn new(
830        controller: Arc<ScsiControllerState>,
831        channel: RawAsyncChannel<T>,
832        channel_index: u16,
833        mem: GuestMemory,
834        channel_control: ChannelControl,
835        io_queue_depth: u32,
836        protocol: Arc<Protocol>,
837        force_path_id: Option<u8>,
838    ) -> anyhow::Result<Self> {
839        let queue = Queue::new(channel)?;
840        let rescan_notification = Arc::new(RescanNotification::default());
841        controller.add_rescan_notification(rescan_notification.clone());
842
843        let max_io_queue_depth = io_queue_depth.max(1) as usize;
844        Ok(Self {
845            inner: WorkerInner {
846                protocol,
847                request_size: storvsp_protocol::SCSI_REQUEST_LEN_V1,
848                controller: controller.clone(),
849                channel_index,
850                scsi_queue: Arc::new(ScsiCommandQueue {
851                    controller,
852                    mem,
853                    force_path_id,
854                }),
855                scsi_requests: FuturesUnordered::new(),
856                scsi_requests_states: Slab::with_capacity(max_io_queue_depth),
857                channel_control,
858                max_io_queue_depth,
859                future_pool: Vec::new(),
860                full_request_pool: Vec::new(),
861                stats: Default::default(),
862            },
863            queue,
864            rescan_notification,
865        })
866    }
867
868    async fn wait_for_scsi_requests_complete(&mut self) {
869        tracing::debug!(
870            channel_index = self.inner.channel_index,
871            "wait for IOs completed..."
872        );
873        while let Some((id, _, _)) = self.inner.scsi_requests.next().await {
874            self.inner.scsi_requests_states.remove(id);
875        }
876    }
877}
878
879impl InspectTask<Worker> for WorkerState {
880    fn inspect(&self, req: inspect::Request<'_>, worker: Option<&Worker>) {
881        if let Some(worker) = worker {
882            let mut resp = req.respond();
883            if worker.inner.channel_index == 0 {
884                let (state, version, subchannel_count) = match *worker.inner.protocol.state.read() {
885                    ProtocolState::Init(state) => match state {
886                        InitState::Begin => ("begin_init", None, None),
887                        InitState::QueryVersion => ("query_version", None, None),
888                        InitState::QueryProperties { version } => {
889                            ("query_properties", Some(version), None)
890                        }
891                        InitState::EndInitialization {
892                            version,
893                            subchannel_count,
894                        } => ("end_init", Some(version), subchannel_count),
895                    },
896                    ProtocolState::Ready {
897                        version,
898                        subchannel_count,
899                    } => ("ready", Some(version), Some(subchannel_count)),
900                };
901                resp.field("state", state)
902                    .field("version", version)
903                    .field("subchannel_count", subchannel_count);
904            }
905            resp.field("pending_packets", worker.inner.scsi_requests_states.len())
906                .fields("io", worker.inner.scsi_requests_states.iter())
907                .field("stats", &worker.inner.stats)
908                .field("ring", &worker.queue)
909                .field("max_io_queue_depth", worker.inner.max_io_queue_depth);
910        }
911    }
912}
913
914impl<T: 'static + Send + Sync + RingMem> AsyncRun<Worker<T>> for WorkerState {
915    async fn run(
916        &mut self,
917        stop: &mut StopTask<'_>,
918        worker: &mut Worker<T>,
919    ) -> Result<(), task_control::Cancelled> {
920        let check = stop.check_handle()?;
921        let fut = async {
922            if worker.inner.channel_index == 0 {
923                worker.process_primary(&check).await
924            } else {
925                // Wait for initialization to end before processing any
926                // subchannel packets.
927                let protocol_version = loop {
928                    check.check()?;
929                    let listener = worker.inner.protocol.ready.listen();
930                    if let ProtocolState::Ready { version, .. } =
931                        *worker.inner.protocol.state.read()
932                    {
933                        break version;
934                    }
935                    tracing::debug!("subchannel waiting for initialization to end");
936                    listener.await
937                };
938                worker
939                    .inner
940                    .process_ready(&mut worker.queue, protocol_version, &check)
941                    .await
942            }
943        };
944
945        match stop.until_stopped(fut).await? {
946            Err(WorkerError::Cancelled(cancelled)) => return Err(cancelled),
947            Ok(_) => {}
948            Err(e) => tracing::error!(error = e.as_error(), "process_packets error"),
949        }
950        Ok(())
951    }
952}
953
954impl WorkerInner {
955    /// Awaits the next incoming packet, without checking for any other events (device add/remove notifications or available completions).
956    /// Increments the count of outstanding packets when returning `Ok(Packet)`.
957    async fn next_packet<'a, M: RingMem>(
958        &mut self,
959        reader: &'a mut queue::ReadHalf<'a, M>,
960    ) -> Result<Packet, WorkerError> {
961        let packet = reader.read().await.map_err(WorkerError::Queue)?;
962        let stor_packet =
963            parse_packet(&packet, &mut self.full_request_pool).map_err(WorkerError::PacketError)?;
964        Ok(stor_packet)
965    }
966
967    /// Polls for enough ring space in the outgoing ring to send a packet.
968    ///
969    /// This is used to ensure there is enough space in the ring before
970    /// committing to sending a packet. This avoids the need to save pending
971    /// packets on the side if queue processing is interrupted while the ring is
972    /// full.
973    fn poll_for_ring_space<M: RingMem>(
974        &mut self,
975        cx: &mut Context<'_>,
976        writer: &mut queue::WriteHalf<'_, M>,
977    ) -> Poll<Result<(), WorkerError>> {
978        writer
979            .poll_ready(cx, MAX_VMBUS_PACKET_SIZE)
980            .map_err(WorkerError::Queue)
981    }
982}
983
984const MAX_VMBUS_PACKET_SIZE: usize = ring::PacketSize::in_band(
985    size_of::<storvsp_protocol::Packet>() + storvsp_protocol::SCSI_REQUEST_LEN_MAX,
986);
987
988impl<T: RingMem> Worker<T> {
989    /// Processes the protocol state machine.
990    async fn process_primary(
991        &mut self,
992        check: &task_control::StopCheck<'_>,
993    ) -> Result<(), WorkerError> {
994        loop {
995            check.check()?;
996            let current_state = *self.inner.protocol.state.read();
997            match current_state {
998                ProtocolState::Ready { version, .. } => {
999                    self.inner.request_size = version.max_request_size();
1000                    let mut rescan = self.rescan_notification.receiver();
1001                    break poll_fn(|cx| {
1002                        loop {
1003                            check.check()?;
1004                            if let Poll::Ready(r) =
1005                                self.inner.poll_process_ready(cx, &mut self.queue, check)
1006                            {
1007                                return Poll::Ready(r);
1008                            }
1009                            if rescan.poll(cx).is_pending() {
1010                                return Poll::Pending;
1011                            }
1012                            if version >= Version::Win7 {
1013                                tracing::debug!(
1014                                    "rescan notification received, sending ENUMERATE_BUS"
1015                                );
1016                                self.inner.send_packet(
1017                                    &mut self.queue.split().1,
1018                                    storvsp_protocol::Operation::ENUMERATE_BUS,
1019                                    storvsp_protocol::NtStatus::SUCCESS,
1020                                    &(),
1021                                )?;
1022                            }
1023                        }
1024                    })
1025                    .await;
1026                }
1027                ProtocolState::Init(state) => {
1028                    let (mut reader, mut writer) = self.queue.split();
1029
1030                    // Ensure that subsequent calls to `send_completion` won't
1031                    // fail due to lack of ring space, to avoid keeping (and saving/restoring) interim states.
1032                    poll_fn(|cx| self.inner.poll_for_ring_space(cx, &mut writer)).await?;
1033
1034                    tracing::debug!(?state, "process_primary");
1035                    match state {
1036                        InitState::Begin => {
1037                            let packet = self.inner.next_packet(&mut reader).await?;
1038                            if let PacketData::BeginInitialization = packet.data {
1039                                self.inner.send_completion(
1040                                    &mut writer,
1041                                    &packet,
1042                                    storvsp_protocol::NtStatus::SUCCESS,
1043                                    &(),
1044                                )?;
1045                                *self.inner.protocol.state.write() =
1046                                    ProtocolState::Init(InitState::QueryVersion);
1047                            } else {
1048                                tracelimit::warn_ratelimited!(?state, data = ?packet.data, "unexpected packet order");
1049                                self.inner.send_completion(
1050                                    &mut writer,
1051                                    &packet,
1052                                    storvsp_protocol::NtStatus::INVALID_DEVICE_STATE,
1053                                    &(),
1054                                )?;
1055                            }
1056                        }
1057                        InitState::QueryVersion => {
1058                            let packet = self.inner.next_packet(&mut reader).await?;
1059                            if let PacketData::QueryProtocolVersion(major_minor) = packet.data {
1060                                if let Ok(version) = Version::parse(major_minor) {
1061                                    self.inner.send_completion(
1062                                        &mut writer,
1063                                        &packet,
1064                                        storvsp_protocol::NtStatus::SUCCESS,
1065                                        &storvsp_protocol::ProtocolVersion {
1066                                            major_minor,
1067                                            reserved: 0,
1068                                        },
1069                                    )?;
1070                                    self.inner.request_size = version.max_request_size();
1071                                    *self.inner.protocol.state.write() =
1072                                        ProtocolState::Init(InitState::QueryProperties { version });
1073
1074                                    tracelimit::info_ratelimited!(
1075                                        ?version,
1076                                        "scsi version negotiated"
1077                                    );
1078                                } else {
1079                                    self.inner.send_completion(
1080                                        &mut writer,
1081                                        &packet,
1082                                        storvsp_protocol::NtStatus::REVISION_MISMATCH,
1083                                        &storvsp_protocol::ProtocolVersion {
1084                                            major_minor,
1085                                            reserved: 0,
1086                                        },
1087                                    )?;
1088                                    *self.inner.protocol.state.write() =
1089                                        ProtocolState::Init(InitState::QueryVersion);
1090                                }
1091                            } else {
1092                                tracelimit::warn_ratelimited!(?state, data = ?packet.data, "unexpected packet order");
1093                                self.inner.send_completion(
1094                                    &mut writer,
1095                                    &packet,
1096                                    storvsp_protocol::NtStatus::INVALID_DEVICE_STATE,
1097                                    &(),
1098                                )?;
1099                            }
1100                        }
1101                        InitState::QueryProperties { version } => {
1102                            let packet = self.inner.next_packet(&mut reader).await?;
1103                            if let PacketData::QueryProperties = packet.data {
1104                                let multi_channel_supported = version >= Version::Win8;
1105
1106                                self.inner.send_completion(
1107                                    &mut writer,
1108                                    &packet,
1109                                    storvsp_protocol::NtStatus::SUCCESS,
1110                                    &storvsp_protocol::ChannelProperties {
1111                                        max_transfer_bytes: 0x40000, // 256KB
1112                                        flags: {
1113                                            if multi_channel_supported {
1114                                                storvsp_protocol::STORAGE_CHANNEL_SUPPORTS_MULTI_CHANNEL
1115                                            } else {
1116                                                0
1117                                            }
1118                                        },
1119                                        maximum_sub_channel_count: if multi_channel_supported {
1120                                            self.inner.channel_control.max_subchannels()
1121                                        } else {
1122                                            0
1123                                        },
1124                                        reserved: 0,
1125                                        reserved2: 0,
1126                                        reserved3: [0, 0],
1127                                    },
1128                                )?;
1129                                *self.inner.protocol.state.write() =
1130                                    ProtocolState::Init(InitState::EndInitialization {
1131                                        version,
1132                                        subchannel_count: if multi_channel_supported {
1133                                            None
1134                                        } else {
1135                                            Some(0)
1136                                        },
1137                                    });
1138                            } else {
1139                                tracelimit::warn_ratelimited!(?state, data = ?packet.data, "unexpected packet order");
1140                                self.inner.send_completion(
1141                                    &mut writer,
1142                                    &packet,
1143                                    storvsp_protocol::NtStatus::INVALID_DEVICE_STATE,
1144                                    &(),
1145                                )?;
1146                            }
1147                        }
1148                        InitState::EndInitialization {
1149                            version,
1150                            subchannel_count,
1151                        } => {
1152                            let packet = self.inner.next_packet(&mut reader).await?;
1153                            match packet.data {
1154                                PacketData::CreateSubChannels(sub_channel_count)
1155                                    if subchannel_count.is_none() =>
1156                                {
1157                                    if let Err(err) = self
1158                                        .inner
1159                                        .channel_control
1160                                        .enable_subchannels(sub_channel_count)
1161                                    {
1162                                        tracelimit::warn_ratelimited!(
1163                                            ?err,
1164                                            "cannot enable subchannels"
1165                                        );
1166                                        self.inner.send_completion(
1167                                            &mut writer,
1168                                            &packet,
1169                                            storvsp_protocol::NtStatus::INVALID_PARAMETER,
1170                                            &(),
1171                                        )?;
1172                                    } else {
1173                                        self.inner.send_completion(
1174                                            &mut writer,
1175                                            &packet,
1176                                            storvsp_protocol::NtStatus::SUCCESS,
1177                                            &(),
1178                                        )?;
1179                                        *self.inner.protocol.state.write() =
1180                                            ProtocolState::Init(InitState::EndInitialization {
1181                                                version,
1182                                                subchannel_count: Some(sub_channel_count),
1183                                            });
1184                                    }
1185                                }
1186                                PacketData::EndInitialization => {
1187                                    self.inner.send_completion(
1188                                        &mut writer,
1189                                        &packet,
1190                                        storvsp_protocol::NtStatus::SUCCESS,
1191                                        &(),
1192                                    )?;
1193                                    // Reset the rescan notification event now, before the guest has a
1194                                    // chance to send any SCSI requests to scan the bus.
1195                                    self.rescan_notification.reset();
1196                                    *self.inner.protocol.state.write() = ProtocolState::Ready {
1197                                        version,
1198                                        subchannel_count: subchannel_count.unwrap_or(0),
1199                                    };
1200                                    // Wake up subchannels waiting for the
1201                                    // protocol state to become ready.
1202                                    self.inner.protocol.ready.notify(usize::MAX);
1203                                }
1204                                _ => {
1205                                    tracelimit::warn_ratelimited!(?state, data = ?packet.data, "unexpected packet order");
1206                                    self.inner.send_completion(
1207                                        &mut writer,
1208                                        &packet,
1209                                        storvsp_protocol::NtStatus::INVALID_DEVICE_STATE,
1210                                        &(),
1211                                    )?;
1212                                }
1213                            }
1214                        }
1215                    }
1216                }
1217            }
1218        }
1219    }
1220}
1221
1222fn convert_srb_status_to_nt_status(srb_status: SrbStatus) -> storvsp_protocol::NtStatus {
1223    match srb_status {
1224        SrbStatus::BUSY => storvsp_protocol::NtStatus::DEVICE_BUSY,
1225        SrbStatus::SUCCESS => storvsp_protocol::NtStatus::SUCCESS,
1226        SrbStatus::INVALID_LUN
1227        | SrbStatus::INVALID_TARGET_ID
1228        | SrbStatus::NO_DEVICE
1229        | SrbStatus::NO_HBA => storvsp_protocol::NtStatus::DEVICE_DOES_NOT_EXIST,
1230        SrbStatus::COMMAND_TIMEOUT | SrbStatus::TIMEOUT => storvsp_protocol::NtStatus::IO_TIMEOUT,
1231        SrbStatus::SELECTION_TIMEOUT => storvsp_protocol::NtStatus::DEVICE_NOT_CONNECTED,
1232        SrbStatus::BAD_FUNCTION | SrbStatus::BAD_SRB_BLOCK_LENGTH => {
1233            storvsp_protocol::NtStatus::INVALID_DEVICE_REQUEST
1234        }
1235        SrbStatus::DATA_OVERRUN => storvsp_protocol::NtStatus::BUFFER_OVERFLOW,
1236        SrbStatus::REQUEST_FLUSHED => storvsp_protocol::NtStatus::UNSUCCESSFUL,
1237        SrbStatus::ABORTED => storvsp_protocol::NtStatus::CANCELLED,
1238        _ => storvsp_protocol::NtStatus::IO_DEVICE_ERROR,
1239    }
1240}
1241
1242impl WorkerInner {
1243    /// Processes packets and SCSI completions after protocol negotiation has finished.
1244    async fn process_ready<M: RingMem>(
1245        &mut self,
1246        queue: &mut Queue<M>,
1247        protocol_version: Version,
1248        check: &task_control::StopCheck<'_>,
1249    ) -> Result<(), WorkerError> {
1250        self.request_size = protocol_version.max_request_size();
1251        poll_fn(|cx| self.poll_process_ready(cx, queue, check)).await
1252    }
1253
1254    /// Processes packets and SCSI completions after protocol negotiation has finished.
1255    fn poll_process_ready<M: RingMem>(
1256        &mut self,
1257        cx: &mut Context<'_>,
1258        queue: &mut Queue<M>,
1259        check: &task_control::StopCheck<'_>,
1260    ) -> Poll<Result<(), WorkerError>> {
1261        self.stats.wakes.increment();
1262
1263        let (mut reader, mut writer) = queue.split();
1264        let mut total_completions = 0;
1265        let mut total_submissions = 0;
1266        let poll_mode_queue_depth = self.controller.poll_mode_queue_depth.load(Relaxed) as usize;
1267
1268        let cancelled = 'processing: loop {
1269            if let Err(cancelled) = check.check() {
1270                break Some(cancelled);
1271            }
1272            // Drive IOs forward and collect completions.
1273            'outer: while !self.scsi_requests_states.is_empty() {
1274                {
1275                    let mut batch = writer.batched();
1276                    loop {
1277                        // Ensure there is room for the completion before consuming
1278                        // the IO so that we don't have to track completed IOs whose
1279                        // completions haven't been sent.
1280                        if !batch
1281                            .can_write(MAX_VMBUS_PACKET_SIZE)
1282                            .map_err(WorkerError::Queue)?
1283                        {
1284                            // This batch is full but there may still be more completions.
1285                            break;
1286                        }
1287                        if let Poll::Ready(Some((request_id, result, future))) =
1288                            self.scsi_requests.poll_next_unpin(cx)
1289                        {
1290                            self.future_pool.push(future);
1291                            self.handle_completion(&mut batch, request_id, result)?;
1292                            total_completions += 1;
1293                        } else {
1294                            tracing::trace!("out of completions");
1295                            break 'outer;
1296                        }
1297                    }
1298                }
1299
1300                // Wait for enough space for any completion packets.
1301                if self.poll_for_ring_space(cx, &mut writer).is_pending() {
1302                    tracing::trace!("out of ring space");
1303                    break;
1304                }
1305            }
1306
1307            let previous_submissions = total_submissions;
1308            // Process new requests.
1309            'outer: loop {
1310                if let Err(cancelled) = check.check() {
1311                    break 'processing Some(cancelled);
1312                }
1313                if self.scsi_requests_states.len() >= self.max_io_queue_depth {
1314                    break;
1315                }
1316                let mut batch = if self.scsi_requests_states.len() < poll_mode_queue_depth {
1317                    if let Poll::Ready(batch) = reader.poll_read_batch(cx) {
1318                        batch.map_err(WorkerError::Queue)?
1319                    } else {
1320                        tracing::trace!("out of incoming packets");
1321                        break;
1322                    }
1323                } else {
1324                    match reader.try_read_batch() {
1325                        Ok(batch) => batch,
1326                        Err(queue::TryReadError::Empty) => {
1327                            tracing::trace!(
1328                                pending_io_count = self.scsi_requests_states.len(),
1329                                "out of incoming packets, keeping interrupts masked"
1330                            );
1331                            break;
1332                        }
1333                        Err(queue::TryReadError::Queue(err)) => Err(WorkerError::Queue(err))?,
1334                    }
1335                };
1336
1337                let mut packets = batch.packets();
1338                loop {
1339                    if self.scsi_requests_states.len() >= self.max_io_queue_depth {
1340                        break 'outer;
1341                    }
1342                    // Wait for enough space for any completion packets that
1343                    // `handle_packet` may need to send, so that it isn't necessary
1344                    // to track pending completions.
1345                    if self.poll_for_ring_space(cx, &mut writer).is_pending() {
1346                        tracing::trace!("out of ring space");
1347                        break 'outer;
1348                    }
1349
1350                    let packet = if let Some(packet) = packets.next() {
1351                        packet.map_err(WorkerError::Queue)?
1352                    } else {
1353                        break;
1354                    };
1355
1356                    if self.handle_packet(&mut writer, &packet)? {
1357                        total_submissions += 1;
1358                    }
1359                }
1360            }
1361
1362            // Loop around to poll the IOs if any new IOs were submitted.
1363            if total_submissions == previous_submissions {
1364                // No need to poll again.
1365                break None;
1366            }
1367        };
1368
1369        // Account for completed batches before propagating cancellation.
1370        if total_submissions != 0 || total_completions != 0 {
1371            self.stats.ios_submitted.add(total_submissions);
1372            self.stats
1373                .per_wake_submissions
1374                .add_sample(total_submissions);
1375            self.stats
1376                .per_wake_completions
1377                .add_sample(total_completions);
1378            self.stats.ios_completed.add(total_completions);
1379        } else {
1380            self.stats.wakes_spurious.increment();
1381        }
1382
1383        if let Some(cancelled) = cancelled {
1384            return Poll::Ready(Err(cancelled.into()));
1385        }
1386        Poll::Pending
1387    }
1388
1389    fn handle_completion<M: RingMem>(
1390        &mut self,
1391        writer: &mut queue::WriteBatch<'_, M>,
1392        request_id: usize,
1393        result: ScsiResult,
1394    ) -> Result<(), WorkerError> {
1395        let state = self.scsi_requests_states.remove(request_id);
1396        let request_size = state.request.request_size;
1397
1398        // Push the request into the pool to avoid reallocating later.
1399        assert_eq!(
1400            Arc::strong_count(&state.request) + Arc::weak_count(&state.request),
1401            1
1402        );
1403        self.full_request_pool.push(state.request);
1404
1405        let status = convert_srb_status_to_nt_status(result.srb_status);
1406        let mut payload = [0; 0x14];
1407        if let Some(sense) = result.sense_data {
1408            payload[..size_of_val(&sense)].copy_from_slice(sense.as_bytes());
1409            tracing::trace!(sense_info = ?payload, sense_key = payload[2], asc = payload[12], "execute_scsi");
1410        };
1411        let response = storvsp_protocol::ScsiRequest {
1412            length: size_of::<storvsp_protocol::ScsiRequest>() as u16,
1413            scsi_status: result.scsi_status,
1414            srb_status: SrbStatusAndFlags::new()
1415                .with_status(result.srb_status)
1416                .with_autosense_valid(result.sense_data.is_some()),
1417            data_transfer_length: result.tx as u32,
1418            cdb_length: storvsp_protocol::CDB16GENERIC_LENGTH as u8,
1419            sense_info_ex_length: storvsp_protocol::VMSCSI_SENSE_BUFFER_SIZE as u8,
1420            payload,
1421            ..storvsp_protocol::ScsiRequest::new_zeroed()
1422        };
1423        self.send_vmbus_packet(
1424            writer,
1425            OutgoingPacketType::Completion,
1426            request_size,
1427            state.transaction_id,
1428            storvsp_protocol::Operation::COMPLETE_IO,
1429            status,
1430            response.as_bytes(),
1431        )?;
1432        Ok(())
1433    }
1434
1435    fn handle_packet<M: RingMem>(
1436        &mut self,
1437        writer: &mut queue::WriteHalf<'_, M>,
1438        packet: &IncomingPacket<'_, M>,
1439    ) -> Result<bool, WorkerError> {
1440        let packet =
1441            parse_packet(packet, &mut self.full_request_pool).map_err(WorkerError::PacketError)?;
1442        let submitted_io = match packet.data {
1443            PacketData::ExecuteScsi(request) => {
1444                self.push_scsi_request(packet.transaction_id, request);
1445                true
1446            }
1447            PacketData::ResetAdapter | PacketData::ResetBus | PacketData::ResetLun => {
1448                // These operations have always been no-ops.
1449                self.send_completion(writer, &packet, storvsp_protocol::NtStatus::SUCCESS, &())?;
1450                false
1451            }
1452            PacketData::CreateSubChannels(new_subchannel_count) if self.channel_index == 0 => {
1453                if let Err(err) = self
1454                    .channel_control
1455                    .enable_subchannels(new_subchannel_count)
1456                {
1457                    tracelimit::warn_ratelimited!(?err, "cannot create subchannels");
1458                    self.send_completion(
1459                        writer,
1460                        &packet,
1461                        storvsp_protocol::NtStatus::INVALID_PARAMETER,
1462                        &(),
1463                    )?;
1464                    false
1465                } else {
1466                    // Update the subchannel count in the protocol state for save.
1467                    if let ProtocolState::Ready {
1468                        subchannel_count, ..
1469                    } = &mut *self.protocol.state.write()
1470                    {
1471                        *subchannel_count = new_subchannel_count;
1472                    } else {
1473                        unreachable!()
1474                    }
1475
1476                    self.send_completion(
1477                        writer,
1478                        &packet,
1479                        storvsp_protocol::NtStatus::SUCCESS,
1480                        &(),
1481                    )?;
1482                    false
1483                }
1484            }
1485            _ => {
1486                tracelimit::warn_ratelimited!(data = ?packet.data, "unexpected packet on ready");
1487                self.send_completion(
1488                    writer,
1489                    &packet,
1490                    storvsp_protocol::NtStatus::INVALID_DEVICE_STATE,
1491                    &(),
1492                )?;
1493                false
1494            }
1495        };
1496        Ok(submitted_io)
1497    }
1498
1499    fn push_scsi_request(&mut self, transaction_id: u64, full_request: Arc<ScsiRequestAndRange>) {
1500        let scsi_queue = self.scsi_queue.clone();
1501        let scsi_request_state = ScsiRequestState {
1502            transaction_id,
1503            request: full_request.clone(),
1504        };
1505        let request_id = self.scsi_requests_states.insert(scsi_request_state);
1506        let future = self
1507            .future_pool
1508            .pop()
1509            .unwrap_or_else(|| OversizedBox::new(()));
1510        let future = OversizedBox::refill(future, async move {
1511            scsi_queue.execute_scsi(full_request.as_ref()).await
1512        });
1513        let request = ScsiRequest::new(request_id, oversized_box::coerce!(future));
1514        self.scsi_requests.push(request);
1515    }
1516}
1517
1518impl<T: RingMem> Drop for Worker<T> {
1519    fn drop(&mut self) {
1520        self.inner
1521            .controller
1522            .remove_rescan_notification(&self.rescan_notification);
1523    }
1524}
1525
1526#[derive(Debug, Error)]
1527#[error("SCSI path {}:{}:{} is already in use", self.0.path, self.0.target, self.0.lun)]
1528pub struct ScsiPathInUse(pub ScsiPath);
1529
1530#[derive(Debug, Error)]
1531#[error("SCSI path {}:{}:{} is not in use", self.0.path, self.0.target, self.0.lun)]
1532pub struct ScsiPathNotInUse(ScsiPath);
1533
1534#[derive(Clone)]
1535struct ScsiRequestState {
1536    transaction_id: u64,
1537    request: Arc<ScsiRequestAndRange>,
1538}
1539
1540#[derive(Debug)]
1541struct ScsiRequestAndRange {
1542    external_data: Range,
1543    external_data_buf: MultiPagedRangeBuf,
1544    request: storvsp_protocol::ScsiRequest,
1545    request_size: usize,
1546}
1547
1548impl Inspect for ScsiRequestState {
1549    fn inspect(&self, req: inspect::Request<'_>) {
1550        req.respond()
1551            .field("transaction_id", self.transaction_id)
1552            .display(
1553                "address",
1554                &ScsiPath {
1555                    path: self.request.request.path_id,
1556                    target: self.request.request.target_id,
1557                    lun: self.request.request.lun,
1558                },
1559            )
1560            .display_debug("operation", &ScsiOp(self.request.request.payload[0]));
1561    }
1562}
1563
1564impl StorageDevice {
1565    /// Returns a new SCSI device.
1566    pub fn build_scsi(
1567        driver_source: &VmTaskDriverSource,
1568        controller: &ScsiController,
1569        instance_id: Guid,
1570        max_sub_channel_count: u16,
1571        io_queue_depth: u32,
1572    ) -> Self {
1573        Self::build_inner(
1574            driver_source,
1575            controller,
1576            instance_id,
1577            None,
1578            max_sub_channel_count,
1579            io_queue_depth,
1580        )
1581    }
1582
1583    /// Returns a new SCSI device for implementing an IDE accelerator channel
1584    /// for IDE device `device_id` on channel `channel_id`.
1585    pub fn build_ide(
1586        driver_source: &VmTaskDriverSource,
1587        channel_id: u8,
1588        device_id: u8,
1589        disk: ScsiControllerDisk,
1590        io_queue_depth: u32,
1591    ) -> Self {
1592        let path = ScsiPath {
1593            path: channel_id,
1594            target: device_id,
1595            lun: 0,
1596        };
1597
1598        let controller = ScsiController::new();
1599        controller.attach(path, disk).unwrap();
1600
1601        // Construct the specific GUID that drivers in the guest expect for this
1602        // IDE device.
1603        let instance_id = Guid {
1604            data1: channel_id.into(),
1605            data2: device_id.into(),
1606            data3: 0x8899,
1607            data4: [0; 8],
1608        };
1609        Self::build_inner(
1610            driver_source,
1611            &controller,
1612            instance_id,
1613            Some(path),
1614            0,
1615            io_queue_depth,
1616        )
1617    }
1618
1619    fn build_inner(
1620        driver_source: &VmTaskDriverSource,
1621        controller: &ScsiController,
1622        instance_id: Guid,
1623        ide_path: Option<ScsiPath>,
1624        max_sub_channel_count: u16,
1625        io_queue_depth: u32,
1626    ) -> Self {
1627        let workers = (0..max_sub_channel_count + 1)
1628            .map(|channel_index| WorkerAndDriver {
1629                worker: TaskControl::new(WorkerState),
1630                driver: driver_source
1631                    .builder()
1632                    .target_vp(0)
1633                    .run_on_target(true)
1634                    .build(format!("storvsp-{}-{}", instance_id, channel_index)),
1635            })
1636            .collect();
1637
1638        Self {
1639            instance_id,
1640            ide_path,
1641            workers,
1642            controller: controller.state.clone(),
1643            resources: Default::default(),
1644            max_sub_channel_count,
1645            driver_source: driver_source.clone(),
1646            protocol: Arc::new(Protocol {
1647                state: RwLock::new(ProtocolState::Init(InitState::Begin)),
1648                ready: Default::default(),
1649            }),
1650            io_queue_depth,
1651        }
1652    }
1653
1654    fn new_worker(
1655        &mut self,
1656        open_request: &OpenRequest,
1657        channel_index: u16,
1658    ) -> anyhow::Result<&mut Worker> {
1659        let controller = self.controller.clone();
1660
1661        // VMBus doesn't provide a target VP if the channel is not using interrupts. Run on VP 0 in
1662        // that case.
1663        let target_vp = open_request.open_data.target_vp.unwrap_or_default();
1664        let driver = self
1665            .driver_source
1666            .builder()
1667            .target_vp(target_vp)
1668            .run_on_target(true)
1669            .build(format!("storvsp-{}-{}", self.instance_id, channel_index));
1670
1671        let channel = gpadl_channel(&driver, &self.resources, open_request, channel_index)
1672            .context("failed to create vmbus channel")?;
1673
1674        let channel_control = self.resources.channel_control.clone();
1675
1676        tracing::debug!(
1677            target_vp = open_request.open_data.target_vp,
1678            channel_index,
1679            "packet processing starting...",
1680        );
1681
1682        // Force the path ID on incoming SCSI requests to match the IDE
1683        // channel ID, since guests do not reliably set the path ID
1684        // correctly.
1685        let force_path_id = self.ide_path.map(|p| p.path);
1686
1687        let worker = Worker::new(
1688            controller,
1689            channel,
1690            channel_index,
1691            self.resources
1692                .offer_resources
1693                .guest_memory(open_request)
1694                .clone(),
1695            channel_control,
1696            self.io_queue_depth,
1697            self.protocol.clone(),
1698            force_path_id,
1699        )
1700        .map_err(RestoreError::Other)?;
1701
1702        self.workers[channel_index as usize]
1703            .driver
1704            .retarget_vp(target_vp);
1705
1706        Ok(self.workers[channel_index as usize].worker.insert(
1707            &driver,
1708            format!("storvsp worker {}-{}", self.instance_id, channel_index),
1709            worker,
1710        ))
1711    }
1712}
1713
1714/// A disk that can be added to a SCSI controller.
1715#[derive(Clone)]
1716pub struct ScsiControllerDisk {
1717    disk: Arc<dyn AsyncScsiDisk>,
1718}
1719
1720impl ScsiControllerDisk {
1721    /// Creates a new controller disk from an async SCSI disk.
1722    pub fn new(disk: Arc<dyn AsyncScsiDisk>) -> Self {
1723        Self { disk }
1724    }
1725}
1726
1727struct ScsiControllerState {
1728    disks: RwLock<HashMap<ScsiPath, ScsiControllerDisk>>,
1729    rescan_notifications: Mutex<Vec<Arc<RescanNotification>>>,
1730    poll_mode_queue_depth: AtomicU32,
1731}
1732
1733impl ScsiControllerState {
1734    /// Tells every attached device that the initiator it was talking to is gone.
1735    fn clear_nexus_state(&self) {
1736        for disk in self.disks.read().values() {
1737            disk.disk.clear_nexus_state();
1738        }
1739    }
1740}
1741
1742pub struct ScsiController {
1743    state: Arc<ScsiControllerState>,
1744}
1745
1746impl ScsiController {
1747    pub fn new() -> Self {
1748        Self::new_with_poll_mode_queue_depth(None)
1749    }
1750
1751    pub fn new_with_poll_mode_queue_depth(poll_mode_queue_depth: Option<u32>) -> Self {
1752        Self {
1753            state: Arc::new(ScsiControllerState {
1754                disks: Default::default(),
1755                rescan_notifications: Mutex::new(Vec::new()),
1756                poll_mode_queue_depth: AtomicU32::new(
1757                    poll_mode_queue_depth.unwrap_or(DEFAULT_POLL_MODE_QUEUE_DEPTH),
1758                ),
1759            }),
1760        }
1761    }
1762
1763    pub fn attach(&self, path: ScsiPath, disk: ScsiControllerDisk) -> Result<(), ScsiPathInUse> {
1764        match self.state.disks.write().entry(path) {
1765            Entry::Occupied(_) => return Err(ScsiPathInUse(path)),
1766            Entry::Vacant(entry) => entry.insert(disk),
1767        };
1768        for notification in self.state.rescan_notifications.lock().iter() {
1769            notification.notify();
1770        }
1771        Ok(())
1772    }
1773
1774    pub fn remove(&self, path: ScsiPath) -> Result<(), ScsiPathNotInUse> {
1775        match self.state.disks.write().entry(path) {
1776            Entry::Vacant(_) => return Err(ScsiPathNotInUse(path)),
1777            Entry::Occupied(entry) => {
1778                entry.remove();
1779            }
1780        }
1781        for notification in self.state.rescan_notifications.lock().iter() {
1782            notification.notify();
1783        }
1784        Ok(())
1785    }
1786}
1787
1788impl ScsiControllerState {
1789    fn add_rescan_notification(&self, notification: Arc<RescanNotification>) {
1790        self.rescan_notifications.lock().push(notification);
1791    }
1792
1793    fn remove_rescan_notification(&self, notification: &Arc<RescanNotification>) {
1794        let mut notifications = self.rescan_notifications.lock();
1795        if let Some(index) = notifications
1796            .iter()
1797            .position(|source| Arc::ptr_eq(source, notification))
1798        {
1799            notifications.remove(index);
1800        }
1801    }
1802}
1803
1804#[async_trait]
1805impl VmbusDevice for StorageDevice {
1806    fn offer(&self) -> OfferParams {
1807        if let Some(path) = self.ide_path {
1808            let offer_properties = storvsp_protocol::OfferProperties {
1809                path_id: path.path,
1810                target_id: path.target,
1811                flags: storvsp_protocol::OFFER_PROPERTIES_FLAG_IDE_DEVICE,
1812                ..FromZeros::new_zeroed()
1813            };
1814            let mut user_defined = UserDefinedData::new_zeroed();
1815            offer_properties
1816                .write_to_prefix(&mut user_defined[..])
1817                .unwrap();
1818            OfferParams {
1819                interface_name: "ide-accel".to_owned(),
1820                instance_id: self.instance_id,
1821                interface_id: storvsp_protocol::IDE_ACCELERATOR_INTERFACE_ID,
1822                channel_type: ChannelType::Interface { user_defined },
1823                ..Default::default()
1824            }
1825        } else {
1826            OfferParams {
1827                interface_name: "scsi".to_owned(),
1828                instance_id: self.instance_id,
1829                interface_id: storvsp_protocol::SCSI_INTERFACE_ID,
1830                ..Default::default()
1831            }
1832        }
1833    }
1834
1835    fn max_subchannels(&self) -> u16 {
1836        self.max_sub_channel_count
1837    }
1838
1839    fn install(&mut self, resources: DeviceResources) {
1840        self.resources = resources;
1841    }
1842
1843    async fn open(
1844        &mut self,
1845        channel_index: u16,
1846        open_request: &OpenRequest,
1847    ) -> Result<(), ChannelOpenError> {
1848        tracing::debug!(channel_index, "scsi open channel");
1849        self.new_worker(open_request, channel_index)?;
1850        self.workers[channel_index as usize].worker.start();
1851        Ok(())
1852    }
1853
1854    async fn close(&mut self, channel_index: u16) {
1855        tracing::debug!(channel_index, "scsi close channel");
1856        let worker = &mut self.workers[channel_index as usize].worker;
1857        worker.stop().await;
1858        if worker.state_mut().is_some() {
1859            worker
1860                .state_mut()
1861                .unwrap()
1862                .wait_for_scsi_requests_complete()
1863                .await;
1864            worker.remove();
1865        }
1866        if channel_index == 0 {
1867            *self.protocol.state.write() = ProtocolState::Init(InitState::Begin);
1868            // Closing the primary channel ends the initiator-target nexus, so the
1869            // state scoped to it goes with the protocol state. A machine reset
1870            // arrives here too: the server closes every open channel on the way
1871            // down (vmbus_server's client_release_channel notifies Action::Close),
1872            // and without this the next boot inherits the previous guest's tray
1873            // lock and pending sense.
1874            // Correctness here rests on channel 0 being closed last, which vmbus
1875            // does in both paths: the teardown loop closes subchannels in reverse
1876            // index order, and the close-request path closes 1..len before
1877            // channel 0. Each of those closes stops and drains its worker, so
1878            // none is left to repopulate the state cleared below. `VmbusDevice`
1879            // does not document that order.
1880            self.controller.clear_nexus_state();
1881        }
1882    }
1883
1884    async fn retarget_vp(&mut self, channel_index: u16, target_vp: u32) {
1885        self.workers[channel_index as usize]
1886            .driver
1887            .retarget_vp(target_vp);
1888    }
1889
1890    fn start(&mut self) {
1891        for task in self
1892            .workers
1893            .iter_mut()
1894            .filter(|task| task.worker.has_state() && !task.worker.is_running())
1895        {
1896            task.worker.start();
1897        }
1898    }
1899
1900    async fn stop(&mut self) {
1901        tracing::debug!(instance_id = ?self.instance_id, "StorageDevice stopping...");
1902        for task in self
1903            .workers
1904            .iter_mut()
1905            .filter(|task| task.worker.has_state() && task.worker.is_running())
1906        {
1907            task.worker.stop().await;
1908        }
1909    }
1910
1911    fn supports_save_restore(&mut self) -> Option<&mut dyn SaveRestoreVmbusDevice> {
1912        Some(self)
1913    }
1914}
1915
1916#[async_trait]
1917impl SaveRestoreVmbusDevice for StorageDevice {
1918    async fn save(&mut self) -> Result<SavedStateBlob, SaveError> {
1919        Ok(SavedStateBlob::new(self.save()?))
1920    }
1921
1922    async fn restore(
1923        &mut self,
1924        control: RestoreControl<'_>,
1925        state: SavedStateBlob,
1926    ) -> Result<(), RestoreError> {
1927        self.restore(control, state.parse()?).await
1928    }
1929}
1930
1931#[cfg(test)]
1932mod tests {
1933    use super::*;
1934    use crate::test_helpers::TestWorker;
1935    use crate::test_helpers::parse_guest_completion;
1936    use crate::test_helpers::parse_guest_completion_check_flags_status;
1937    use pal_async::DefaultDriver;
1938    use pal_async::async_test;
1939    use scsi::srb::SrbStatus;
1940    use scsi_buffers::OwnedRequestBuffers;
1941    use std::task::Wake;
1942    use test_with_tracing::test;
1943    use vmbus_channel::connected_async_channels;
1944    use vmcore::vm_task::SingleDriverBackend;
1945    use vmcore::vm_task::VmTaskDriverSource;
1946
1947    // Discourage `Clone` for `ScsiController` outside the crate, but it is
1948    // necessary for testing. The fuzzer also uses `TestWorker`, which needs
1949    // a `clone` of the inner state, but is not in this crate.
1950    impl Clone for ScsiController {
1951        fn clone(&self) -> Self {
1952            ScsiController {
1953                state: self.state.clone(),
1954            }
1955        }
1956    }
1957
1958    #[test]
1959    fn test_rescan_notification_latches_coalesces_and_resets() {
1960        let notification = RescanNotification::default();
1961        let mut receiver = notification.receiver();
1962        let mut cx = Context::from_waker(Waker::noop());
1963        notification.notify();
1964        notification.notify();
1965        assert!(receiver.poll(&mut cx).is_ready());
1966        assert!(receiver.poll(&mut cx).is_pending());
1967
1968        notification.notify();
1969        notification.reset();
1970        assert!(receiver.poll(&mut cx).is_pending());
1971        notification.notify();
1972        assert!(receiver.poll(&mut cx).is_ready());
1973
1974        assert!(receiver.poll(&mut cx).is_pending());
1975        drop(receiver);
1976        notification.notify();
1977        assert!(notification.receiver().poll(&mut cx).is_ready());
1978    }
1979
1980    #[derive(Default)]
1981    struct RescanWakeCount(AtomicU32);
1982
1983    impl Wake for RescanWakeCount {
1984        fn wake(self: Arc<Self>) {
1985            self.0.fetch_add(1, Relaxed);
1986        }
1987    }
1988
1989    #[test]
1990    fn test_rescan_notification_rearms_and_updates_waker() {
1991        let notification = RescanNotification::default();
1992        let first = Arc::new(RescanWakeCount::default());
1993        let second = Arc::new(RescanWakeCount::default());
1994        let first_waker = Waker::from(first.clone());
1995        let second_waker = Waker::from(second.clone());
1996        let mut first_cx = Context::from_waker(&first_waker);
1997        let mut second_cx = Context::from_waker(&second_waker);
1998
1999        let mut receiver = notification.receiver();
2000        assert!(receiver.poll(&mut first_cx).is_pending());
2001        assert!(receiver.poll(&mut first_cx).is_pending());
2002        assert!(receiver.poll(&mut second_cx).is_pending());
2003        notification.notify();
2004        assert_eq!(first.0.load(Relaxed), 0);
2005        assert_eq!(second.0.load(Relaxed), 1);
2006        assert!(receiver.poll(&mut second_cx).is_ready());
2007
2008        assert!(receiver.poll(&mut second_cx).is_pending());
2009        // A delayed wake from the previous notification must not disarm the
2010        // receiver when its pending flag has already been consumed.
2011        let delayed_waker = notification.waker.lock().clone().unwrap();
2012        delayed_waker.wake();
2013        assert!(receiver.poll(&mut second_cx).is_pending());
2014        notification.notify();
2015        assert_eq!(second.0.load(Relaxed), 3);
2016        assert!(receiver.poll(&mut second_cx).is_ready());
2017    }
2018
2019    #[async_test]
2020    async fn test_channel_working(driver: DefaultDriver) {
2021        // set up the channels and worker
2022        let (host, guest) = connected_async_channels(16 * 1024);
2023        let guest_queue = Queue::new(guest).unwrap();
2024
2025        let test_guest_mem = GuestMemory::allocate(16384);
2026        let controller = ScsiController::new();
2027        let disk = scsidisk::SimpleScsiDisk::new(
2028            disklayer_ram::ram_disk(10 * 1024 * 1024, false).unwrap(),
2029            Default::default(),
2030        );
2031        controller
2032            .attach(
2033                ScsiPath {
2034                    path: 0,
2035                    target: 0,
2036                    lun: 0,
2037                },
2038                ScsiControllerDisk::new(Arc::new(disk)),
2039            )
2040            .unwrap();
2041
2042        let test_worker = TestWorker::start(
2043            controller.clone(),
2044            driver.clone(),
2045            test_guest_mem.clone(),
2046            host,
2047            None,
2048        );
2049
2050        let mut guest = test_helpers::TestGuest {
2051            queue: guest_queue,
2052            transaction_id: 0,
2053        };
2054
2055        guest.perform_protocol_negotiation().await;
2056
2057        // Set up the buffer for a write request
2058        const IO_LEN: usize = 4 * 1024;
2059        let write_buf = [7u8; IO_LEN];
2060        let write_gpa = 4 * 1024u64;
2061        test_guest_mem.write_at(write_gpa, &write_buf).unwrap();
2062        guest
2063            .send_write_packet(ScsiPath::default(), write_gpa, 1, IO_LEN)
2064            .await;
2065
2066        guest
2067            .verify_completion(|p| test_helpers::parse_guest_completed_io(p, SrbStatus::SUCCESS))
2068            .await;
2069
2070        let read_gpa = 8 * 1024u64;
2071        guest
2072            .send_read_packet(ScsiPath::default(), read_gpa, 1, IO_LEN)
2073            .await;
2074
2075        guest
2076            .verify_completion(|p| test_helpers::parse_guest_completed_io(p, SrbStatus::SUCCESS))
2077            .await;
2078        let mut read_buf = [0u8; IO_LEN];
2079        test_guest_mem.read_at(read_gpa, &mut read_buf).unwrap();
2080        for (b1, b2) in read_buf.iter().zip(write_buf.iter()) {
2081            assert_eq!(b1, b2);
2082        }
2083
2084        // stop everything
2085        guest.verify_graceful_close(test_worker).await;
2086    }
2087
2088    #[async_test]
2089    async fn test_packet_sizes(driver: DefaultDriver) {
2090        // set up the channels and worker
2091        let (host, guest) = connected_async_channels(16384);
2092        let guest_queue = Queue::new(guest).unwrap();
2093
2094        let test_guest_mem = GuestMemory::allocate(1024);
2095        let controller = ScsiController::new();
2096
2097        let _worker = TestWorker::start(
2098            controller.clone(),
2099            driver.clone(),
2100            test_guest_mem,
2101            host,
2102            None,
2103        );
2104
2105        let mut guest = test_helpers::TestGuest {
2106            queue: guest_queue,
2107            transaction_id: 0,
2108        };
2109
2110        let negotiate_packet = storvsp_protocol::Packet {
2111            operation: storvsp_protocol::Operation::BEGIN_INITIALIZATION,
2112            flags: 0,
2113            status: storvsp_protocol::NtStatus::SUCCESS,
2114        };
2115        guest
2116            .send_data_packet_sync(&[negotiate_packet.as_bytes()])
2117            .await;
2118
2119        guest.verify_completion(parse_guest_completion).await;
2120
2121        let header = storvsp_protocol::Packet {
2122            operation: storvsp_protocol::Operation::QUERY_PROTOCOL_VERSION,
2123            flags: 0,
2124            status: storvsp_protocol::NtStatus::SUCCESS,
2125        };
2126
2127        let mut buf = [0u8; 128];
2128        storvsp_protocol::ProtocolVersion {
2129            major_minor: !0,
2130            reserved: 0,
2131        }
2132        .write_to_prefix(&mut buf[..])
2133        .unwrap(); // PANIC: Infallable since `ProtcolVersion` is less than 128 bytes
2134
2135        for &(len, resp_len) in &[(48, 48), (50, 56), (56, 56), (64, 64), (72, 64)] {
2136            guest
2137                .send_data_packet_sync(&[header.as_bytes(), &buf[..len - size_of_val(&header)]])
2138                .await;
2139
2140            guest
2141                .verify_completion(|packet| {
2142                    let IncomingPacket::Completion(packet) = packet else {
2143                        unreachable!()
2144                    };
2145                    assert_eq!(packet.reader().len(), resp_len);
2146                    assert_eq!(
2147                        packet
2148                            .reader()
2149                            .read_plain::<storvsp_protocol::Packet>()
2150                            .unwrap()
2151                            .status,
2152                        storvsp_protocol::NtStatus::REVISION_MISMATCH
2153                    );
2154                    Ok(())
2155                })
2156                .await;
2157        }
2158    }
2159
2160    #[async_test]
2161    async fn test_wrong_first_packet(driver: DefaultDriver) {
2162        // set up the channels and worker
2163        let (host, guest) = connected_async_channels(16384);
2164        let guest_queue = Queue::new(guest).unwrap();
2165
2166        let test_guest_mem = GuestMemory::allocate(1024);
2167        let controller = ScsiController::new();
2168
2169        let _worker = TestWorker::start(
2170            controller.clone(),
2171            driver.clone(),
2172            test_guest_mem,
2173            host,
2174            None,
2175        );
2176
2177        let mut guest = test_helpers::TestGuest {
2178            queue: guest_queue,
2179            transaction_id: 0,
2180        };
2181
2182        // Protocol negotiation done out of order
2183        let negotiate_packet = storvsp_protocol::Packet {
2184            operation: storvsp_protocol::Operation::END_INITIALIZATION,
2185            flags: 0,
2186            status: storvsp_protocol::NtStatus::SUCCESS,
2187        };
2188        guest
2189            .send_data_packet_sync(&[negotiate_packet.as_bytes()])
2190            .await;
2191
2192        guest
2193            .verify_completion(|packet| {
2194                let IncomingPacket::Completion(packet) = packet else {
2195                    unreachable!()
2196                };
2197                assert_eq!(
2198                    packet
2199                        .reader()
2200                        .read_plain::<storvsp_protocol::Packet>()
2201                        .unwrap()
2202                        .status,
2203                    storvsp_protocol::NtStatus::INVALID_DEVICE_STATE
2204                );
2205                Ok(())
2206            })
2207            .await;
2208    }
2209
2210    #[async_test]
2211    async fn test_unrecognized_operation(driver: DefaultDriver) {
2212        // set up the channels and worker
2213        let (host, guest) = connected_async_channels(16384);
2214        let guest_queue = Queue::new(guest).unwrap();
2215
2216        let test_guest_mem = GuestMemory::allocate(1024);
2217        let controller = ScsiController::new();
2218
2219        let worker = TestWorker::start(
2220            controller.clone(),
2221            driver.clone(),
2222            test_guest_mem,
2223            host,
2224            None,
2225        );
2226
2227        let mut guest = test_helpers::TestGuest {
2228            queue: guest_queue,
2229            transaction_id: 0,
2230        };
2231
2232        // Send packet with unrecognized operation
2233        let negotiate_packet = storvsp_protocol::Packet {
2234            operation: storvsp_protocol::Operation::REMOVE_DEVICE,
2235            flags: 0,
2236            status: storvsp_protocol::NtStatus::SUCCESS,
2237        };
2238        guest
2239            .send_data_packet_sync(&[negotiate_packet.as_bytes()])
2240            .await;
2241
2242        match worker.teardown().await {
2243            Err(WorkerError::PacketError(PacketError::UnrecognizedOperation(
2244                storvsp_protocol::Operation::REMOVE_DEVICE,
2245            ))) => {}
2246            result => panic!("Worker failed with unexpected result {:?}!", result),
2247        }
2248    }
2249
2250    #[async_test]
2251    async fn test_too_many_subchannels(driver: DefaultDriver) {
2252        // set up the channels and worker
2253        let (host, guest) = connected_async_channels(16384);
2254        let guest_queue = Queue::new(guest).unwrap();
2255
2256        let test_guest_mem = GuestMemory::allocate(1024);
2257        let controller = ScsiController::new();
2258
2259        let _worker = TestWorker::start(
2260            controller.clone(),
2261            driver.clone(),
2262            test_guest_mem,
2263            host,
2264            None,
2265        );
2266
2267        let mut guest = test_helpers::TestGuest {
2268            queue: guest_queue,
2269            transaction_id: 0,
2270        };
2271
2272        let negotiate_packet = storvsp_protocol::Packet {
2273            operation: storvsp_protocol::Operation::BEGIN_INITIALIZATION,
2274            flags: 0,
2275            status: storvsp_protocol::NtStatus::SUCCESS,
2276        };
2277        guest
2278            .send_data_packet_sync(&[negotiate_packet.as_bytes()])
2279            .await;
2280        guest.verify_completion(parse_guest_completion).await;
2281
2282        let version_packet = storvsp_protocol::Packet {
2283            operation: storvsp_protocol::Operation::QUERY_PROTOCOL_VERSION,
2284            flags: 0,
2285            status: storvsp_protocol::NtStatus::SUCCESS,
2286        };
2287        let version = storvsp_protocol::ProtocolVersion {
2288            major_minor: storvsp_protocol::VERSION_BLUE,
2289            reserved: 0,
2290        };
2291        guest
2292            .send_data_packet_sync(&[version_packet.as_bytes(), version.as_bytes()])
2293            .await;
2294        guest.verify_completion(parse_guest_completion).await;
2295
2296        let properties_packet = storvsp_protocol::Packet {
2297            operation: storvsp_protocol::Operation::QUERY_PROPERTIES,
2298            flags: 0,
2299            status: storvsp_protocol::NtStatus::SUCCESS,
2300        };
2301        guest
2302            .send_data_packet_sync(&[properties_packet.as_bytes()])
2303            .await;
2304
2305        guest.verify_completion(parse_guest_completion).await;
2306
2307        let negotiate_packet = storvsp_protocol::Packet {
2308            operation: storvsp_protocol::Operation::CREATE_SUB_CHANNELS,
2309            flags: 0,
2310            status: storvsp_protocol::NtStatus::SUCCESS,
2311        };
2312        // Create sub channels more than maximum_sub_channel_count
2313        guest
2314            .send_data_packet_sync(&[negotiate_packet.as_bytes(), 1_u16.as_bytes()])
2315            .await;
2316
2317        guest
2318            .verify_completion(|packet| {
2319                let IncomingPacket::Completion(packet) = packet else {
2320                    unreachable!()
2321                };
2322                assert_eq!(
2323                    packet
2324                        .reader()
2325                        .read_plain::<storvsp_protocol::Packet>()
2326                        .unwrap()
2327                        .status,
2328                    storvsp_protocol::NtStatus::INVALID_PARAMETER
2329                );
2330                Ok(())
2331            })
2332            .await;
2333    }
2334
2335    #[async_test]
2336    async fn test_begin_init_on_ready(driver: DefaultDriver) {
2337        // set up the channels and worker
2338        let (host, guest) = connected_async_channels(16384);
2339        let guest_queue = Queue::new(guest).unwrap();
2340
2341        let test_guest_mem = GuestMemory::allocate(1024);
2342        let controller = ScsiController::new();
2343
2344        let _worker = TestWorker::start(
2345            controller.clone(),
2346            driver.clone(),
2347            test_guest_mem,
2348            host,
2349            None,
2350        );
2351
2352        let mut guest = test_helpers::TestGuest {
2353            queue: guest_queue,
2354            transaction_id: 0,
2355        };
2356
2357        guest.perform_protocol_negotiation().await;
2358
2359        // Protocol negotiation done out of order
2360        let negotiate_packet = storvsp_protocol::Packet {
2361            operation: storvsp_protocol::Operation::BEGIN_INITIALIZATION,
2362            flags: 0,
2363            status: storvsp_protocol::NtStatus::SUCCESS,
2364        };
2365        guest
2366            .send_data_packet_sync(&[negotiate_packet.as_bytes()])
2367            .await;
2368
2369        guest
2370            .verify_completion(|p| {
2371                parse_guest_completion_check_flags_status(
2372                    p,
2373                    0,
2374                    storvsp_protocol::NtStatus::INVALID_DEVICE_STATE,
2375                )
2376            })
2377            .await;
2378    }
2379
2380    #[async_test]
2381    async fn test_hot_add_remove(driver: DefaultDriver) {
2382        // set up channels and worker.
2383        let (host, guest) = connected_async_channels(16 * 1024);
2384        let guest_queue = Queue::new(guest).unwrap();
2385
2386        let test_guest_mem = GuestMemory::allocate(16384);
2387        // create a controller with no disk yet.
2388        let controller = ScsiController::new();
2389
2390        let test_worker = TestWorker::start(
2391            controller.clone(),
2392            driver.clone(),
2393            test_guest_mem.clone(),
2394            host,
2395            None,
2396        );
2397
2398        let mut guest = test_helpers::TestGuest {
2399            queue: guest_queue,
2400            transaction_id: 0,
2401        };
2402
2403        guest.perform_protocol_negotiation().await;
2404
2405        // Verify no LUNs are reported initially.
2406        let mut lun_list_buffer: [u8; 256] = [0; 256];
2407        let mut disk_count = 0;
2408        guest
2409            .send_report_luns_packet(ScsiPath::default(), 0, lun_list_buffer.len())
2410            .await;
2411        guest
2412            .verify_completion(|p| {
2413                test_helpers::parse_guest_completed_io_check_tx_len(p, SrbStatus::SUCCESS, Some(8))
2414            })
2415            .await;
2416        test_guest_mem.read_at(0, &mut lun_list_buffer).unwrap();
2417        let lun_list_size = u32::from_be_bytes(lun_list_buffer[0..4].try_into().unwrap());
2418        assert_eq!(lun_list_size, disk_count as u32 * 8);
2419
2420        // Set up a buffer for writes.
2421        const IO_LEN: usize = 4 * 1024;
2422        let write_buf = [7u8; IO_LEN];
2423        let write_gpa = 4 * 1024u64;
2424        test_guest_mem.write_at(write_gpa, &write_buf).unwrap();
2425
2426        guest
2427            .send_write_packet(ScsiPath::default(), write_gpa, 1, IO_LEN)
2428            .await;
2429        guest
2430            .verify_completion(|p| {
2431                test_helpers::parse_guest_completed_io(p, SrbStatus::INVALID_LUN)
2432            })
2433            .await;
2434
2435        // Add some disks while the guest is running.
2436        for lun in 0..4 {
2437            let disk = scsidisk::SimpleScsiDisk::new(
2438                disklayer_ram::ram_disk(10 * 1024 * 1024, false).unwrap(),
2439                Default::default(),
2440            );
2441            controller
2442                .attach(
2443                    ScsiPath {
2444                        path: 0,
2445                        target: 0,
2446                        lun,
2447                    },
2448                    ScsiControllerDisk::new(Arc::new(disk)),
2449                )
2450                .unwrap();
2451            guest
2452                .verify_completion(test_helpers::parse_guest_enumerate_bus)
2453                .await;
2454
2455            disk_count += 1;
2456            guest
2457                .send_report_luns_packet(ScsiPath::default(), 0, 256)
2458                .await;
2459            guest
2460                .verify_completion(|p| {
2461                    test_helpers::parse_guest_completed_io_check_tx_len(
2462                        p,
2463                        SrbStatus::SUCCESS,
2464                        Some((disk_count + 1) * 8),
2465                    )
2466                })
2467                .await;
2468            test_guest_mem.read_at(0, &mut lun_list_buffer).unwrap();
2469            let lun_list_size = u32::from_be_bytes(lun_list_buffer[0..4].try_into().unwrap());
2470            assert_eq!(lun_list_size, disk_count as u32 * 8);
2471
2472            guest
2473                .send_write_packet(
2474                    ScsiPath {
2475                        path: 0,
2476                        target: 0,
2477                        lun,
2478                    },
2479                    write_gpa,
2480                    1,
2481                    IO_LEN,
2482                )
2483                .await;
2484            guest
2485                .verify_completion(|p| {
2486                    test_helpers::parse_guest_completed_io(p, SrbStatus::SUCCESS)
2487                })
2488                .await;
2489        }
2490
2491        // Remove all disks while the guest is running.
2492        for lun in 0..4 {
2493            controller
2494                .remove(ScsiPath {
2495                    path: 0,
2496                    target: 0,
2497                    lun,
2498                })
2499                .unwrap();
2500            guest
2501                .verify_completion(test_helpers::parse_guest_enumerate_bus)
2502                .await;
2503
2504            disk_count -= 1;
2505            guest
2506                .send_report_luns_packet(ScsiPath::default(), 0, 4096)
2507                .await;
2508            guest
2509                .verify_completion(|p| {
2510                    test_helpers::parse_guest_completed_io_check_tx_len(
2511                        p,
2512                        SrbStatus::SUCCESS,
2513                        Some((disk_count + 1) * 8),
2514                    )
2515                })
2516                .await;
2517            test_guest_mem.read_at(0, &mut lun_list_buffer).unwrap();
2518            let lun_list_size = u32::from_be_bytes(lun_list_buffer[0..4].try_into().unwrap());
2519            assert_eq!(lun_list_size, disk_count as u32 * 8);
2520
2521            guest
2522                .send_write_packet(
2523                    ScsiPath {
2524                        path: 0,
2525                        target: 0,
2526                        lun,
2527                    },
2528                    write_gpa,
2529                    1,
2530                    IO_LEN,
2531                )
2532                .await;
2533            guest
2534                .verify_completion(|p| {
2535                    test_helpers::parse_guest_completed_io(p, SrbStatus::INVALID_LUN)
2536                })
2537                .await;
2538        }
2539
2540        guest.verify_graceful_close(test_worker).await;
2541    }
2542
2543    #[async_test]
2544    async fn test_async_disk(driver: DefaultDriver) {
2545        let device = disklayer_ram::ram_disk(64 * 1024, false).unwrap();
2546        let controller = ScsiController::new();
2547        let disk = ScsiControllerDisk::new(Arc::new(scsidisk::SimpleScsiDisk::new(
2548            device,
2549            Default::default(),
2550        )));
2551        controller
2552            .attach(
2553                ScsiPath {
2554                    path: 0,
2555                    target: 0,
2556                    lun: 0,
2557                },
2558                disk,
2559            )
2560            .unwrap();
2561
2562        let (host, guest) = connected_async_channels(16 * 1024);
2563        let guest_queue = Queue::new(guest).unwrap();
2564
2565        let mut guest = test_helpers::TestGuest {
2566            queue: guest_queue,
2567            transaction_id: 0,
2568        };
2569
2570        let test_guest_mem = GuestMemory::allocate(16384);
2571        let worker = TestWorker::start(
2572            controller.clone(),
2573            &driver,
2574            test_guest_mem.clone(),
2575            host,
2576            None,
2577        );
2578
2579        let negotiate_packet = storvsp_protocol::Packet {
2580            operation: storvsp_protocol::Operation::BEGIN_INITIALIZATION,
2581            flags: 0,
2582            status: storvsp_protocol::NtStatus::SUCCESS,
2583        };
2584        guest
2585            .send_data_packet_sync(&[negotiate_packet.as_bytes()])
2586            .await;
2587        guest.verify_completion(parse_guest_completion).await;
2588
2589        let version_packet = storvsp_protocol::Packet {
2590            operation: storvsp_protocol::Operation::QUERY_PROTOCOL_VERSION,
2591            flags: 0,
2592            status: storvsp_protocol::NtStatus::SUCCESS,
2593        };
2594        let version = storvsp_protocol::ProtocolVersion {
2595            major_minor: storvsp_protocol::VERSION_BLUE,
2596            reserved: 0,
2597        };
2598        guest
2599            .send_data_packet_sync(&[version_packet.as_bytes(), version.as_bytes()])
2600            .await;
2601        guest.verify_completion(parse_guest_completion).await;
2602
2603        let properties_packet = storvsp_protocol::Packet {
2604            operation: storvsp_protocol::Operation::QUERY_PROPERTIES,
2605            flags: 0,
2606            status: storvsp_protocol::NtStatus::SUCCESS,
2607        };
2608        guest
2609            .send_data_packet_sync(&[properties_packet.as_bytes()])
2610            .await;
2611        guest.verify_completion(parse_guest_completion).await;
2612
2613        let negotiate_packet = storvsp_protocol::Packet {
2614            operation: storvsp_protocol::Operation::END_INITIALIZATION,
2615            flags: 0,
2616            status: storvsp_protocol::NtStatus::SUCCESS,
2617        };
2618        guest
2619            .send_data_packet_sync(&[negotiate_packet.as_bytes()])
2620            .await;
2621        guest.verify_completion(parse_guest_completion).await;
2622
2623        const IO_LEN: usize = 4 * 1024;
2624        let write_buf = [7u8; IO_LEN];
2625        let write_gpa = 4 * 1024u64;
2626        test_guest_mem.write_at(write_gpa, &write_buf).unwrap();
2627        guest
2628            .send_write_packet(ScsiPath::default(), write_gpa, 1, IO_LEN)
2629            .await;
2630        guest
2631            .verify_completion(|p| test_helpers::parse_guest_completed_io(p, SrbStatus::SUCCESS))
2632            .await;
2633
2634        let read_gpa = 8 * 1024u64;
2635        guest
2636            .send_read_packet(ScsiPath::default(), read_gpa, 1, IO_LEN)
2637            .await;
2638        guest
2639            .verify_completion(|p| test_helpers::parse_guest_completed_io(p, SrbStatus::SUCCESS))
2640            .await;
2641        let mut read_buf = [0u8; IO_LEN];
2642        test_guest_mem.read_at(read_gpa, &mut read_buf).unwrap();
2643        for (b1, b2) in read_buf.iter().zip(write_buf.iter()) {
2644            assert_eq!(b1, b2);
2645        }
2646
2647        guest.verify_graceful_close(worker).await;
2648    }
2649    fn make_medium_removal_request(prevent: bool) -> Request {
2650        let cdb = scsi::CdbMediaRemoval {
2651            operation_code: ScsiOp::MEDIUM_REMOVAL,
2652            lun: 0,
2653            reserved: [0; 2],
2654            flags: scsi::MediaRemovalFlags::new()
2655                .with_prevent(prevent)
2656                .with_persistent(false),
2657        };
2658        let mut data = [0u8; 16];
2659        data[..size_of::<scsi::CdbMediaRemoval>()].copy_from_slice(cdb.as_bytes());
2660        Request {
2661            cdb: data,
2662            srb_flags: 0,
2663        }
2664    }
2665
2666    fn make_eject_request() -> Request {
2667        let cdb = scsi::StartStop {
2668            operation_code: ScsiOp::START_STOP_UNIT,
2669            immediate: 0,
2670            reserved2: [0; 2],
2671            flag: scsi::StartStopFlags::new()
2672                .with_start(false)
2673                .with_load_eject(true),
2674            control: 0,
2675        };
2676        let mut data = [0u8; 16];
2677        data[..size_of::<scsi::StartStop>()].copy_from_slice(cdb.as_bytes());
2678        Request {
2679            cdb: data,
2680            srb_flags: 0,
2681        }
2682    }
2683
2684    /// Closing the primary channel is what ends the nexus, so it is what has to
2685    /// be tested.
2686    ///
2687    /// The scsidisk tests call `clear_nexus_state` on the device directly, which
2688    /// says the device clears its own state but nothing about the caller. With
2689    /// only those, deleting the `clear_nexus_state` call in `close()` leaves
2690    /// both of them green while the reopened-channel and machine-reset behaviour
2691    /// comes back. This drives the real `VmbusDevice::close` and observes the
2692    /// result the way an initiator would, through a SCSI command, rather than
2693    /// through the device's internals.
2694    #[async_test]
2695    async fn test_close_ends_the_nexus(driver: DefaultDriver) {
2696        let controller = ScsiController::new();
2697        let dvd = Arc::new(scsidisk::scsidvd::SimpleScsiDvd::new(Some(
2698            disklayer_ram::ram_disk(2 * 1024 * 1024, false).unwrap(),
2699        )));
2700        controller
2701            .attach(
2702                ScsiPath {
2703                    path: 0,
2704                    target: 0,
2705                    lun: 0,
2706                },
2707                ScsiControllerDisk::new(dvd.clone()),
2708            )
2709            .unwrap();
2710
2711        let mut device = StorageDevice::build_scsi(
2712            &VmTaskDriverSource::new(SingleDriverBackend::new(driver)),
2713            &controller,
2714            Guid::new_random(),
2715            0,
2716            64,
2717        );
2718
2719        let external_data = OwnedRequestBuffers::new(&[0]);
2720        let guest_mem = GuestMemory::allocate(4096);
2721        let buffers = external_data.buffer(&guest_mem);
2722
2723        // The initiator locks the tray. This is the nexus-scoped state that must
2724        // not reach whoever opens the channel next.
2725        let locked = dvd
2726            .execute_scsi(&buffers, &make_medium_removal_request(true))
2727            .await;
2728        assert_eq!(locked.scsi_status, ScsiStatus::GOOD);
2729
2730        // Control: while the nexus lives the lock holds, so the eject is refused.
2731        // Without this the final assertion would only show that ejecting works.
2732        let refused = dvd.execute_scsi(&buffers, &make_eject_request()).await;
2733        assert_ne!(
2734            refused.scsi_status,
2735            ScsiStatus::GOOD,
2736            "the tray lock did not take, so the rest of this test proves nothing"
2737        );
2738
2739        device.close(0).await;
2740
2741        // Same command, same drive, same media: it succeeds only because closing
2742        // the primary channel dropped the lock.
2743        let allowed = dvd.execute_scsi(&buffers, &make_eject_request()).await;
2744        assert_eq!(
2745            allowed.scsi_status,
2746            ScsiStatus::GOOD,
2747            "the tray lock outlived the nexus"
2748        );
2749    }
2750}