1#![expect(missing_docs)]
45#![forbid(unsafe_code)]
46
47#[cfg(feature = "ioperf")]
48pub mod ioperf;
49
50#[cfg(feature = "test")]
51pub mod test_helpers;
52
53#[cfg(not(feature = "test"))]
54mod test_helpers;
55
56pub mod resolver;
57mod save_restore;
58
59use crate::ring::gparange::MultiPagedRangeBuf;
60use anyhow::Context as _;
61use async_trait::async_trait;
62use futures::StreamExt;
63use guestmem::AccessError;
64use guestmem::GuestMemory;
65use guestmem::MemoryRead;
66use guestmem::MemoryWrite;
67use guestmem::ranges::PagedRange;
68use guid::Guid;
69use inspect::Inspect;
70use inspect::InspectMut;
71use inspect_counters::Counter;
72use inspect_counters::Histogram;
73use oversized_box::OversizedBox;
74use parking_lot::Mutex;
75use parking_lot::RwLock;
76use ring::OutgoingPacketType;
77use scsi::AdditionalSenseCode;
78use scsi::ScsiOp;
79use scsi::ScsiStatus;
80use scsi::srb::SrbStatus;
81use scsi::srb::SrbStatusAndFlags;
82use scsi_buffers::RequestBuffers;
83use scsi_core::AsyncScsiDisk;
84use scsi_core::Request;
85use scsi_core::ScsiResult;
86use scsi_defs as scsi;
87use scsidisk::illegal_request_sense;
88use slab::Slab;
89use std::collections::hash_map::Entry;
90use std::collections::hash_map::HashMap;
91use std::fmt::Debug;
92use std::future::Future;
93use std::future::poll_fn;
94use std::pin::Pin;
95use std::sync::Arc;
96use std::sync::atomic::AtomicBool;
97use std::sync::atomic::AtomicU32;
98use std::sync::atomic::Ordering::Acquire;
99use std::sync::atomic::Ordering::Relaxed;
100use std::sync::atomic::Ordering::Release;
101use std::task::Context;
102use std::task::Poll;
103use std::task::Waker;
104use storvsp_resources::ScsiPath;
105use task_control::AsyncRun;
106use task_control::InspectTask;
107use task_control::StopTask;
108use task_control::TaskControl;
109use thiserror::Error;
110use tracing_helpers::ErrorValueExt;
111use unicycle::FuturesUnordered;
112use vmbus_async::queue;
113use vmbus_async::queue::ExternalDataError;
114use vmbus_async::queue::IncomingPacket;
115use vmbus_async::queue::OutgoingPacket;
116use vmbus_async::queue::Queue;
117use vmbus_channel::RawAsyncChannel;
118use vmbus_channel::bus::ChannelType;
119use vmbus_channel::bus::OfferParams;
120use vmbus_channel::bus::OpenRequest;
121use vmbus_channel::channel::ChannelControl;
122use vmbus_channel::channel::ChannelOpenError;
123use vmbus_channel::channel::DeviceResources;
124use vmbus_channel::channel::RestoreControl;
125use vmbus_channel::channel::SaveRestoreVmbusDevice;
126use vmbus_channel::channel::VmbusDevice;
127use vmbus_channel::gpadl_ring::GpadlRingMem;
128use vmbus_channel::gpadl_ring::gpadl_channel;
129use vmbus_core::protocol::UserDefinedData;
130use vmbus_ring as ring;
131use vmbus_ring::RingMem;
132use vmcore::save_restore::RestoreError;
133use vmcore::save_restore::SaveError;
134use vmcore::save_restore::SavedStateBlob;
135use vmcore::vm_task::VmTaskDriver;
136use vmcore::vm_task::VmTaskDriverSource;
137use zerocopy::FromBytes;
138use zerocopy::FromZeros;
139use zerocopy::Immutable;
140use zerocopy::IntoBytes;
141use zerocopy::KnownLayout;
142
143const DEFAULT_POLL_MODE_QUEUE_DEPTH: u32 = 1;
148
149pub struct StorageDevice {
150 instance_id: Guid,
151 ide_path: Option<ScsiPath>,
152 workers: Vec<WorkerAndDriver>,
153 controller: Arc<ScsiControllerState>,
154 resources: DeviceResources,
155 driver_source: VmTaskDriverSource,
156 max_sub_channel_count: u16,
157 protocol: Arc<Protocol>,
158 io_queue_depth: u32,
159}
160
161#[derive(Inspect)]
162struct WorkerAndDriver {
163 #[inspect(flatten)]
164 worker: TaskControl<WorkerState, Worker>,
165 driver: VmTaskDriver,
166}
167
168struct WorkerState;
169
170impl InspectMut for StorageDevice {
171 fn inspect_mut(&mut self, req: inspect::Request<'_>) {
172 let mut resp = req.respond();
173
174 let disks = self.controller.disks.read();
175 for (path, controller_disk) in disks.iter() {
176 resp.child(&format!("disks/{}", path), |req| {
177 controller_disk.disk.inspect(req);
178 });
179 }
180
181 resp.fields(
182 "channels",
183 self.workers
184 .iter()
185 .filter(|task| task.worker.has_state())
186 .enumerate(),
187 )
188 .field(
189 "poll_mode_queue_depth",
190 inspect::AtomicMut(&self.controller.poll_mode_queue_depth),
191 );
192 }
193}
194
195struct Worker<T: RingMem = GpadlRingMem> {
196 inner: WorkerInner,
197 rescan_notification: Arc<RescanNotification>,
198 queue: Queue<T>,
199}
200
201#[derive(Default)]
203struct RescanNotification {
204 pending: AtomicBool,
205 waker: Mutex<Option<Waker>>,
206}
207
208impl RescanNotification {
209 fn notify(&self) {
210 self.pending.store(true, Release);
211 let waker = self.waker.lock().clone();
214 if let Some(waker) = waker {
215 waker.wake();
216 }
217 }
218
219 fn reset(&self) {
220 self.pending.store(false, Relaxed);
221 }
222
223 fn receiver(&self) -> RescanReceiver<'_> {
224 RescanReceiver {
225 notification: self,
226 last_waker: None,
227 }
228 }
229}
230
231struct RescanReceiver<'a> {
232 notification: &'a RescanNotification,
233 last_waker: Option<Waker>,
234}
235
236impl RescanReceiver<'_> {
237 fn poll(&mut self, cx: &mut Context<'_>) -> Poll<()> {
238 if self
241 .last_waker
242 .as_ref()
243 .is_none_or(|waker| !cx.waker().will_wake(waker))
244 {
245 let waker = cx.waker().clone();
246 let old_waker = self.notification.waker.lock().replace(waker.clone());
247 self.last_waker = Some(waker);
248 drop(old_waker);
249 }
250 if self.notification.pending.load(Relaxed) {
253 self.notification.pending.swap(false, Acquire);
254 Poll::Ready(())
255 } else {
256 Poll::Pending
257 }
258 }
259}
260
261struct Protocol {
262 state: RwLock<ProtocolState>,
263 ready: event_listener::Event,
265}
266
267struct WorkerInner {
268 protocol: Arc<Protocol>,
269 request_size: usize,
270 controller: Arc<ScsiControllerState>,
271 channel_index: u16,
272 scsi_queue: Arc<ScsiCommandQueue>,
273 scsi_requests: FuturesUnordered<ScsiRequest>,
274 scsi_requests_states: Slab<ScsiRequestState>,
275 full_request_pool: Vec<Arc<ScsiRequestAndRange>>,
276 future_pool: Vec<OversizedBox<(), ScsiOpStorage>>,
277 channel_control: ChannelControl,
278 max_io_queue_depth: usize,
279 stats: WorkerStats,
280}
281
282#[derive(Debug, Default, Inspect)]
283struct WorkerStats {
284 ios_submitted: Counter,
285 ios_completed: Counter,
286 wakes: Counter,
287 wakes_spurious: Counter,
288 per_wake_submissions: Histogram<10>,
289 per_wake_completions: Histogram<10>,
290}
291
292#[repr(u16)]
293#[derive(Copy, Clone, Debug, Inspect, PartialEq, Eq, PartialOrd, Ord)]
294enum Version {
295 Win6 = storvsp_protocol::VERSION_WIN6,
296 Win7 = storvsp_protocol::VERSION_WIN7,
297 Win8 = storvsp_protocol::VERSION_WIN8,
298 Blue = storvsp_protocol::VERSION_BLUE,
299}
300
301#[derive(Debug, Error)]
302#[error("protocol version {0:#x} not supported")]
303struct UnsupportedVersion(u16);
304
305impl Version {
306 fn parse(major_minor: u16) -> Result<Self, UnsupportedVersion> {
307 let version = match major_minor {
308 storvsp_protocol::VERSION_WIN6 => Self::Win6,
309 storvsp_protocol::VERSION_WIN7 => Self::Win7,
310 storvsp_protocol::VERSION_WIN8 => Self::Win8,
311 storvsp_protocol::VERSION_BLUE => Self::Blue,
312 version => return Err(UnsupportedVersion(version)),
313 };
314 assert_eq!(version as u16, major_minor);
315 Ok(version)
316 }
317
318 fn max_request_size(&self) -> usize {
319 match self {
320 Version::Win8 | Version::Blue => storvsp_protocol::SCSI_REQUEST_LEN_V2,
321 Version::Win6 | Version::Win7 => storvsp_protocol::SCSI_REQUEST_LEN_V1,
322 }
323 }
324}
325
326#[derive(Copy, Clone)]
327enum ProtocolState {
328 Init(InitState),
329 Ready {
330 version: Version,
331 subchannel_count: u16,
332 },
333}
334
335#[derive(Copy, Clone, Debug)]
336enum InitState {
337 Begin,
338 QueryVersion,
339 QueryProperties {
340 version: Version,
341 },
342 EndInitialization {
343 version: Version,
344 subchannel_count: Option<u16>,
345 },
346}
347
348type ScsiOpStorage = [u64; SCSI_REQUEST_STACK_SIZE / 8];
356type ScsiOpFuture = Pin<OversizedBox<dyn Future<Output = ScsiResult> + Send, ScsiOpStorage>>;
357
358const SCSI_REQUEST_STACK_SIZE: usize = scsi_core::ASYNC_SCSI_DISK_STACK_SIZE + 272;
363
364struct ScsiRequest {
365 request_id: usize,
366 future: Option<ScsiOpFuture>,
367}
368
369impl ScsiRequest {
370 fn new(
371 request_id: usize,
372 future: OversizedBox<dyn Future<Output = ScsiResult> + Send, ScsiOpStorage>,
373 ) -> Self {
374 Self {
375 request_id,
376 future: Some(future.into()),
377 }
378 }
379}
380
381impl Future for ScsiRequest {
382 type Output = (usize, ScsiResult, OversizedBox<(), ScsiOpStorage>);
383
384 fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Self::Output> {
385 let this = self.get_mut();
386 let future = this.future.as_mut().unwrap().as_mut();
387 let result = std::task::ready!(future.poll(cx));
388 let future = this.future.take().unwrap();
390 Poll::Ready((this.request_id, result, OversizedBox::empty_pinned(future)))
391 }
392}
393
394#[derive(Debug, Error)]
395enum WorkerError {
396 #[error("cancelled")]
397 Cancelled(task_control::Cancelled),
398 #[error("packet error")]
399 PacketError(#[source] PacketError),
400 #[error("queue error")]
401 Queue(#[source] queue::Error),
402 #[error("queue should have enough space but no longer does")]
403 NotEnoughSpace,
404}
405
406impl From<task_control::Cancelled> for WorkerError {
407 fn from(value: task_control::Cancelled) -> Self {
408 Self::Cancelled(value)
409 }
410}
411
412#[derive(Debug, Error)]
413enum PacketError {
414 #[error("Not transactional")]
415 NotTransactional,
416 #[error("Unrecognized operation {0:?}")]
417 UnrecognizedOperation(storvsp_protocol::Operation),
418 #[error("Invalid packet type")]
419 InvalidPacketType,
420 #[error("Invalid data transfer length")]
421 InvalidDataTransferLength,
422 #[error("Access error: {0}")]
423 Access(#[source] AccessError),
424 #[error("Range error")]
425 Range(#[source] ExternalDataError),
426}
427
428#[derive(Debug, Default, Clone)]
429struct Range {
430 len: usize,
431 is_write: bool,
432}
433
434impl Range {
435 fn new(buf: &MultiPagedRangeBuf, request: &storvsp_protocol::ScsiRequest) -> Option<Self> {
436 let len = request.data_transfer_length as usize;
437 let is_write = request.data_in != 0;
438 if buf.range_count() > 1 || (len > 0 && buf.first()?.len() < len) {
441 return None;
442 }
443 Some(Self { len, is_write })
444 }
445
446 fn buffer<'a>(
447 &'a self,
448 buf: &'a MultiPagedRangeBuf,
449 guest_memory: &'a GuestMemory,
450 ) -> RequestBuffers<'a> {
451 let mut range = buf.first().unwrap_or_else(PagedRange::empty);
452 range.truncate(self.len);
453 RequestBuffers::new(guest_memory, range, self.is_write)
454 }
455}
456
457#[derive(Debug)]
458struct Packet {
459 data: PacketData,
460 transaction_id: u64,
461 request_size: usize,
462}
463
464#[derive(Debug)]
465enum PacketData {
466 BeginInitialization,
467 EndInitialization,
468 QueryProtocolVersion(u16),
469 QueryProperties,
470 CreateSubChannels(u16),
471 ExecuteScsi(Arc<ScsiRequestAndRange>),
472 ResetBus,
473 ResetAdapter,
474 ResetLun,
475}
476
477fn parse_packet<T: RingMem>(
478 packet: &IncomingPacket<'_, T>,
479 pool: &mut Vec<Arc<ScsiRequestAndRange>>,
480) -> Result<Packet, PacketError> {
481 let packet = match packet {
482 IncomingPacket::Completion(_) => return Err(PacketError::InvalidPacketType),
483 IncomingPacket::Data(packet) => packet,
484 };
485 let transaction_id = packet
486 .transaction_id()
487 .ok_or(PacketError::NotTransactional)?;
488
489 let mut reader = packet.reader();
490 let header: storvsp_protocol::Packet = reader.read_plain().map_err(PacketError::Access)?;
491 let request_size = reader.len().min(storvsp_protocol::SCSI_REQUEST_LEN_MAX);
495 let data = match header.operation {
496 storvsp_protocol::Operation::BEGIN_INITIALIZATION => PacketData::BeginInitialization,
497 storvsp_protocol::Operation::END_INITIALIZATION => PacketData::EndInitialization,
498 storvsp_protocol::Operation::QUERY_PROTOCOL_VERSION => {
499 let mut version = storvsp_protocol::ProtocolVersion::new_zeroed();
500 reader
501 .read(version.as_mut_bytes())
502 .map_err(PacketError::Access)?;
503 PacketData::QueryProtocolVersion(version.major_minor)
504 }
505 storvsp_protocol::Operation::QUERY_PROPERTIES => PacketData::QueryProperties,
506 storvsp_protocol::Operation::EXECUTE_SRB => {
507 let mut full_request = pool.pop().unwrap_or_else(|| {
508 Arc::new(ScsiRequestAndRange {
509 external_data: Range::default(),
510 external_data_buf: MultiPagedRangeBuf::new(),
511 request: storvsp_protocol::ScsiRequest::new_zeroed(),
512 request_size,
513 })
514 });
515
516 {
517 let full_request = Arc::get_mut(&mut full_request).unwrap();
518 let request_buf = &mut full_request.request.as_mut_bytes()[..request_size];
519 reader.read(request_buf).map_err(PacketError::Access)?;
520
521 full_request.external_data_buf.clear();
522 packet
523 .read_external_ranges(&mut full_request.external_data_buf)
524 .map_err(PacketError::Range)?;
525
526 full_request.external_data =
527 Range::new(&full_request.external_data_buf, &full_request.request)
528 .ok_or(PacketError::InvalidDataTransferLength)?;
529 }
530
531 PacketData::ExecuteScsi(full_request)
532 }
533 storvsp_protocol::Operation::RESET_LUN => PacketData::ResetLun,
534 storvsp_protocol::Operation::RESET_ADAPTER => PacketData::ResetAdapter,
535 storvsp_protocol::Operation::RESET_BUS => PacketData::ResetBus,
536 storvsp_protocol::Operation::CREATE_SUB_CHANNELS => {
537 let mut sub_channel_count: u16 = 0;
538 reader
539 .read(sub_channel_count.as_mut_bytes())
540 .map_err(PacketError::Access)?;
541 PacketData::CreateSubChannels(sub_channel_count)
542 }
543 _ => return Err(PacketError::UnrecognizedOperation(header.operation)),
544 };
545
546 if let PacketData::ExecuteScsi(_) = data {
547 tracing::trace!(transaction_id, ?data, "parse_packet");
548 } else {
549 tracing::debug!(transaction_id, ?data, "parse_packet");
550 }
551
552 Ok(Packet {
553 data,
554 request_size,
555 transaction_id,
556 })
557}
558
559impl WorkerInner {
560 fn send_vmbus_packet<M: RingMem>(
561 &mut self,
562 writer: &mut queue::WriteBatch<'_, M>,
563 packet_type: OutgoingPacketType<'_>,
564 request_size: usize,
565 transaction_id: u64,
566 operation: storvsp_protocol::Operation,
567 status: storvsp_protocol::NtStatus,
568 payload: &[u8],
569 ) -> Result<(), WorkerError> {
570 let header = storvsp_protocol::Packet {
571 operation,
572 flags: 0,
573 status,
574 };
575
576 let packet_size = size_of_val(&header) + request_size;
577
578 let len = size_of_val(&header) + size_of_val(payload);
583 let padding = [0; storvsp_protocol::SCSI_REQUEST_LEN_MAX];
584 let (payload_bytes, padding_bytes) = if len > packet_size {
585 (&payload[..packet_size - size_of_val(&header)], &[][..])
586 } else {
587 (payload, &padding[..packet_size - len])
588 };
589 assert_eq!(
590 size_of_val(&header) + payload_bytes.len() + padding_bytes.len(),
591 packet_size
592 );
593 writer
594 .try_write(&OutgoingPacket {
595 transaction_id,
596 packet_type,
597 payload: &[header.as_bytes(), payload_bytes, padding_bytes],
598 })
599 .map_err(|err| match err {
600 queue::TryWriteError::Full(_) => WorkerError::NotEnoughSpace,
601 queue::TryWriteError::Queue(err) => WorkerError::Queue(err),
602 })
603 }
604
605 fn send_packet<M: RingMem, P: IntoBytes + Immutable + KnownLayout>(
606 &mut self,
607 writer: &mut queue::WriteHalf<'_, M>,
608 operation: storvsp_protocol::Operation,
609 status: storvsp_protocol::NtStatus,
610 payload: &P,
611 ) -> Result<(), WorkerError> {
612 self.send_vmbus_packet(
613 &mut writer.batched(),
614 OutgoingPacketType::InBandNoCompletion,
615 self.request_size,
616 0,
617 operation,
618 status,
619 payload.as_bytes(),
620 )
621 }
622
623 fn send_completion<M: RingMem, P: IntoBytes + Immutable + KnownLayout>(
624 &mut self,
625 writer: &mut queue::WriteHalf<'_, M>,
626 packet: &Packet,
627 status: storvsp_protocol::NtStatus,
628 payload: &P,
629 ) -> Result<(), WorkerError> {
630 self.send_vmbus_packet(
631 &mut writer.batched(),
632 OutgoingPacketType::Completion,
633 packet.request_size,
634 packet.transaction_id,
635 storvsp_protocol::Operation::COMPLETE_IO,
636 status,
637 payload.as_bytes(),
638 )
639 }
640}
641
642struct ScsiCommandQueue {
643 controller: Arc<ScsiControllerState>,
644 mem: GuestMemory,
645 force_path_id: Option<u8>,
646}
647
648impl ScsiCommandQueue {
649 async fn execute_scsi(&self, full_request: &ScsiRequestAndRange) -> ScsiResult {
650 let request = &full_request.request;
651 let op = ScsiOp(request.payload[0]);
652 let external_data = full_request
653 .external_data
654 .buffer(&full_request.external_data_buf, &self.mem);
655
656 tracing::trace!(
657 path_id = request.path_id,
658 target_id = request.target_id,
659 lun = request.lun,
660 op = ?op,
661 "execute_scsi start...",
662 );
663
664 let path_id = self.force_path_id.unwrap_or(request.path_id);
665
666 let controller_disk = self
667 .controller
668 .disks
669 .read()
670 .get(&ScsiPath {
671 path: path_id,
672 target: request.target_id,
673 lun: request.lun,
674 })
675 .cloned();
676
677 let result = match op {
678 ScsiOp::REPORT_LUNS => {
679 const HEADER_SIZE: usize = size_of::<scsi::LunList>();
680 let mut luns: Vec<u8> = self
681 .controller
682 .disks
683 .read()
684 .keys()
685 .flat_map(|path| {
686 if request.path_id == path.path && request.target_id == path.target {
689 Some(path.lun)
690 } else {
691 None
692 }
693 })
694 .collect();
695 luns.sort_unstable();
696 let mut data: Vec<u64> = vec![0; luns.len() + 1];
697 let header = scsi::LunList {
698 length: (luns.len() as u32 * 8).into(),
699 reserved: [0; 4],
700 };
701 data.as_mut_bytes()[..HEADER_SIZE].copy_from_slice(header.as_bytes());
702 for (i, lun) in luns.iter().enumerate() {
703 data[i + 1].as_mut_bytes()[..2].copy_from_slice(&(*lun as u16).to_be_bytes());
704 }
705 if external_data.len() >= HEADER_SIZE {
706 let tx = std::cmp::min(external_data.len(), data.as_bytes().len());
707 external_data.writer().write(&data.as_bytes()[..tx]).map_or(
708 ScsiResult {
709 scsi_status: ScsiStatus::CHECK_CONDITION,
710 srb_status: SrbStatus::INVALID_REQUEST,
711 tx: 0,
712 sense_data: Some(illegal_request_sense(
713 AdditionalSenseCode::INVALID_CDB,
714 )),
715 },
716 |_| ScsiResult {
717 scsi_status: ScsiStatus::GOOD,
718 srb_status: SrbStatus::SUCCESS,
719 tx,
720 sense_data: None,
721 },
722 )
723 } else {
724 ScsiResult {
725 scsi_status: ScsiStatus::GOOD,
726 srb_status: SrbStatus::SUCCESS,
727 tx: 0,
728 sense_data: None,
729 }
730 }
731 }
732 _ if let Some(controller_disk) = controller_disk => {
733 let mut cdb = [0; 16];
734 cdb.copy_from_slice(&request.payload[0..storvsp_protocol::CDB16GENERIC_LENGTH]);
735 controller_disk
736 .disk
737 .execute_scsi(
738 &external_data,
739 &Request {
740 cdb,
741 srb_flags: request.srb_flags,
742 },
743 )
744 .await
745 }
746 ScsiOp::INQUIRY => {
747 let cdb = scsi::CdbInquiry::ref_from_prefix(&request.payload)
748 .unwrap()
749 .0; if external_data.len() < cdb.allocation_length.get() as usize
751 || request.data_in != storvsp_protocol::SCSI_IOCTL_DATA_IN
752 || (cdb.allocation_length.get() as usize) < size_of::<scsi::InquiryDataHeader>()
753 {
754 ScsiResult {
755 scsi_status: ScsiStatus::CHECK_CONDITION,
756 srb_status: SrbStatus::INVALID_REQUEST,
757 tx: 0,
758 sense_data: Some(illegal_request_sense(AdditionalSenseCode::INVALID_CDB)),
759 }
760 } else {
761 let enable_vpd = cdb.flags.vpd();
762 if enable_vpd || cdb.page_code != 0 {
763 ScsiResult {
765 scsi_status: ScsiStatus::CHECK_CONDITION,
766 srb_status: SrbStatus::INVALID_REQUEST,
767 tx: 0,
768 sense_data: Some(illegal_request_sense(
769 AdditionalSenseCode::INVALID_CDB,
770 )),
771 }
772 } else {
773 const LOGICAL_UNIT_NOT_PRESENT_DEVICE: u8 = 0x7F;
774 let mut data = scsidisk::INQUIRY_DATA_TEMPLATE;
775 data.header.device_type = LOGICAL_UNIT_NOT_PRESENT_DEVICE;
776
777 if request.lun != 0 {
778 data.vendor_id = [0; 8];
780 data.product_id = [0; 16];
781 data.product_revision_level = [0; 4];
782 }
783
784 let datab = data.as_bytes();
785 let tx = std::cmp::min(
786 cdb.allocation_length.get() as usize,
787 size_of::<scsi::InquiryData>(),
788 );
789 external_data.writer().write(&datab[..tx]).map_or(
790 ScsiResult {
791 scsi_status: ScsiStatus::CHECK_CONDITION,
792 srb_status: SrbStatus::INVALID_REQUEST,
793 tx: 0,
794 sense_data: Some(illegal_request_sense(
795 AdditionalSenseCode::INVALID_CDB,
796 )),
797 },
798 |_| ScsiResult {
799 scsi_status: ScsiStatus::GOOD,
800 srb_status: SrbStatus::SUCCESS,
801 tx,
802 sense_data: None,
803 },
804 )
805 }
806 }
807 }
808 _ => ScsiResult {
809 scsi_status: ScsiStatus::CHECK_CONDITION,
810 srb_status: SrbStatus::INVALID_LUN,
811 tx: 0,
812 sense_data: None,
813 },
814 };
815
816 tracing::trace!(
817 path_id = request.path_id,
818 target_id = request.target_id,
819 lun = request.lun,
820 op = ?op,
821 result = ?result,
822 "execute_scsi completed.",
823 );
824 result
825 }
826}
827
828impl<T: RingMem + 'static> Worker<T> {
829 fn new(
830 controller: Arc<ScsiControllerState>,
831 channel: RawAsyncChannel<T>,
832 channel_index: u16,
833 mem: GuestMemory,
834 channel_control: ChannelControl,
835 io_queue_depth: u32,
836 protocol: Arc<Protocol>,
837 force_path_id: Option<u8>,
838 ) -> anyhow::Result<Self> {
839 let queue = Queue::new(channel)?;
840 let rescan_notification = Arc::new(RescanNotification::default());
841 controller.add_rescan_notification(rescan_notification.clone());
842
843 let max_io_queue_depth = io_queue_depth.max(1) as usize;
844 Ok(Self {
845 inner: WorkerInner {
846 protocol,
847 request_size: storvsp_protocol::SCSI_REQUEST_LEN_V1,
848 controller: controller.clone(),
849 channel_index,
850 scsi_queue: Arc::new(ScsiCommandQueue {
851 controller,
852 mem,
853 force_path_id,
854 }),
855 scsi_requests: FuturesUnordered::new(),
856 scsi_requests_states: Slab::with_capacity(max_io_queue_depth),
857 channel_control,
858 max_io_queue_depth,
859 future_pool: Vec::new(),
860 full_request_pool: Vec::new(),
861 stats: Default::default(),
862 },
863 queue,
864 rescan_notification,
865 })
866 }
867
868 async fn wait_for_scsi_requests_complete(&mut self) {
869 tracing::debug!(
870 channel_index = self.inner.channel_index,
871 "wait for IOs completed..."
872 );
873 while let Some((id, _, _)) = self.inner.scsi_requests.next().await {
874 self.inner.scsi_requests_states.remove(id);
875 }
876 }
877}
878
879impl InspectTask<Worker> for WorkerState {
880 fn inspect(&self, req: inspect::Request<'_>, worker: Option<&Worker>) {
881 if let Some(worker) = worker {
882 let mut resp = req.respond();
883 if worker.inner.channel_index == 0 {
884 let (state, version, subchannel_count) = match *worker.inner.protocol.state.read() {
885 ProtocolState::Init(state) => match state {
886 InitState::Begin => ("begin_init", None, None),
887 InitState::QueryVersion => ("query_version", None, None),
888 InitState::QueryProperties { version } => {
889 ("query_properties", Some(version), None)
890 }
891 InitState::EndInitialization {
892 version,
893 subchannel_count,
894 } => ("end_init", Some(version), subchannel_count),
895 },
896 ProtocolState::Ready {
897 version,
898 subchannel_count,
899 } => ("ready", Some(version), Some(subchannel_count)),
900 };
901 resp.field("state", state)
902 .field("version", version)
903 .field("subchannel_count", subchannel_count);
904 }
905 resp.field("pending_packets", worker.inner.scsi_requests_states.len())
906 .fields("io", worker.inner.scsi_requests_states.iter())
907 .field("stats", &worker.inner.stats)
908 .field("ring", &worker.queue)
909 .field("max_io_queue_depth", worker.inner.max_io_queue_depth);
910 }
911 }
912}
913
914impl<T: 'static + Send + Sync + RingMem> AsyncRun<Worker<T>> for WorkerState {
915 async fn run(
916 &mut self,
917 stop: &mut StopTask<'_>,
918 worker: &mut Worker<T>,
919 ) -> Result<(), task_control::Cancelled> {
920 let check = stop.check_handle()?;
921 let fut = async {
922 if worker.inner.channel_index == 0 {
923 worker.process_primary(&check).await
924 } else {
925 let protocol_version = loop {
928 check.check()?;
929 let listener = worker.inner.protocol.ready.listen();
930 if let ProtocolState::Ready { version, .. } =
931 *worker.inner.protocol.state.read()
932 {
933 break version;
934 }
935 tracing::debug!("subchannel waiting for initialization to end");
936 listener.await
937 };
938 worker
939 .inner
940 .process_ready(&mut worker.queue, protocol_version, &check)
941 .await
942 }
943 };
944
945 match stop.until_stopped(fut).await? {
946 Err(WorkerError::Cancelled(cancelled)) => return Err(cancelled),
947 Ok(_) => {}
948 Err(e) => tracing::error!(error = e.as_error(), "process_packets error"),
949 }
950 Ok(())
951 }
952}
953
954impl WorkerInner {
955 async fn next_packet<'a, M: RingMem>(
958 &mut self,
959 reader: &'a mut queue::ReadHalf<'a, M>,
960 ) -> Result<Packet, WorkerError> {
961 let packet = reader.read().await.map_err(WorkerError::Queue)?;
962 let stor_packet =
963 parse_packet(&packet, &mut self.full_request_pool).map_err(WorkerError::PacketError)?;
964 Ok(stor_packet)
965 }
966
967 fn poll_for_ring_space<M: RingMem>(
974 &mut self,
975 cx: &mut Context<'_>,
976 writer: &mut queue::WriteHalf<'_, M>,
977 ) -> Poll<Result<(), WorkerError>> {
978 writer
979 .poll_ready(cx, MAX_VMBUS_PACKET_SIZE)
980 .map_err(WorkerError::Queue)
981 }
982}
983
984const MAX_VMBUS_PACKET_SIZE: usize = ring::PacketSize::in_band(
985 size_of::<storvsp_protocol::Packet>() + storvsp_protocol::SCSI_REQUEST_LEN_MAX,
986);
987
988impl<T: RingMem> Worker<T> {
989 async fn process_primary(
991 &mut self,
992 check: &task_control::StopCheck<'_>,
993 ) -> Result<(), WorkerError> {
994 loop {
995 check.check()?;
996 let current_state = *self.inner.protocol.state.read();
997 match current_state {
998 ProtocolState::Ready { version, .. } => {
999 self.inner.request_size = version.max_request_size();
1000 let mut rescan = self.rescan_notification.receiver();
1001 break poll_fn(|cx| {
1002 loop {
1003 check.check()?;
1004 if let Poll::Ready(r) =
1005 self.inner.poll_process_ready(cx, &mut self.queue, check)
1006 {
1007 return Poll::Ready(r);
1008 }
1009 if rescan.poll(cx).is_pending() {
1010 return Poll::Pending;
1011 }
1012 if version >= Version::Win7 {
1013 tracing::debug!(
1014 "rescan notification received, sending ENUMERATE_BUS"
1015 );
1016 self.inner.send_packet(
1017 &mut self.queue.split().1,
1018 storvsp_protocol::Operation::ENUMERATE_BUS,
1019 storvsp_protocol::NtStatus::SUCCESS,
1020 &(),
1021 )?;
1022 }
1023 }
1024 })
1025 .await;
1026 }
1027 ProtocolState::Init(state) => {
1028 let (mut reader, mut writer) = self.queue.split();
1029
1030 poll_fn(|cx| self.inner.poll_for_ring_space(cx, &mut writer)).await?;
1033
1034 tracing::debug!(?state, "process_primary");
1035 match state {
1036 InitState::Begin => {
1037 let packet = self.inner.next_packet(&mut reader).await?;
1038 if let PacketData::BeginInitialization = packet.data {
1039 self.inner.send_completion(
1040 &mut writer,
1041 &packet,
1042 storvsp_protocol::NtStatus::SUCCESS,
1043 &(),
1044 )?;
1045 *self.inner.protocol.state.write() =
1046 ProtocolState::Init(InitState::QueryVersion);
1047 } else {
1048 tracelimit::warn_ratelimited!(?state, data = ?packet.data, "unexpected packet order");
1049 self.inner.send_completion(
1050 &mut writer,
1051 &packet,
1052 storvsp_protocol::NtStatus::INVALID_DEVICE_STATE,
1053 &(),
1054 )?;
1055 }
1056 }
1057 InitState::QueryVersion => {
1058 let packet = self.inner.next_packet(&mut reader).await?;
1059 if let PacketData::QueryProtocolVersion(major_minor) = packet.data {
1060 if let Ok(version) = Version::parse(major_minor) {
1061 self.inner.send_completion(
1062 &mut writer,
1063 &packet,
1064 storvsp_protocol::NtStatus::SUCCESS,
1065 &storvsp_protocol::ProtocolVersion {
1066 major_minor,
1067 reserved: 0,
1068 },
1069 )?;
1070 self.inner.request_size = version.max_request_size();
1071 *self.inner.protocol.state.write() =
1072 ProtocolState::Init(InitState::QueryProperties { version });
1073
1074 tracelimit::info_ratelimited!(
1075 ?version,
1076 "scsi version negotiated"
1077 );
1078 } else {
1079 self.inner.send_completion(
1080 &mut writer,
1081 &packet,
1082 storvsp_protocol::NtStatus::REVISION_MISMATCH,
1083 &storvsp_protocol::ProtocolVersion {
1084 major_minor,
1085 reserved: 0,
1086 },
1087 )?;
1088 *self.inner.protocol.state.write() =
1089 ProtocolState::Init(InitState::QueryVersion);
1090 }
1091 } else {
1092 tracelimit::warn_ratelimited!(?state, data = ?packet.data, "unexpected packet order");
1093 self.inner.send_completion(
1094 &mut writer,
1095 &packet,
1096 storvsp_protocol::NtStatus::INVALID_DEVICE_STATE,
1097 &(),
1098 )?;
1099 }
1100 }
1101 InitState::QueryProperties { version } => {
1102 let packet = self.inner.next_packet(&mut reader).await?;
1103 if let PacketData::QueryProperties = packet.data {
1104 let multi_channel_supported = version >= Version::Win8;
1105
1106 self.inner.send_completion(
1107 &mut writer,
1108 &packet,
1109 storvsp_protocol::NtStatus::SUCCESS,
1110 &storvsp_protocol::ChannelProperties {
1111 max_transfer_bytes: 0x40000, flags: {
1113 if multi_channel_supported {
1114 storvsp_protocol::STORAGE_CHANNEL_SUPPORTS_MULTI_CHANNEL
1115 } else {
1116 0
1117 }
1118 },
1119 maximum_sub_channel_count: if multi_channel_supported {
1120 self.inner.channel_control.max_subchannels()
1121 } else {
1122 0
1123 },
1124 reserved: 0,
1125 reserved2: 0,
1126 reserved3: [0, 0],
1127 },
1128 )?;
1129 *self.inner.protocol.state.write() =
1130 ProtocolState::Init(InitState::EndInitialization {
1131 version,
1132 subchannel_count: if multi_channel_supported {
1133 None
1134 } else {
1135 Some(0)
1136 },
1137 });
1138 } else {
1139 tracelimit::warn_ratelimited!(?state, data = ?packet.data, "unexpected packet order");
1140 self.inner.send_completion(
1141 &mut writer,
1142 &packet,
1143 storvsp_protocol::NtStatus::INVALID_DEVICE_STATE,
1144 &(),
1145 )?;
1146 }
1147 }
1148 InitState::EndInitialization {
1149 version,
1150 subchannel_count,
1151 } => {
1152 let packet = self.inner.next_packet(&mut reader).await?;
1153 match packet.data {
1154 PacketData::CreateSubChannels(sub_channel_count)
1155 if subchannel_count.is_none() =>
1156 {
1157 if let Err(err) = self
1158 .inner
1159 .channel_control
1160 .enable_subchannels(sub_channel_count)
1161 {
1162 tracelimit::warn_ratelimited!(
1163 ?err,
1164 "cannot enable subchannels"
1165 );
1166 self.inner.send_completion(
1167 &mut writer,
1168 &packet,
1169 storvsp_protocol::NtStatus::INVALID_PARAMETER,
1170 &(),
1171 )?;
1172 } else {
1173 self.inner.send_completion(
1174 &mut writer,
1175 &packet,
1176 storvsp_protocol::NtStatus::SUCCESS,
1177 &(),
1178 )?;
1179 *self.inner.protocol.state.write() =
1180 ProtocolState::Init(InitState::EndInitialization {
1181 version,
1182 subchannel_count: Some(sub_channel_count),
1183 });
1184 }
1185 }
1186 PacketData::EndInitialization => {
1187 self.inner.send_completion(
1188 &mut writer,
1189 &packet,
1190 storvsp_protocol::NtStatus::SUCCESS,
1191 &(),
1192 )?;
1193 self.rescan_notification.reset();
1196 *self.inner.protocol.state.write() = ProtocolState::Ready {
1197 version,
1198 subchannel_count: subchannel_count.unwrap_or(0),
1199 };
1200 self.inner.protocol.ready.notify(usize::MAX);
1203 }
1204 _ => {
1205 tracelimit::warn_ratelimited!(?state, data = ?packet.data, "unexpected packet order");
1206 self.inner.send_completion(
1207 &mut writer,
1208 &packet,
1209 storvsp_protocol::NtStatus::INVALID_DEVICE_STATE,
1210 &(),
1211 )?;
1212 }
1213 }
1214 }
1215 }
1216 }
1217 }
1218 }
1219 }
1220}
1221
1222fn convert_srb_status_to_nt_status(srb_status: SrbStatus) -> storvsp_protocol::NtStatus {
1223 match srb_status {
1224 SrbStatus::BUSY => storvsp_protocol::NtStatus::DEVICE_BUSY,
1225 SrbStatus::SUCCESS => storvsp_protocol::NtStatus::SUCCESS,
1226 SrbStatus::INVALID_LUN
1227 | SrbStatus::INVALID_TARGET_ID
1228 | SrbStatus::NO_DEVICE
1229 | SrbStatus::NO_HBA => storvsp_protocol::NtStatus::DEVICE_DOES_NOT_EXIST,
1230 SrbStatus::COMMAND_TIMEOUT | SrbStatus::TIMEOUT => storvsp_protocol::NtStatus::IO_TIMEOUT,
1231 SrbStatus::SELECTION_TIMEOUT => storvsp_protocol::NtStatus::DEVICE_NOT_CONNECTED,
1232 SrbStatus::BAD_FUNCTION | SrbStatus::BAD_SRB_BLOCK_LENGTH => {
1233 storvsp_protocol::NtStatus::INVALID_DEVICE_REQUEST
1234 }
1235 SrbStatus::DATA_OVERRUN => storvsp_protocol::NtStatus::BUFFER_OVERFLOW,
1236 SrbStatus::REQUEST_FLUSHED => storvsp_protocol::NtStatus::UNSUCCESSFUL,
1237 SrbStatus::ABORTED => storvsp_protocol::NtStatus::CANCELLED,
1238 _ => storvsp_protocol::NtStatus::IO_DEVICE_ERROR,
1239 }
1240}
1241
1242impl WorkerInner {
1243 async fn process_ready<M: RingMem>(
1245 &mut self,
1246 queue: &mut Queue<M>,
1247 protocol_version: Version,
1248 check: &task_control::StopCheck<'_>,
1249 ) -> Result<(), WorkerError> {
1250 self.request_size = protocol_version.max_request_size();
1251 poll_fn(|cx| self.poll_process_ready(cx, queue, check)).await
1252 }
1253
1254 fn poll_process_ready<M: RingMem>(
1256 &mut self,
1257 cx: &mut Context<'_>,
1258 queue: &mut Queue<M>,
1259 check: &task_control::StopCheck<'_>,
1260 ) -> Poll<Result<(), WorkerError>> {
1261 self.stats.wakes.increment();
1262
1263 let (mut reader, mut writer) = queue.split();
1264 let mut total_completions = 0;
1265 let mut total_submissions = 0;
1266 let poll_mode_queue_depth = self.controller.poll_mode_queue_depth.load(Relaxed) as usize;
1267
1268 let cancelled = 'processing: loop {
1269 if let Err(cancelled) = check.check() {
1270 break Some(cancelled);
1271 }
1272 'outer: while !self.scsi_requests_states.is_empty() {
1274 {
1275 let mut batch = writer.batched();
1276 loop {
1277 if !batch
1281 .can_write(MAX_VMBUS_PACKET_SIZE)
1282 .map_err(WorkerError::Queue)?
1283 {
1284 break;
1286 }
1287 if let Poll::Ready(Some((request_id, result, future))) =
1288 self.scsi_requests.poll_next_unpin(cx)
1289 {
1290 self.future_pool.push(future);
1291 self.handle_completion(&mut batch, request_id, result)?;
1292 total_completions += 1;
1293 } else {
1294 tracing::trace!("out of completions");
1295 break 'outer;
1296 }
1297 }
1298 }
1299
1300 if self.poll_for_ring_space(cx, &mut writer).is_pending() {
1302 tracing::trace!("out of ring space");
1303 break;
1304 }
1305 }
1306
1307 let previous_submissions = total_submissions;
1308 'outer: loop {
1310 if let Err(cancelled) = check.check() {
1311 break 'processing Some(cancelled);
1312 }
1313 if self.scsi_requests_states.len() >= self.max_io_queue_depth {
1314 break;
1315 }
1316 let mut batch = if self.scsi_requests_states.len() < poll_mode_queue_depth {
1317 if let Poll::Ready(batch) = reader.poll_read_batch(cx) {
1318 batch.map_err(WorkerError::Queue)?
1319 } else {
1320 tracing::trace!("out of incoming packets");
1321 break;
1322 }
1323 } else {
1324 match reader.try_read_batch() {
1325 Ok(batch) => batch,
1326 Err(queue::TryReadError::Empty) => {
1327 tracing::trace!(
1328 pending_io_count = self.scsi_requests_states.len(),
1329 "out of incoming packets, keeping interrupts masked"
1330 );
1331 break;
1332 }
1333 Err(queue::TryReadError::Queue(err)) => Err(WorkerError::Queue(err))?,
1334 }
1335 };
1336
1337 let mut packets = batch.packets();
1338 loop {
1339 if self.scsi_requests_states.len() >= self.max_io_queue_depth {
1340 break 'outer;
1341 }
1342 if self.poll_for_ring_space(cx, &mut writer).is_pending() {
1346 tracing::trace!("out of ring space");
1347 break 'outer;
1348 }
1349
1350 let packet = if let Some(packet) = packets.next() {
1351 packet.map_err(WorkerError::Queue)?
1352 } else {
1353 break;
1354 };
1355
1356 if self.handle_packet(&mut writer, &packet)? {
1357 total_submissions += 1;
1358 }
1359 }
1360 }
1361
1362 if total_submissions == previous_submissions {
1364 break None;
1366 }
1367 };
1368
1369 if total_submissions != 0 || total_completions != 0 {
1371 self.stats.ios_submitted.add(total_submissions);
1372 self.stats
1373 .per_wake_submissions
1374 .add_sample(total_submissions);
1375 self.stats
1376 .per_wake_completions
1377 .add_sample(total_completions);
1378 self.stats.ios_completed.add(total_completions);
1379 } else {
1380 self.stats.wakes_spurious.increment();
1381 }
1382
1383 if let Some(cancelled) = cancelled {
1384 return Poll::Ready(Err(cancelled.into()));
1385 }
1386 Poll::Pending
1387 }
1388
1389 fn handle_completion<M: RingMem>(
1390 &mut self,
1391 writer: &mut queue::WriteBatch<'_, M>,
1392 request_id: usize,
1393 result: ScsiResult,
1394 ) -> Result<(), WorkerError> {
1395 let state = self.scsi_requests_states.remove(request_id);
1396 let request_size = state.request.request_size;
1397
1398 assert_eq!(
1400 Arc::strong_count(&state.request) + Arc::weak_count(&state.request),
1401 1
1402 );
1403 self.full_request_pool.push(state.request);
1404
1405 let status = convert_srb_status_to_nt_status(result.srb_status);
1406 let mut payload = [0; 0x14];
1407 if let Some(sense) = result.sense_data {
1408 payload[..size_of_val(&sense)].copy_from_slice(sense.as_bytes());
1409 tracing::trace!(sense_info = ?payload, sense_key = payload[2], asc = payload[12], "execute_scsi");
1410 };
1411 let response = storvsp_protocol::ScsiRequest {
1412 length: size_of::<storvsp_protocol::ScsiRequest>() as u16,
1413 scsi_status: result.scsi_status,
1414 srb_status: SrbStatusAndFlags::new()
1415 .with_status(result.srb_status)
1416 .with_autosense_valid(result.sense_data.is_some()),
1417 data_transfer_length: result.tx as u32,
1418 cdb_length: storvsp_protocol::CDB16GENERIC_LENGTH as u8,
1419 sense_info_ex_length: storvsp_protocol::VMSCSI_SENSE_BUFFER_SIZE as u8,
1420 payload,
1421 ..storvsp_protocol::ScsiRequest::new_zeroed()
1422 };
1423 self.send_vmbus_packet(
1424 writer,
1425 OutgoingPacketType::Completion,
1426 request_size,
1427 state.transaction_id,
1428 storvsp_protocol::Operation::COMPLETE_IO,
1429 status,
1430 response.as_bytes(),
1431 )?;
1432 Ok(())
1433 }
1434
1435 fn handle_packet<M: RingMem>(
1436 &mut self,
1437 writer: &mut queue::WriteHalf<'_, M>,
1438 packet: &IncomingPacket<'_, M>,
1439 ) -> Result<bool, WorkerError> {
1440 let packet =
1441 parse_packet(packet, &mut self.full_request_pool).map_err(WorkerError::PacketError)?;
1442 let submitted_io = match packet.data {
1443 PacketData::ExecuteScsi(request) => {
1444 self.push_scsi_request(packet.transaction_id, request);
1445 true
1446 }
1447 PacketData::ResetAdapter | PacketData::ResetBus | PacketData::ResetLun => {
1448 self.send_completion(writer, &packet, storvsp_protocol::NtStatus::SUCCESS, &())?;
1450 false
1451 }
1452 PacketData::CreateSubChannels(new_subchannel_count) if self.channel_index == 0 => {
1453 if let Err(err) = self
1454 .channel_control
1455 .enable_subchannels(new_subchannel_count)
1456 {
1457 tracelimit::warn_ratelimited!(?err, "cannot create subchannels");
1458 self.send_completion(
1459 writer,
1460 &packet,
1461 storvsp_protocol::NtStatus::INVALID_PARAMETER,
1462 &(),
1463 )?;
1464 false
1465 } else {
1466 if let ProtocolState::Ready {
1468 subchannel_count, ..
1469 } = &mut *self.protocol.state.write()
1470 {
1471 *subchannel_count = new_subchannel_count;
1472 } else {
1473 unreachable!()
1474 }
1475
1476 self.send_completion(
1477 writer,
1478 &packet,
1479 storvsp_protocol::NtStatus::SUCCESS,
1480 &(),
1481 )?;
1482 false
1483 }
1484 }
1485 _ => {
1486 tracelimit::warn_ratelimited!(data = ?packet.data, "unexpected packet on ready");
1487 self.send_completion(
1488 writer,
1489 &packet,
1490 storvsp_protocol::NtStatus::INVALID_DEVICE_STATE,
1491 &(),
1492 )?;
1493 false
1494 }
1495 };
1496 Ok(submitted_io)
1497 }
1498
1499 fn push_scsi_request(&mut self, transaction_id: u64, full_request: Arc<ScsiRequestAndRange>) {
1500 let scsi_queue = self.scsi_queue.clone();
1501 let scsi_request_state = ScsiRequestState {
1502 transaction_id,
1503 request: full_request.clone(),
1504 };
1505 let request_id = self.scsi_requests_states.insert(scsi_request_state);
1506 let future = self
1507 .future_pool
1508 .pop()
1509 .unwrap_or_else(|| OversizedBox::new(()));
1510 let future = OversizedBox::refill(future, async move {
1511 scsi_queue.execute_scsi(full_request.as_ref()).await
1512 });
1513 let request = ScsiRequest::new(request_id, oversized_box::coerce!(future));
1514 self.scsi_requests.push(request);
1515 }
1516}
1517
1518impl<T: RingMem> Drop for Worker<T> {
1519 fn drop(&mut self) {
1520 self.inner
1521 .controller
1522 .remove_rescan_notification(&self.rescan_notification);
1523 }
1524}
1525
1526#[derive(Debug, Error)]
1527#[error("SCSI path {}:{}:{} is already in use", self.0.path, self.0.target, self.0.lun)]
1528pub struct ScsiPathInUse(pub ScsiPath);
1529
1530#[derive(Debug, Error)]
1531#[error("SCSI path {}:{}:{} is not in use", self.0.path, self.0.target, self.0.lun)]
1532pub struct ScsiPathNotInUse(ScsiPath);
1533
1534#[derive(Clone)]
1535struct ScsiRequestState {
1536 transaction_id: u64,
1537 request: Arc<ScsiRequestAndRange>,
1538}
1539
1540#[derive(Debug)]
1541struct ScsiRequestAndRange {
1542 external_data: Range,
1543 external_data_buf: MultiPagedRangeBuf,
1544 request: storvsp_protocol::ScsiRequest,
1545 request_size: usize,
1546}
1547
1548impl Inspect for ScsiRequestState {
1549 fn inspect(&self, req: inspect::Request<'_>) {
1550 req.respond()
1551 .field("transaction_id", self.transaction_id)
1552 .display(
1553 "address",
1554 &ScsiPath {
1555 path: self.request.request.path_id,
1556 target: self.request.request.target_id,
1557 lun: self.request.request.lun,
1558 },
1559 )
1560 .display_debug("operation", &ScsiOp(self.request.request.payload[0]));
1561 }
1562}
1563
1564impl StorageDevice {
1565 pub fn build_scsi(
1567 driver_source: &VmTaskDriverSource,
1568 controller: &ScsiController,
1569 instance_id: Guid,
1570 max_sub_channel_count: u16,
1571 io_queue_depth: u32,
1572 ) -> Self {
1573 Self::build_inner(
1574 driver_source,
1575 controller,
1576 instance_id,
1577 None,
1578 max_sub_channel_count,
1579 io_queue_depth,
1580 )
1581 }
1582
1583 pub fn build_ide(
1586 driver_source: &VmTaskDriverSource,
1587 channel_id: u8,
1588 device_id: u8,
1589 disk: ScsiControllerDisk,
1590 io_queue_depth: u32,
1591 ) -> Self {
1592 let path = ScsiPath {
1593 path: channel_id,
1594 target: device_id,
1595 lun: 0,
1596 };
1597
1598 let controller = ScsiController::new();
1599 controller.attach(path, disk).unwrap();
1600
1601 let instance_id = Guid {
1604 data1: channel_id.into(),
1605 data2: device_id.into(),
1606 data3: 0x8899,
1607 data4: [0; 8],
1608 };
1609 Self::build_inner(
1610 driver_source,
1611 &controller,
1612 instance_id,
1613 Some(path),
1614 0,
1615 io_queue_depth,
1616 )
1617 }
1618
1619 fn build_inner(
1620 driver_source: &VmTaskDriverSource,
1621 controller: &ScsiController,
1622 instance_id: Guid,
1623 ide_path: Option<ScsiPath>,
1624 max_sub_channel_count: u16,
1625 io_queue_depth: u32,
1626 ) -> Self {
1627 let workers = (0..max_sub_channel_count + 1)
1628 .map(|channel_index| WorkerAndDriver {
1629 worker: TaskControl::new(WorkerState),
1630 driver: driver_source
1631 .builder()
1632 .target_vp(0)
1633 .run_on_target(true)
1634 .build(format!("storvsp-{}-{}", instance_id, channel_index)),
1635 })
1636 .collect();
1637
1638 Self {
1639 instance_id,
1640 ide_path,
1641 workers,
1642 controller: controller.state.clone(),
1643 resources: Default::default(),
1644 max_sub_channel_count,
1645 driver_source: driver_source.clone(),
1646 protocol: Arc::new(Protocol {
1647 state: RwLock::new(ProtocolState::Init(InitState::Begin)),
1648 ready: Default::default(),
1649 }),
1650 io_queue_depth,
1651 }
1652 }
1653
1654 fn new_worker(
1655 &mut self,
1656 open_request: &OpenRequest,
1657 channel_index: u16,
1658 ) -> anyhow::Result<&mut Worker> {
1659 let controller = self.controller.clone();
1660
1661 let target_vp = open_request.open_data.target_vp.unwrap_or_default();
1664 let driver = self
1665 .driver_source
1666 .builder()
1667 .target_vp(target_vp)
1668 .run_on_target(true)
1669 .build(format!("storvsp-{}-{}", self.instance_id, channel_index));
1670
1671 let channel = gpadl_channel(&driver, &self.resources, open_request, channel_index)
1672 .context("failed to create vmbus channel")?;
1673
1674 let channel_control = self.resources.channel_control.clone();
1675
1676 tracing::debug!(
1677 target_vp = open_request.open_data.target_vp,
1678 channel_index,
1679 "packet processing starting...",
1680 );
1681
1682 let force_path_id = self.ide_path.map(|p| p.path);
1686
1687 let worker = Worker::new(
1688 controller,
1689 channel,
1690 channel_index,
1691 self.resources
1692 .offer_resources
1693 .guest_memory(open_request)
1694 .clone(),
1695 channel_control,
1696 self.io_queue_depth,
1697 self.protocol.clone(),
1698 force_path_id,
1699 )
1700 .map_err(RestoreError::Other)?;
1701
1702 self.workers[channel_index as usize]
1703 .driver
1704 .retarget_vp(target_vp);
1705
1706 Ok(self.workers[channel_index as usize].worker.insert(
1707 &driver,
1708 format!("storvsp worker {}-{}", self.instance_id, channel_index),
1709 worker,
1710 ))
1711 }
1712}
1713
1714#[derive(Clone)]
1716pub struct ScsiControllerDisk {
1717 disk: Arc<dyn AsyncScsiDisk>,
1718}
1719
1720impl ScsiControllerDisk {
1721 pub fn new(disk: Arc<dyn AsyncScsiDisk>) -> Self {
1723 Self { disk }
1724 }
1725}
1726
1727struct ScsiControllerState {
1728 disks: RwLock<HashMap<ScsiPath, ScsiControllerDisk>>,
1729 rescan_notifications: Mutex<Vec<Arc<RescanNotification>>>,
1730 poll_mode_queue_depth: AtomicU32,
1731}
1732
1733impl ScsiControllerState {
1734 fn clear_nexus_state(&self) {
1736 for disk in self.disks.read().values() {
1737 disk.disk.clear_nexus_state();
1738 }
1739 }
1740}
1741
1742pub struct ScsiController {
1743 state: Arc<ScsiControllerState>,
1744}
1745
1746impl ScsiController {
1747 pub fn new() -> Self {
1748 Self::new_with_poll_mode_queue_depth(None)
1749 }
1750
1751 pub fn new_with_poll_mode_queue_depth(poll_mode_queue_depth: Option<u32>) -> Self {
1752 Self {
1753 state: Arc::new(ScsiControllerState {
1754 disks: Default::default(),
1755 rescan_notifications: Mutex::new(Vec::new()),
1756 poll_mode_queue_depth: AtomicU32::new(
1757 poll_mode_queue_depth.unwrap_or(DEFAULT_POLL_MODE_QUEUE_DEPTH),
1758 ),
1759 }),
1760 }
1761 }
1762
1763 pub fn attach(&self, path: ScsiPath, disk: ScsiControllerDisk) -> Result<(), ScsiPathInUse> {
1764 match self.state.disks.write().entry(path) {
1765 Entry::Occupied(_) => return Err(ScsiPathInUse(path)),
1766 Entry::Vacant(entry) => entry.insert(disk),
1767 };
1768 for notification in self.state.rescan_notifications.lock().iter() {
1769 notification.notify();
1770 }
1771 Ok(())
1772 }
1773
1774 pub fn remove(&self, path: ScsiPath) -> Result<(), ScsiPathNotInUse> {
1775 match self.state.disks.write().entry(path) {
1776 Entry::Vacant(_) => return Err(ScsiPathNotInUse(path)),
1777 Entry::Occupied(entry) => {
1778 entry.remove();
1779 }
1780 }
1781 for notification in self.state.rescan_notifications.lock().iter() {
1782 notification.notify();
1783 }
1784 Ok(())
1785 }
1786}
1787
1788impl ScsiControllerState {
1789 fn add_rescan_notification(&self, notification: Arc<RescanNotification>) {
1790 self.rescan_notifications.lock().push(notification);
1791 }
1792
1793 fn remove_rescan_notification(&self, notification: &Arc<RescanNotification>) {
1794 let mut notifications = self.rescan_notifications.lock();
1795 if let Some(index) = notifications
1796 .iter()
1797 .position(|source| Arc::ptr_eq(source, notification))
1798 {
1799 notifications.remove(index);
1800 }
1801 }
1802}
1803
1804#[async_trait]
1805impl VmbusDevice for StorageDevice {
1806 fn offer(&self) -> OfferParams {
1807 if let Some(path) = self.ide_path {
1808 let offer_properties = storvsp_protocol::OfferProperties {
1809 path_id: path.path,
1810 target_id: path.target,
1811 flags: storvsp_protocol::OFFER_PROPERTIES_FLAG_IDE_DEVICE,
1812 ..FromZeros::new_zeroed()
1813 };
1814 let mut user_defined = UserDefinedData::new_zeroed();
1815 offer_properties
1816 .write_to_prefix(&mut user_defined[..])
1817 .unwrap();
1818 OfferParams {
1819 interface_name: "ide-accel".to_owned(),
1820 instance_id: self.instance_id,
1821 interface_id: storvsp_protocol::IDE_ACCELERATOR_INTERFACE_ID,
1822 channel_type: ChannelType::Interface { user_defined },
1823 ..Default::default()
1824 }
1825 } else {
1826 OfferParams {
1827 interface_name: "scsi".to_owned(),
1828 instance_id: self.instance_id,
1829 interface_id: storvsp_protocol::SCSI_INTERFACE_ID,
1830 ..Default::default()
1831 }
1832 }
1833 }
1834
1835 fn max_subchannels(&self) -> u16 {
1836 self.max_sub_channel_count
1837 }
1838
1839 fn install(&mut self, resources: DeviceResources) {
1840 self.resources = resources;
1841 }
1842
1843 async fn open(
1844 &mut self,
1845 channel_index: u16,
1846 open_request: &OpenRequest,
1847 ) -> Result<(), ChannelOpenError> {
1848 tracing::debug!(channel_index, "scsi open channel");
1849 self.new_worker(open_request, channel_index)?;
1850 self.workers[channel_index as usize].worker.start();
1851 Ok(())
1852 }
1853
1854 async fn close(&mut self, channel_index: u16) {
1855 tracing::debug!(channel_index, "scsi close channel");
1856 let worker = &mut self.workers[channel_index as usize].worker;
1857 worker.stop().await;
1858 if worker.state_mut().is_some() {
1859 worker
1860 .state_mut()
1861 .unwrap()
1862 .wait_for_scsi_requests_complete()
1863 .await;
1864 worker.remove();
1865 }
1866 if channel_index == 0 {
1867 *self.protocol.state.write() = ProtocolState::Init(InitState::Begin);
1868 self.controller.clear_nexus_state();
1881 }
1882 }
1883
1884 async fn retarget_vp(&mut self, channel_index: u16, target_vp: u32) {
1885 self.workers[channel_index as usize]
1886 .driver
1887 .retarget_vp(target_vp);
1888 }
1889
1890 fn start(&mut self) {
1891 for task in self
1892 .workers
1893 .iter_mut()
1894 .filter(|task| task.worker.has_state() && !task.worker.is_running())
1895 {
1896 task.worker.start();
1897 }
1898 }
1899
1900 async fn stop(&mut self) {
1901 tracing::debug!(instance_id = ?self.instance_id, "StorageDevice stopping...");
1902 for task in self
1903 .workers
1904 .iter_mut()
1905 .filter(|task| task.worker.has_state() && task.worker.is_running())
1906 {
1907 task.worker.stop().await;
1908 }
1909 }
1910
1911 fn supports_save_restore(&mut self) -> Option<&mut dyn SaveRestoreVmbusDevice> {
1912 Some(self)
1913 }
1914}
1915
1916#[async_trait]
1917impl SaveRestoreVmbusDevice for StorageDevice {
1918 async fn save(&mut self) -> Result<SavedStateBlob, SaveError> {
1919 Ok(SavedStateBlob::new(self.save()?))
1920 }
1921
1922 async fn restore(
1923 &mut self,
1924 control: RestoreControl<'_>,
1925 state: SavedStateBlob,
1926 ) -> Result<(), RestoreError> {
1927 self.restore(control, state.parse()?).await
1928 }
1929}
1930
1931#[cfg(test)]
1932mod tests {
1933 use super::*;
1934 use crate::test_helpers::TestWorker;
1935 use crate::test_helpers::parse_guest_completion;
1936 use crate::test_helpers::parse_guest_completion_check_flags_status;
1937 use pal_async::DefaultDriver;
1938 use pal_async::async_test;
1939 use scsi::srb::SrbStatus;
1940 use scsi_buffers::OwnedRequestBuffers;
1941 use std::task::Wake;
1942 use test_with_tracing::test;
1943 use vmbus_channel::connected_async_channels;
1944 use vmcore::vm_task::SingleDriverBackend;
1945 use vmcore::vm_task::VmTaskDriverSource;
1946
1947 impl Clone for ScsiController {
1951 fn clone(&self) -> Self {
1952 ScsiController {
1953 state: self.state.clone(),
1954 }
1955 }
1956 }
1957
1958 #[test]
1959 fn test_rescan_notification_latches_coalesces_and_resets() {
1960 let notification = RescanNotification::default();
1961 let mut receiver = notification.receiver();
1962 let mut cx = Context::from_waker(Waker::noop());
1963 notification.notify();
1964 notification.notify();
1965 assert!(receiver.poll(&mut cx).is_ready());
1966 assert!(receiver.poll(&mut cx).is_pending());
1967
1968 notification.notify();
1969 notification.reset();
1970 assert!(receiver.poll(&mut cx).is_pending());
1971 notification.notify();
1972 assert!(receiver.poll(&mut cx).is_ready());
1973
1974 assert!(receiver.poll(&mut cx).is_pending());
1975 drop(receiver);
1976 notification.notify();
1977 assert!(notification.receiver().poll(&mut cx).is_ready());
1978 }
1979
1980 #[derive(Default)]
1981 struct RescanWakeCount(AtomicU32);
1982
1983 impl Wake for RescanWakeCount {
1984 fn wake(self: Arc<Self>) {
1985 self.0.fetch_add(1, Relaxed);
1986 }
1987 }
1988
1989 #[test]
1990 fn test_rescan_notification_rearms_and_updates_waker() {
1991 let notification = RescanNotification::default();
1992 let first = Arc::new(RescanWakeCount::default());
1993 let second = Arc::new(RescanWakeCount::default());
1994 let first_waker = Waker::from(first.clone());
1995 let second_waker = Waker::from(second.clone());
1996 let mut first_cx = Context::from_waker(&first_waker);
1997 let mut second_cx = Context::from_waker(&second_waker);
1998
1999 let mut receiver = notification.receiver();
2000 assert!(receiver.poll(&mut first_cx).is_pending());
2001 assert!(receiver.poll(&mut first_cx).is_pending());
2002 assert!(receiver.poll(&mut second_cx).is_pending());
2003 notification.notify();
2004 assert_eq!(first.0.load(Relaxed), 0);
2005 assert_eq!(second.0.load(Relaxed), 1);
2006 assert!(receiver.poll(&mut second_cx).is_ready());
2007
2008 assert!(receiver.poll(&mut second_cx).is_pending());
2009 let delayed_waker = notification.waker.lock().clone().unwrap();
2012 delayed_waker.wake();
2013 assert!(receiver.poll(&mut second_cx).is_pending());
2014 notification.notify();
2015 assert_eq!(second.0.load(Relaxed), 3);
2016 assert!(receiver.poll(&mut second_cx).is_ready());
2017 }
2018
2019 #[async_test]
2020 async fn test_channel_working(driver: DefaultDriver) {
2021 let (host, guest) = connected_async_channels(16 * 1024);
2023 let guest_queue = Queue::new(guest).unwrap();
2024
2025 let test_guest_mem = GuestMemory::allocate(16384);
2026 let controller = ScsiController::new();
2027 let disk = scsidisk::SimpleScsiDisk::new(
2028 disklayer_ram::ram_disk(10 * 1024 * 1024, false).unwrap(),
2029 Default::default(),
2030 );
2031 controller
2032 .attach(
2033 ScsiPath {
2034 path: 0,
2035 target: 0,
2036 lun: 0,
2037 },
2038 ScsiControllerDisk::new(Arc::new(disk)),
2039 )
2040 .unwrap();
2041
2042 let test_worker = TestWorker::start(
2043 controller.clone(),
2044 driver.clone(),
2045 test_guest_mem.clone(),
2046 host,
2047 None,
2048 );
2049
2050 let mut guest = test_helpers::TestGuest {
2051 queue: guest_queue,
2052 transaction_id: 0,
2053 };
2054
2055 guest.perform_protocol_negotiation().await;
2056
2057 const IO_LEN: usize = 4 * 1024;
2059 let write_buf = [7u8; IO_LEN];
2060 let write_gpa = 4 * 1024u64;
2061 test_guest_mem.write_at(write_gpa, &write_buf).unwrap();
2062 guest
2063 .send_write_packet(ScsiPath::default(), write_gpa, 1, IO_LEN)
2064 .await;
2065
2066 guest
2067 .verify_completion(|p| test_helpers::parse_guest_completed_io(p, SrbStatus::SUCCESS))
2068 .await;
2069
2070 let read_gpa = 8 * 1024u64;
2071 guest
2072 .send_read_packet(ScsiPath::default(), read_gpa, 1, IO_LEN)
2073 .await;
2074
2075 guest
2076 .verify_completion(|p| test_helpers::parse_guest_completed_io(p, SrbStatus::SUCCESS))
2077 .await;
2078 let mut read_buf = [0u8; IO_LEN];
2079 test_guest_mem.read_at(read_gpa, &mut read_buf).unwrap();
2080 for (b1, b2) in read_buf.iter().zip(write_buf.iter()) {
2081 assert_eq!(b1, b2);
2082 }
2083
2084 guest.verify_graceful_close(test_worker).await;
2086 }
2087
2088 #[async_test]
2089 async fn test_packet_sizes(driver: DefaultDriver) {
2090 let (host, guest) = connected_async_channels(16384);
2092 let guest_queue = Queue::new(guest).unwrap();
2093
2094 let test_guest_mem = GuestMemory::allocate(1024);
2095 let controller = ScsiController::new();
2096
2097 let _worker = TestWorker::start(
2098 controller.clone(),
2099 driver.clone(),
2100 test_guest_mem,
2101 host,
2102 None,
2103 );
2104
2105 let mut guest = test_helpers::TestGuest {
2106 queue: guest_queue,
2107 transaction_id: 0,
2108 };
2109
2110 let negotiate_packet = storvsp_protocol::Packet {
2111 operation: storvsp_protocol::Operation::BEGIN_INITIALIZATION,
2112 flags: 0,
2113 status: storvsp_protocol::NtStatus::SUCCESS,
2114 };
2115 guest
2116 .send_data_packet_sync(&[negotiate_packet.as_bytes()])
2117 .await;
2118
2119 guest.verify_completion(parse_guest_completion).await;
2120
2121 let header = storvsp_protocol::Packet {
2122 operation: storvsp_protocol::Operation::QUERY_PROTOCOL_VERSION,
2123 flags: 0,
2124 status: storvsp_protocol::NtStatus::SUCCESS,
2125 };
2126
2127 let mut buf = [0u8; 128];
2128 storvsp_protocol::ProtocolVersion {
2129 major_minor: !0,
2130 reserved: 0,
2131 }
2132 .write_to_prefix(&mut buf[..])
2133 .unwrap(); for &(len, resp_len) in &[(48, 48), (50, 56), (56, 56), (64, 64), (72, 64)] {
2136 guest
2137 .send_data_packet_sync(&[header.as_bytes(), &buf[..len - size_of_val(&header)]])
2138 .await;
2139
2140 guest
2141 .verify_completion(|packet| {
2142 let IncomingPacket::Completion(packet) = packet else {
2143 unreachable!()
2144 };
2145 assert_eq!(packet.reader().len(), resp_len);
2146 assert_eq!(
2147 packet
2148 .reader()
2149 .read_plain::<storvsp_protocol::Packet>()
2150 .unwrap()
2151 .status,
2152 storvsp_protocol::NtStatus::REVISION_MISMATCH
2153 );
2154 Ok(())
2155 })
2156 .await;
2157 }
2158 }
2159
2160 #[async_test]
2161 async fn test_wrong_first_packet(driver: DefaultDriver) {
2162 let (host, guest) = connected_async_channels(16384);
2164 let guest_queue = Queue::new(guest).unwrap();
2165
2166 let test_guest_mem = GuestMemory::allocate(1024);
2167 let controller = ScsiController::new();
2168
2169 let _worker = TestWorker::start(
2170 controller.clone(),
2171 driver.clone(),
2172 test_guest_mem,
2173 host,
2174 None,
2175 );
2176
2177 let mut guest = test_helpers::TestGuest {
2178 queue: guest_queue,
2179 transaction_id: 0,
2180 };
2181
2182 let negotiate_packet = storvsp_protocol::Packet {
2184 operation: storvsp_protocol::Operation::END_INITIALIZATION,
2185 flags: 0,
2186 status: storvsp_protocol::NtStatus::SUCCESS,
2187 };
2188 guest
2189 .send_data_packet_sync(&[negotiate_packet.as_bytes()])
2190 .await;
2191
2192 guest
2193 .verify_completion(|packet| {
2194 let IncomingPacket::Completion(packet) = packet else {
2195 unreachable!()
2196 };
2197 assert_eq!(
2198 packet
2199 .reader()
2200 .read_plain::<storvsp_protocol::Packet>()
2201 .unwrap()
2202 .status,
2203 storvsp_protocol::NtStatus::INVALID_DEVICE_STATE
2204 );
2205 Ok(())
2206 })
2207 .await;
2208 }
2209
2210 #[async_test]
2211 async fn test_unrecognized_operation(driver: DefaultDriver) {
2212 let (host, guest) = connected_async_channels(16384);
2214 let guest_queue = Queue::new(guest).unwrap();
2215
2216 let test_guest_mem = GuestMemory::allocate(1024);
2217 let controller = ScsiController::new();
2218
2219 let worker = TestWorker::start(
2220 controller.clone(),
2221 driver.clone(),
2222 test_guest_mem,
2223 host,
2224 None,
2225 );
2226
2227 let mut guest = test_helpers::TestGuest {
2228 queue: guest_queue,
2229 transaction_id: 0,
2230 };
2231
2232 let negotiate_packet = storvsp_protocol::Packet {
2234 operation: storvsp_protocol::Operation::REMOVE_DEVICE,
2235 flags: 0,
2236 status: storvsp_protocol::NtStatus::SUCCESS,
2237 };
2238 guest
2239 .send_data_packet_sync(&[negotiate_packet.as_bytes()])
2240 .await;
2241
2242 match worker.teardown().await {
2243 Err(WorkerError::PacketError(PacketError::UnrecognizedOperation(
2244 storvsp_protocol::Operation::REMOVE_DEVICE,
2245 ))) => {}
2246 result => panic!("Worker failed with unexpected result {:?}!", result),
2247 }
2248 }
2249
2250 #[async_test]
2251 async fn test_too_many_subchannels(driver: DefaultDriver) {
2252 let (host, guest) = connected_async_channels(16384);
2254 let guest_queue = Queue::new(guest).unwrap();
2255
2256 let test_guest_mem = GuestMemory::allocate(1024);
2257 let controller = ScsiController::new();
2258
2259 let _worker = TestWorker::start(
2260 controller.clone(),
2261 driver.clone(),
2262 test_guest_mem,
2263 host,
2264 None,
2265 );
2266
2267 let mut guest = test_helpers::TestGuest {
2268 queue: guest_queue,
2269 transaction_id: 0,
2270 };
2271
2272 let negotiate_packet = storvsp_protocol::Packet {
2273 operation: storvsp_protocol::Operation::BEGIN_INITIALIZATION,
2274 flags: 0,
2275 status: storvsp_protocol::NtStatus::SUCCESS,
2276 };
2277 guest
2278 .send_data_packet_sync(&[negotiate_packet.as_bytes()])
2279 .await;
2280 guest.verify_completion(parse_guest_completion).await;
2281
2282 let version_packet = storvsp_protocol::Packet {
2283 operation: storvsp_protocol::Operation::QUERY_PROTOCOL_VERSION,
2284 flags: 0,
2285 status: storvsp_protocol::NtStatus::SUCCESS,
2286 };
2287 let version = storvsp_protocol::ProtocolVersion {
2288 major_minor: storvsp_protocol::VERSION_BLUE,
2289 reserved: 0,
2290 };
2291 guest
2292 .send_data_packet_sync(&[version_packet.as_bytes(), version.as_bytes()])
2293 .await;
2294 guest.verify_completion(parse_guest_completion).await;
2295
2296 let properties_packet = storvsp_protocol::Packet {
2297 operation: storvsp_protocol::Operation::QUERY_PROPERTIES,
2298 flags: 0,
2299 status: storvsp_protocol::NtStatus::SUCCESS,
2300 };
2301 guest
2302 .send_data_packet_sync(&[properties_packet.as_bytes()])
2303 .await;
2304
2305 guest.verify_completion(parse_guest_completion).await;
2306
2307 let negotiate_packet = storvsp_protocol::Packet {
2308 operation: storvsp_protocol::Operation::CREATE_SUB_CHANNELS,
2309 flags: 0,
2310 status: storvsp_protocol::NtStatus::SUCCESS,
2311 };
2312 guest
2314 .send_data_packet_sync(&[negotiate_packet.as_bytes(), 1_u16.as_bytes()])
2315 .await;
2316
2317 guest
2318 .verify_completion(|packet| {
2319 let IncomingPacket::Completion(packet) = packet else {
2320 unreachable!()
2321 };
2322 assert_eq!(
2323 packet
2324 .reader()
2325 .read_plain::<storvsp_protocol::Packet>()
2326 .unwrap()
2327 .status,
2328 storvsp_protocol::NtStatus::INVALID_PARAMETER
2329 );
2330 Ok(())
2331 })
2332 .await;
2333 }
2334
2335 #[async_test]
2336 async fn test_begin_init_on_ready(driver: DefaultDriver) {
2337 let (host, guest) = connected_async_channels(16384);
2339 let guest_queue = Queue::new(guest).unwrap();
2340
2341 let test_guest_mem = GuestMemory::allocate(1024);
2342 let controller = ScsiController::new();
2343
2344 let _worker = TestWorker::start(
2345 controller.clone(),
2346 driver.clone(),
2347 test_guest_mem,
2348 host,
2349 None,
2350 );
2351
2352 let mut guest = test_helpers::TestGuest {
2353 queue: guest_queue,
2354 transaction_id: 0,
2355 };
2356
2357 guest.perform_protocol_negotiation().await;
2358
2359 let negotiate_packet = storvsp_protocol::Packet {
2361 operation: storvsp_protocol::Operation::BEGIN_INITIALIZATION,
2362 flags: 0,
2363 status: storvsp_protocol::NtStatus::SUCCESS,
2364 };
2365 guest
2366 .send_data_packet_sync(&[negotiate_packet.as_bytes()])
2367 .await;
2368
2369 guest
2370 .verify_completion(|p| {
2371 parse_guest_completion_check_flags_status(
2372 p,
2373 0,
2374 storvsp_protocol::NtStatus::INVALID_DEVICE_STATE,
2375 )
2376 })
2377 .await;
2378 }
2379
2380 #[async_test]
2381 async fn test_hot_add_remove(driver: DefaultDriver) {
2382 let (host, guest) = connected_async_channels(16 * 1024);
2384 let guest_queue = Queue::new(guest).unwrap();
2385
2386 let test_guest_mem = GuestMemory::allocate(16384);
2387 let controller = ScsiController::new();
2389
2390 let test_worker = TestWorker::start(
2391 controller.clone(),
2392 driver.clone(),
2393 test_guest_mem.clone(),
2394 host,
2395 None,
2396 );
2397
2398 let mut guest = test_helpers::TestGuest {
2399 queue: guest_queue,
2400 transaction_id: 0,
2401 };
2402
2403 guest.perform_protocol_negotiation().await;
2404
2405 let mut lun_list_buffer: [u8; 256] = [0; 256];
2407 let mut disk_count = 0;
2408 guest
2409 .send_report_luns_packet(ScsiPath::default(), 0, lun_list_buffer.len())
2410 .await;
2411 guest
2412 .verify_completion(|p| {
2413 test_helpers::parse_guest_completed_io_check_tx_len(p, SrbStatus::SUCCESS, Some(8))
2414 })
2415 .await;
2416 test_guest_mem.read_at(0, &mut lun_list_buffer).unwrap();
2417 let lun_list_size = u32::from_be_bytes(lun_list_buffer[0..4].try_into().unwrap());
2418 assert_eq!(lun_list_size, disk_count as u32 * 8);
2419
2420 const IO_LEN: usize = 4 * 1024;
2422 let write_buf = [7u8; IO_LEN];
2423 let write_gpa = 4 * 1024u64;
2424 test_guest_mem.write_at(write_gpa, &write_buf).unwrap();
2425
2426 guest
2427 .send_write_packet(ScsiPath::default(), write_gpa, 1, IO_LEN)
2428 .await;
2429 guest
2430 .verify_completion(|p| {
2431 test_helpers::parse_guest_completed_io(p, SrbStatus::INVALID_LUN)
2432 })
2433 .await;
2434
2435 for lun in 0..4 {
2437 let disk = scsidisk::SimpleScsiDisk::new(
2438 disklayer_ram::ram_disk(10 * 1024 * 1024, false).unwrap(),
2439 Default::default(),
2440 );
2441 controller
2442 .attach(
2443 ScsiPath {
2444 path: 0,
2445 target: 0,
2446 lun,
2447 },
2448 ScsiControllerDisk::new(Arc::new(disk)),
2449 )
2450 .unwrap();
2451 guest
2452 .verify_completion(test_helpers::parse_guest_enumerate_bus)
2453 .await;
2454
2455 disk_count += 1;
2456 guest
2457 .send_report_luns_packet(ScsiPath::default(), 0, 256)
2458 .await;
2459 guest
2460 .verify_completion(|p| {
2461 test_helpers::parse_guest_completed_io_check_tx_len(
2462 p,
2463 SrbStatus::SUCCESS,
2464 Some((disk_count + 1) * 8),
2465 )
2466 })
2467 .await;
2468 test_guest_mem.read_at(0, &mut lun_list_buffer).unwrap();
2469 let lun_list_size = u32::from_be_bytes(lun_list_buffer[0..4].try_into().unwrap());
2470 assert_eq!(lun_list_size, disk_count as u32 * 8);
2471
2472 guest
2473 .send_write_packet(
2474 ScsiPath {
2475 path: 0,
2476 target: 0,
2477 lun,
2478 },
2479 write_gpa,
2480 1,
2481 IO_LEN,
2482 )
2483 .await;
2484 guest
2485 .verify_completion(|p| {
2486 test_helpers::parse_guest_completed_io(p, SrbStatus::SUCCESS)
2487 })
2488 .await;
2489 }
2490
2491 for lun in 0..4 {
2493 controller
2494 .remove(ScsiPath {
2495 path: 0,
2496 target: 0,
2497 lun,
2498 })
2499 .unwrap();
2500 guest
2501 .verify_completion(test_helpers::parse_guest_enumerate_bus)
2502 .await;
2503
2504 disk_count -= 1;
2505 guest
2506 .send_report_luns_packet(ScsiPath::default(), 0, 4096)
2507 .await;
2508 guest
2509 .verify_completion(|p| {
2510 test_helpers::parse_guest_completed_io_check_tx_len(
2511 p,
2512 SrbStatus::SUCCESS,
2513 Some((disk_count + 1) * 8),
2514 )
2515 })
2516 .await;
2517 test_guest_mem.read_at(0, &mut lun_list_buffer).unwrap();
2518 let lun_list_size = u32::from_be_bytes(lun_list_buffer[0..4].try_into().unwrap());
2519 assert_eq!(lun_list_size, disk_count as u32 * 8);
2520
2521 guest
2522 .send_write_packet(
2523 ScsiPath {
2524 path: 0,
2525 target: 0,
2526 lun,
2527 },
2528 write_gpa,
2529 1,
2530 IO_LEN,
2531 )
2532 .await;
2533 guest
2534 .verify_completion(|p| {
2535 test_helpers::parse_guest_completed_io(p, SrbStatus::INVALID_LUN)
2536 })
2537 .await;
2538 }
2539
2540 guest.verify_graceful_close(test_worker).await;
2541 }
2542
2543 #[async_test]
2544 async fn test_async_disk(driver: DefaultDriver) {
2545 let device = disklayer_ram::ram_disk(64 * 1024, false).unwrap();
2546 let controller = ScsiController::new();
2547 let disk = ScsiControllerDisk::new(Arc::new(scsidisk::SimpleScsiDisk::new(
2548 device,
2549 Default::default(),
2550 )));
2551 controller
2552 .attach(
2553 ScsiPath {
2554 path: 0,
2555 target: 0,
2556 lun: 0,
2557 },
2558 disk,
2559 )
2560 .unwrap();
2561
2562 let (host, guest) = connected_async_channels(16 * 1024);
2563 let guest_queue = Queue::new(guest).unwrap();
2564
2565 let mut guest = test_helpers::TestGuest {
2566 queue: guest_queue,
2567 transaction_id: 0,
2568 };
2569
2570 let test_guest_mem = GuestMemory::allocate(16384);
2571 let worker = TestWorker::start(
2572 controller.clone(),
2573 &driver,
2574 test_guest_mem.clone(),
2575 host,
2576 None,
2577 );
2578
2579 let negotiate_packet = storvsp_protocol::Packet {
2580 operation: storvsp_protocol::Operation::BEGIN_INITIALIZATION,
2581 flags: 0,
2582 status: storvsp_protocol::NtStatus::SUCCESS,
2583 };
2584 guest
2585 .send_data_packet_sync(&[negotiate_packet.as_bytes()])
2586 .await;
2587 guest.verify_completion(parse_guest_completion).await;
2588
2589 let version_packet = storvsp_protocol::Packet {
2590 operation: storvsp_protocol::Operation::QUERY_PROTOCOL_VERSION,
2591 flags: 0,
2592 status: storvsp_protocol::NtStatus::SUCCESS,
2593 };
2594 let version = storvsp_protocol::ProtocolVersion {
2595 major_minor: storvsp_protocol::VERSION_BLUE,
2596 reserved: 0,
2597 };
2598 guest
2599 .send_data_packet_sync(&[version_packet.as_bytes(), version.as_bytes()])
2600 .await;
2601 guest.verify_completion(parse_guest_completion).await;
2602
2603 let properties_packet = storvsp_protocol::Packet {
2604 operation: storvsp_protocol::Operation::QUERY_PROPERTIES,
2605 flags: 0,
2606 status: storvsp_protocol::NtStatus::SUCCESS,
2607 };
2608 guest
2609 .send_data_packet_sync(&[properties_packet.as_bytes()])
2610 .await;
2611 guest.verify_completion(parse_guest_completion).await;
2612
2613 let negotiate_packet = storvsp_protocol::Packet {
2614 operation: storvsp_protocol::Operation::END_INITIALIZATION,
2615 flags: 0,
2616 status: storvsp_protocol::NtStatus::SUCCESS,
2617 };
2618 guest
2619 .send_data_packet_sync(&[negotiate_packet.as_bytes()])
2620 .await;
2621 guest.verify_completion(parse_guest_completion).await;
2622
2623 const IO_LEN: usize = 4 * 1024;
2624 let write_buf = [7u8; IO_LEN];
2625 let write_gpa = 4 * 1024u64;
2626 test_guest_mem.write_at(write_gpa, &write_buf).unwrap();
2627 guest
2628 .send_write_packet(ScsiPath::default(), write_gpa, 1, IO_LEN)
2629 .await;
2630 guest
2631 .verify_completion(|p| test_helpers::parse_guest_completed_io(p, SrbStatus::SUCCESS))
2632 .await;
2633
2634 let read_gpa = 8 * 1024u64;
2635 guest
2636 .send_read_packet(ScsiPath::default(), read_gpa, 1, IO_LEN)
2637 .await;
2638 guest
2639 .verify_completion(|p| test_helpers::parse_guest_completed_io(p, SrbStatus::SUCCESS))
2640 .await;
2641 let mut read_buf = [0u8; IO_LEN];
2642 test_guest_mem.read_at(read_gpa, &mut read_buf).unwrap();
2643 for (b1, b2) in read_buf.iter().zip(write_buf.iter()) {
2644 assert_eq!(b1, b2);
2645 }
2646
2647 guest.verify_graceful_close(worker).await;
2648 }
2649 fn make_medium_removal_request(prevent: bool) -> Request {
2650 let cdb = scsi::CdbMediaRemoval {
2651 operation_code: ScsiOp::MEDIUM_REMOVAL,
2652 lun: 0,
2653 reserved: [0; 2],
2654 flags: scsi::MediaRemovalFlags::new()
2655 .with_prevent(prevent)
2656 .with_persistent(false),
2657 };
2658 let mut data = [0u8; 16];
2659 data[..size_of::<scsi::CdbMediaRemoval>()].copy_from_slice(cdb.as_bytes());
2660 Request {
2661 cdb: data,
2662 srb_flags: 0,
2663 }
2664 }
2665
2666 fn make_eject_request() -> Request {
2667 let cdb = scsi::StartStop {
2668 operation_code: ScsiOp::START_STOP_UNIT,
2669 immediate: 0,
2670 reserved2: [0; 2],
2671 flag: scsi::StartStopFlags::new()
2672 .with_start(false)
2673 .with_load_eject(true),
2674 control: 0,
2675 };
2676 let mut data = [0u8; 16];
2677 data[..size_of::<scsi::StartStop>()].copy_from_slice(cdb.as_bytes());
2678 Request {
2679 cdb: data,
2680 srb_flags: 0,
2681 }
2682 }
2683
2684 #[async_test]
2695 async fn test_close_ends_the_nexus(driver: DefaultDriver) {
2696 let controller = ScsiController::new();
2697 let dvd = Arc::new(scsidisk::scsidvd::SimpleScsiDvd::new(Some(
2698 disklayer_ram::ram_disk(2 * 1024 * 1024, false).unwrap(),
2699 )));
2700 controller
2701 .attach(
2702 ScsiPath {
2703 path: 0,
2704 target: 0,
2705 lun: 0,
2706 },
2707 ScsiControllerDisk::new(dvd.clone()),
2708 )
2709 .unwrap();
2710
2711 let mut device = StorageDevice::build_scsi(
2712 &VmTaskDriverSource::new(SingleDriverBackend::new(driver)),
2713 &controller,
2714 Guid::new_random(),
2715 0,
2716 64,
2717 );
2718
2719 let external_data = OwnedRequestBuffers::new(&[0]);
2720 let guest_mem = GuestMemory::allocate(4096);
2721 let buffers = external_data.buffer(&guest_mem);
2722
2723 let locked = dvd
2726 .execute_scsi(&buffers, &make_medium_removal_request(true))
2727 .await;
2728 assert_eq!(locked.scsi_status, ScsiStatus::GOOD);
2729
2730 let refused = dvd.execute_scsi(&buffers, &make_eject_request()).await;
2733 assert_ne!(
2734 refused.scsi_status,
2735 ScsiStatus::GOOD,
2736 "the tray lock did not take, so the rest of this test proves nothing"
2737 );
2738
2739 device.close(0).await;
2740
2741 let allowed = dvd.execute_scsi(&buffers, &make_eject_request()).await;
2744 assert_eq!(
2745 allowed.scsi_status,
2746 ScsiStatus::GOOD,
2747 "the tray lock outlived the nexus"
2748 );
2749 }
2750}