1use super::LoadedVm;
7use crate::nvme_manager::NvmeDiskConfig;
8use crate::worker::NicConfig;
9use anyhow::Context;
10use cvm_tracing::CVM_ALLOWED;
11use disk_backend::Disk;
12use disk_backend::resolve::ResolveDiskParameters;
13use disk_backend_resources::AutoFormattedDiskHandle;
14use disk_blockdevice::OpenBlockDeviceConfig;
15use futures::StreamExt;
16use guest_emulation_transport::api::platform_settings::DevicePlatformSettings;
17use guid::Guid;
18use ide_resources::GuestMedia;
19use ide_resources::IdeControllerConfig;
20use ide_resources::IdeDeviceConfig;
21use ide_resources::IdePath;
22use mesh::CancelContext;
23use mesh::rpc::Rpc;
24use mesh::rpc::RpcError;
25use mesh::rpc::RpcSend;
26use nvme_resources::NamespaceDefinition;
27use nvme_resources::NvmeControllerHandle;
28use scsidisk_resources::SimpleScsiDiskHandle;
29use scsidisk_resources::SimpleScsiDvdHandle;
30use scsidisk_resources::SimpleScsiDvdRequest;
31use std::collections::HashMap;
32use std::error::Error as _;
33use std::fmt::Write;
34use std::path::Path;
35use std::path::PathBuf;
36use std::time::Duration;
37use storage_string::InvalidAsciiString;
38use storvsp_resources::ScsiControllerHandle;
39use storvsp_resources::ScsiControllerRequest;
40use storvsp_resources::ScsiDeviceAndPath;
41use thiserror::Error;
42use tracing::Instrument;
43use tracing::instrument;
44use uevent::UeventListener;
45use underhill_config::DiskParameters;
46use underhill_config::NicDevice;
47use underhill_config::PhysicalDevice;
48use underhill_config::PhysicalDevices;
49use underhill_config::StorageDisk;
50use underhill_config::Vtl2Settings;
51use underhill_config::Vtl2SettingsDynamic;
52use underhill_config::Vtl2SettingsErrorCode;
53use underhill_config::Vtl2SettingsErrorInfo;
54use underhill_threadpool::AffinitizedThreadpool;
55use vm_resource::IntoResource;
56use vm_resource::ResolveError;
57use vm_resource::Resource;
58use vm_resource::ResourceResolver;
59use vm_resource::kind::DiskHandleKind;
60use vm_resource::kind::PciDeviceHandleKind;
61use vm_resource::kind::VmbusDeviceHandleKind;
62
63#[derive(Error, Debug)]
64enum Error<'a> {
65 #[error("RPC error")]
66 Rpc(#[source] RpcError),
67 #[error("cannot add/remove storage controllers at runtime")]
68 StorageCannotAddRemoveControllerAtRuntime,
69 #[error("Striping devices don't support runtime change")]
70 StripStorageCannotChangeControllerAtRuntime,
71 #[error("failed to open disk")]
72 StorageCannotOpenDisk(#[source] ResolveError),
73 #[error("could not disable io scheduling")]
74 StorageCannotDisableIoScheduling(#[source] std::io::Error),
75 #[error("failed to open {device_type:?} disk {path} at {instance_id}/{sub_device_path}")]
76 StorageCannotOpenVtl2Device {
77 device_type: underhill_config::DeviceType,
78 instance_id: Guid,
79 sub_device_path: u32,
80 path: &'a PathBuf,
81 #[source]
82 source: std::io::Error,
83 },
84 #[error("failed to find {device_type:?} disk at {instance_id}/{sub_device_path}")]
85 StorageCannotFindVtl2Device {
86 device_type: underhill_config::DeviceType,
87 instance_id: Guid,
88 sub_device_path: u32,
89 #[source]
90 source: anyhow::Error,
91 },
92 #[error("no SCSI controller {0}")]
93 StorageScsiControllerNotFound(Guid),
94 #[error("failed to add disk")]
95 StorageScsiPathInUse(#[source] anyhow::Error),
96 #[error("failed to add disk at lun {0}")]
97 StorageAttachDiskFailed(u8, #[source] anyhow::Error),
98 #[error("failed to remove disk")]
99 StorageScsiPathNotInUse(#[source] anyhow::Error),
100 #[error("failed to remove disk at lun {0}")]
101 StorageRemoveDiskFailed(u8, #[source] anyhow::Error),
102 #[error("failed to change media at lun {0}")]
103 StorageChangeMediaFailed(u8, #[source] anyhow::Error),
104 #[error("failed to modify networking instance {0}")]
105 NetworkingModifyNicFailed(Guid, #[source] anyhow::Error),
106 #[error("failed to add network interface {0}")]
107 NetworkingAddNicFailed(Guid, #[source] anyhow::Error),
108 #[error("failed to remove network interface {0}")]
109 NetworkingRemoveNicFailed(Guid, #[source] anyhow::Error),
110 #[error("failed to parse Vendor ID: LUN = {lun:?}, vendor_id = {vendor_id:?}")]
111 StorageBadVendorId {
112 lun: u8,
113 vendor_id: &'a str,
114 #[source]
115 source: InvalidAsciiString,
116 },
117 #[error("failed to parse Product ID: LUN = {lun:?}, product_id = {product_id:?}")]
118 StorageBadProductId {
119 lun: u8,
120 product_id: &'a str,
121 #[source]
122 source: InvalidAsciiString,
123 },
124 #[error("failed to parse Device ID: LUN = {lun:?}, device_id = {device_id:?}")]
125 StorageBadDeviceId { lun: u8, device_id: &'a str },
126 #[error(
127 "failed to parse Product Revision Level: LUN = {lun:?}, product_revision_level = {product_revision_level:?}"
128 )]
129 StorageBadProductRevisionLevel {
130 lun: u8,
131 product_revision_level: &'a str,
132 #[source]
133 source: InvalidAsciiString,
134 },
135 #[error("cannot modify IDE configuration at runtime")]
136 StorageCannotModifyIdeAtRuntime,
137}
138
139impl Error<'_> {
140 fn code(&self) -> Vtl2SettingsErrorCode {
141 match self {
142 Error::Rpc(_) => Vtl2SettingsErrorCode::InternalFailure,
143 Error::StorageCannotAddRemoveControllerAtRuntime => {
144 Vtl2SettingsErrorCode::StorageCannotAddRemoveControllerAtRuntime
145 }
146 Error::StripStorageCannotChangeControllerAtRuntime => {
147 Vtl2SettingsErrorCode::StripedStorageCannotChangeControllerAtRuntime
148 }
149 Error::StorageCannotOpenDisk(_) => Vtl2SettingsErrorCode::StorageCannotOpenVtl2Device,
150 Error::StorageCannotDisableIoScheduling(_) => {
151 Vtl2SettingsErrorCode::StorageCannotOpenVtl2Device
152 }
153 Error::StorageCannotOpenVtl2Device { .. } => {
154 Vtl2SettingsErrorCode::StorageCannotOpenVtl2Device
155 }
156 Error::StorageCannotFindVtl2Device { .. } => {
157 Vtl2SettingsErrorCode::StorageCannotFindVtl2Device
158 }
159 Error::StorageScsiControllerNotFound(_) => {
160 Vtl2SettingsErrorCode::StorageScsiControllerNotFound
161 }
162 Error::StorageScsiPathInUse(_) => Vtl2SettingsErrorCode::StorageAttachDiskFailed,
163 Error::StorageAttachDiskFailed(..) => Vtl2SettingsErrorCode::StorageAttachDiskFailed,
164 Error::StorageScsiPathNotInUse(_) => Vtl2SettingsErrorCode::StorageRmDiskFailed,
165 Error::StorageRemoveDiskFailed(..) => Vtl2SettingsErrorCode::StorageRmDiskFailed,
166 Error::NetworkingModifyNicFailed(..) => {
167 Vtl2SettingsErrorCode::NetworkingModifyNicFailed
168 }
169 Error::NetworkingAddNicFailed(..) => Vtl2SettingsErrorCode::NetworkingAddNicFailed,
170 Error::NetworkingRemoveNicFailed(..) => {
171 Vtl2SettingsErrorCode::NetworkingRemoveNicFailed
172 }
173 Error::StorageBadVendorId { .. } => Vtl2SettingsErrorCode::StorageInvalidVendorId,
174 Error::StorageBadProductId { .. } => Vtl2SettingsErrorCode::StorageInvalidProductId,
175 Error::StorageBadProductRevisionLevel { .. } => {
176 Vtl2SettingsErrorCode::StorageInvalidProductRevisionLevel
177 }
178 Error::StorageCannotModifyIdeAtRuntime => {
179 Vtl2SettingsErrorCode::StorageCannotModifyIdeAtRuntime
180 }
181 Error::StorageBadDeviceId { .. } => Vtl2SettingsErrorCode::StorageInvalidDeviceId,
182 Error::StorageChangeMediaFailed { .. } => {
183 Vtl2SettingsErrorCode::StorageChangeMediaFailed
184 }
185 }
186 }
187}
188
189impl From<Error<'_>> for Vtl2SettingsErrorInfo {
190 #[track_caller]
191 fn from(e: Error<'_>) -> Vtl2SettingsErrorInfo {
192 let mut message = e.to_string();
195 let mut source = e.source();
196 while let Some(inner) = source {
197 write!(&mut message, ": {}", inner).unwrap();
198 source = inner.source();
199 }
200
201 Vtl2SettingsErrorInfo::new(e.code(), message)
202 }
203}
204
205pub enum Vtl2ConfigNicRpc {
206 Modify(Rpc<(Guid, Option<Guid>), anyhow::Result<()>>),
207 Add(Rpc<(Guid, Option<Guid>, Option<u16>), anyhow::Result<()>>),
208 Remove(Rpc<Guid, anyhow::Result<()>>),
209}
210
211pub enum Vtl2ConfigAcquireResource {
212 AddDisk(Guid, underhill_config::ScsiDisk),
213 RmDisk(Guid, underhill_config::ScsiDisk),
214 ChangeMedia(Guid, StorageDisk),
215 ModifyNic((Guid, Option<Guid>)),
216 AddNic((Guid, Option<Guid>, Option<u16>)),
217 RemoveNic(Guid),
218}
219
220pub enum Vtl2ConfigCommit {
221 AddDisk(
222 Guid,
223 ScsiDeviceAndPath,
224 Option<mesh::Sender<SimpleScsiDvdRequest>>,
225 ),
226 RmDisk(Guid, storvsp_resources::ScsiPath),
227 ChangeMedia(Guid, StorageDevicePath, Option<Resource<DiskHandleKind>>),
228 ModifyNic((Guid, Option<Guid>)),
229 AddNic((Guid, Option<Guid>, Option<u16>)),
230 RemoveNic(Guid),
231}
232
233pub struct Vtl2SettingsWorker {
235 old_settings: Vtl2SettingsDynamic,
236 device_config_send: mesh::Sender<Vtl2ConfigNicRpc>,
237 get_client: guest_emulation_transport::GuestEmulationTransportClient,
238 interfaces: DeviceInterfaces,
239}
240
241pub struct DeviceInterfaces {
242 scsi_dvds: HashMap<StorageDevicePath, mesh::Sender<SimpleScsiDvdRequest>>,
243 scsi_request: HashMap<Guid, mesh::Sender<ScsiControllerRequest>>,
244 use_nvme_vfio: bool,
245}
246
247impl Vtl2SettingsWorker {
248 pub fn new(
249 initial_settings: Vtl2SettingsDynamic,
250 device_config_send: mesh::Sender<Vtl2ConfigNicRpc>,
251 get_client: guest_emulation_transport::GuestEmulationTransportClient,
252 interfaces: DeviceInterfaces,
253 ) -> Vtl2SettingsWorker {
254 Vtl2SettingsWorker {
255 old_settings: initial_settings,
256 device_config_send,
257 get_client,
258 interfaces,
259 }
260 }
261
262 pub async fn run(&mut self, uevent_listener: &UeventListener) {
263 let mut settings_recv = self.get_client.take_vtl2_settings_recv().await.unwrap();
264
265 while let Some(req) = settings_recv.next().await {
266 req.0
267 .handle(async |buf| {
268 self.handle_modify_vtl2_settings(uevent_listener, &{ buf })
269 .await
270 })
271 .await
272 }
273 }
274
275 async fn handle_modify_vtl2_settings(
276 &mut self,
277 uevent_listener: &UeventListener,
278 buf: &[u8],
279 ) -> Result<(), Vec<Vtl2SettingsErrorInfo>> {
280 const MODIFY_VTL2_SETTINGS_TIMEOUT_IN_SECONDS: u64 = 5;
281 let mut context = CancelContext::new()
282 .with_timeout(Duration::from_secs(MODIFY_VTL2_SETTINGS_TIMEOUT_IN_SECONDS));
283
284 let old_settings = Vtl2Settings {
285 fixed: Default::default(),
286 dynamic: self.old_settings.clone(),
287 };
288 let vtl2_settings =
289 Vtl2Settings::read_from(buf, old_settings).map_err(|err| match err {
290 underhill_config::schema::ParseError::Json(err) => {
291 vec![Vtl2SettingsErrorInfo::new(
292 Vtl2SettingsErrorCode::JsonFormatError,
293 err.to_string(),
294 )]
295 }
296 underhill_config::schema::ParseError::Protobuf(err) => {
297 vec![Vtl2SettingsErrorInfo::new(
298 Vtl2SettingsErrorCode::ProtobufFormatError,
299 err.to_string(),
300 )]
301 }
302 underhill_config::schema::ParseError::Validation(err) => err.errors,
303 })?;
304
305 let new_settings = vtl2_settings.dynamic;
306
307 tracing::info!(CVM_ALLOWED, ?new_settings, "Received VTL2 settings");
308
309 let mut todos: Vec<Vtl2ConfigAcquireResource> = Vec::new();
310
311 let mut errors = Vec::new();
312
313 modify_storage_configuration(&self.old_settings, &new_settings, &mut todos, &mut errors);
314 if let Err(err) =
315 modify_network_configuration(&self.old_settings, &new_settings, &mut todos)
316 {
317 errors.push(err);
318 }
319
320 if !errors.is_empty() {
321 return Err(errors);
322 }
323
324 if errors.is_empty() {
325 let mut to_commits: Vec<Vtl2ConfigCommit> = Vec::new();
326 match self
327 .acquire_configuration_resources(
328 &mut context,
329 uevent_listener,
330 todos,
331 &mut to_commits,
332 )
333 .await
334 {
335 Err(e) => {
336 tracing::error!(
337 CVM_ALLOWED,
338 ?e,
339 "Error acquiring VTL2 configuration resources"
340 );
341 errors.push(e);
342 }
343 Ok(()) => {
344 if let Err(e) = self.commit_configuration_changes(to_commits).await {
347 tracing::error!(CVM_ALLOWED, ?e, "Error commit VTL2 configuration changes");
348 errors.push(e);
349 }
350 }
351 }
352 }
353
354 if !errors.is_empty() {
355 return Err(errors);
356 }
357
358 tracing::info!(CVM_ALLOWED, "VTL2 settings modified");
359 self.old_settings = new_settings;
360 Ok(())
361 }
362
363 async fn acquire_configuration_resources(
364 &mut self,
365 ctx: &mut CancelContext,
366 uevent_listener: &UeventListener,
367 todos: Vec<Vtl2ConfigAcquireResource>,
368 to_commits: &mut Vec<Vtl2ConfigCommit>,
369 ) -> Result<(), Vtl2SettingsErrorInfo> {
370 for todo in todos {
371 match todo {
372 Vtl2ConfigAcquireResource::AddDisk(guid, disk) => {
373 let (disk_cfg, dvd) = make_scsi_disk_config(
374 ctx,
375 &StorageContext {
376 uevent_listener,
377 use_nvme_vfio: self.interfaces.use_nvme_vfio,
378 },
379 &disk,
380 false,
381 )
382 .await?;
383 to_commits.push(Vtl2ConfigCommit::AddDisk(guid, disk_cfg, dvd));
384 }
385 Vtl2ConfigAcquireResource::RmDisk(guid, disk) => {
386 let scsi_path = scsi_path_from_config(&disk)?;
387 to_commits.push(Vtl2ConfigCommit::RmDisk(guid, scsi_path));
388 }
389 Vtl2ConfigAcquireResource::ChangeMedia(guid, disk) => {
390 let path = storage_path_from_config(&disk)?;
391 let disk_type = make_disk_type(
392 ctx,
393 &StorageContext {
394 uevent_listener,
395 use_nvme_vfio: self.interfaces.use_nvme_vfio,
396 },
397 &disk,
398 false,
399 )
400 .await?;
401 to_commits.push(Vtl2ConfigCommit::ChangeMedia(guid, path, disk_type));
402 }
403 Vtl2ConfigAcquireResource::ModifyNic(nic_settings) => {
404 to_commits.push(Vtl2ConfigCommit::ModifyNic(nic_settings));
405 }
406 Vtl2ConfigAcquireResource::AddNic(nic_settings) => {
407 to_commits.push(Vtl2ConfigCommit::AddNic(nic_settings));
408 }
409 Vtl2ConfigAcquireResource::RemoveNic(instance_id) => {
410 to_commits.push(Vtl2ConfigCommit::RemoveNic(instance_id));
411 }
412 }
413 }
414 Ok(())
415 }
416
417 async fn commit_configuration_changes(
418 &mut self,
419 to_commits: Vec<Vtl2ConfigCommit>,
420 ) -> Result<(), Vtl2SettingsErrorInfo> {
421 for commit in to_commits {
422 match commit {
423 Vtl2ConfigCommit::AddDisk(controller_id, disk_cfg, dvd) => {
424 let scsi_path = disk_cfg.path;
425 self.interfaces
426 .scsi_request
427 .get(&controller_id)
428 .ok_or(Error::StorageScsiControllerNotFound(controller_id))?
429 .call_failable(ScsiControllerRequest::AddDevice, disk_cfg)
430 .await
431 .map_err(|err| {
432 Error::StorageAttachDiskFailed(
433 scsi_path.lun,
434 Error::StorageScsiPathInUse(err.into()).into(),
435 )
436 })?;
437
438 if let Some(dvd) = dvd {
439 assert!(
440 self.interfaces
441 .scsi_dvds
442 .insert(StorageDevicePath::Scsi(scsi_path), dvd)
443 .is_none()
444 );
445 }
446 }
447 Vtl2ConfigCommit::RmDisk(controller_id, scsi_path) => {
448 self.interfaces
449 .scsi_request
450 .get(&controller_id)
451 .ok_or(Error::StorageScsiControllerNotFound(controller_id))?
452 .call_failable(ScsiControllerRequest::RemoveDevice, scsi_path)
453 .await
454 .map_err(|err| {
455 Error::StorageRemoveDiskFailed(
456 scsi_path.lun,
457 Error::StorageScsiPathNotInUse(err.into()).into(),
458 )
459 })?;
460
461 let _ = self
462 .interfaces
463 .scsi_dvds
464 .remove(&StorageDevicePath::Scsi(scsi_path));
465 }
466 Vtl2ConfigCommit::ChangeMedia(controller_id, path, disk_cfg) => {
467 let lun = if let StorageDevicePath::Scsi(scsi_path) = path {
469 scsi_path.lun
470 } else {
471 0
472 };
473
474 async {
475 let target = self
476 .interfaces
477 .scsi_dvds
478 .get(&path)
479 .ok_or(Error::StorageScsiControllerNotFound(controller_id))?;
480
481 target
482 .call_failable(SimpleScsiDvdRequest::ChangeMedia, disk_cfg)
483 .await?;
484
485 anyhow::Ok(())
486 }
487 .await
488 .map_err(|e| Error::StorageChangeMediaFailed(lun, e))?;
489 }
490 Vtl2ConfigCommit::ModifyNic(nic_settings) => {
491 let instance_id = nic_settings.0;
492 self.device_config_send
493 .call(Vtl2ConfigNicRpc::Modify, nic_settings)
494 .await
495 .map_err(Error::Rpc)?
496 .map_err(|e| Error::NetworkingModifyNicFailed(instance_id, e))?;
497 }
498 Vtl2ConfigCommit::AddNic(nic_settings) => {
499 let instance_id = nic_settings.0;
500 self.device_config_send
501 .call(Vtl2ConfigNicRpc::Add, nic_settings)
502 .await
503 .map_err(Error::Rpc)?
504 .map_err(|e| Error::NetworkingAddNicFailed(instance_id, e))?;
505 }
506 Vtl2ConfigCommit::RemoveNic(instance_id) => {
507 self.device_config_send
508 .call(Vtl2ConfigNicRpc::Remove, instance_id)
509 .await
510 .map_err(Error::Rpc)?
511 .map_err(|e| Error::NetworkingRemoveNicFailed(instance_id, e))?;
512 }
513 }
514 }
515
516 Ok(())
517 }
518}
519
520pub(crate) async fn handle_vtl2_config_rpc(
521 message: Vtl2ConfigNicRpc,
522 vm: &mut LoadedVm,
523 threadpool: &AffinitizedThreadpool,
524) {
525 match message {
526 Vtl2ConfigNicRpc::Modify(rpc) => {
527 rpc.handle(async |nic_settings| {
528 let modify_settings = vm.network_settings.as_mut().map(|settings| {
529 settings.modify_network_settings(nic_settings.0, nic_settings.1)
530 });
531 modify_settings
532 .context("network modifications not supported for this VM")?
533 .await
534 })
535 .await
536 }
537 Vtl2ConfigNicRpc::Add(rpc) => {
538 rpc.handle(async |nic_settings| {
539 vm.add_vf_manager(threadpool, nic_settings.0, nic_settings.1, nic_settings.2)
540 .await
541 })
542 .await
543 }
544 Vtl2ConfigNicRpc::Remove(rpc) => {
545 rpc.handle(async |instance_id| vm.remove_vf_manager(instance_id).await)
546 .await
547 }
548 }
549}
550
551pub async fn disk_from_disk_type(
552 disk_type: Resource<DiskHandleKind>,
553 read_only: bool,
554 resolver: &ResourceResolver,
555) -> Result<Disk, Vtl2SettingsErrorInfo> {
556 let disk = resolver
557 .resolve(
558 disk_type,
559 ResolveDiskParameters {
560 read_only,
561 _async_trait_workaround: &(),
562 },
563 )
564 .await
565 .map_err(Error::StorageCannotOpenDisk)?;
566 Ok(disk.0)
567}
568
569fn modify_storage_configuration(
570 old_settings: &Vtl2SettingsDynamic,
571 new_settings: &Vtl2SettingsDynamic,
572 todos: &mut Vec<Vtl2ConfigAcquireResource>,
573 errors: &mut Vec<Vtl2SettingsErrorInfo>,
574) {
575 if let Err(e) = modify_ide_configuration(old_settings, new_settings, todos) {
576 errors.push(e);
577 }
578 if let Err(e) = modify_scsi_configuration(old_settings, new_settings, todos) {
579 errors.push(e);
580 }
581}
582
583fn modify_ide_configuration(
584 old_settings: &Vtl2SettingsDynamic,
585 new_settings: &Vtl2SettingsDynamic,
586 todos: &mut Vec<Vtl2ConfigAcquireResource>,
587) -> Result<(), Vtl2SettingsErrorInfo> {
588 if old_settings.ide_controller != new_settings.ide_controller {
589 if let (Some(old_ide), Some(new_ide)) =
590 (&old_settings.ide_controller, &new_settings.ide_controller)
591 {
592 if old_ide.instance_id == new_ide.instance_id {
593 let instance_id = old_ide.instance_id;
594 if old_ide.disks.len() != new_ide.disks.len() {
595 return Err(Error::StorageCannotModifyIdeAtRuntime.into());
596 }
597 for (old_disk, new_disk) in old_ide.disks.iter().zip(new_ide.disks.iter()) {
598 if old_disk.is_dvd
599 && new_disk.is_dvd
600 && old_disk.channel == new_disk.channel
601 && old_disk.location == new_disk.location
602 && old_disk.disk_params == new_disk.disk_params
603 && old_disk.physical_devices != new_disk.physical_devices
604 {
605 match (
606 old_disk.physical_devices.is_empty(),
607 new_disk.physical_devices.is_empty(),
608 ) {
609 (true, true) | (false, false) => {
610 return Err(Error::StorageCannotModifyIdeAtRuntime.into());
611 }
612 (true, false) | (false, true) => {
613 todos.push(Vtl2ConfigAcquireResource::ChangeMedia(
617 instance_id,
618 StorageDisk::Ide(new_disk.clone()),
619 ));
620 return Ok(());
621 }
622 }
623 }
624 }
625 }
626 }
627
628 return Err(Error::StorageCannotModifyIdeAtRuntime.into());
629 }
630
631 Ok(())
632}
633
634fn modify_scsi_configuration(
635 old_settings: &Vtl2SettingsDynamic,
636 new_settings: &Vtl2SettingsDynamic,
637 todos: &mut Vec<Vtl2ConfigAcquireResource>,
638) -> Result<(), Vtl2SettingsErrorInfo> {
639 let old_controller_map = create_device_map_from_settings(&old_settings.scsi_controllers);
640 let new_controller_map = create_device_map_from_settings(&new_settings.scsi_controllers);
641
642 let (_, controllers_to_remove, controllers_to_add) =
643 calculate_device_change_from_map(&old_controller_map, &new_controller_map);
644
645 if !controllers_to_add.is_empty() || !controllers_to_remove.is_empty() {
646 return Err(Error::StorageCannotAddRemoveControllerAtRuntime.into());
647 }
648
649 modify_scsi_disks_configuration(&old_controller_map, &new_controller_map, todos)?;
650
651 Ok(())
652}
653
654fn modify_scsi_disks_configuration(
655 old_controller_map: &HashMap<Guid, &underhill_config::ScsiController>,
656 new_controller_map: &HashMap<Guid, &underhill_config::ScsiController>,
657 todos: &mut Vec<Vtl2ConfigAcquireResource>,
658) -> Result<(), Vtl2SettingsErrorInfo> {
659 let (modify_disks, mut remove_disks, mut add_disks) =
660 calculate_scsi_disks_change(old_controller_map, new_controller_map)?;
661 let mut change_media_disks = Vec::new();
662
663 for (old_config, new_config, controller_id) in modify_disks {
664 if (old_config.physical_devices.is_striping()) || new_config.physical_devices.is_striping()
665 {
666 return Err(Error::StripStorageCannotChangeControllerAtRuntime.into());
667 }
668
669 match (
670 old_config.physical_devices.is_empty(),
671 new_config.physical_devices.is_empty(),
672 ) {
673 (true, true) | (false, false) => {
674 remove_disks.push((old_config, controller_id));
675 add_disks.push((new_config, controller_id));
676 }
677 (true, false) | (false, true) => {
678 if old_config.is_dvd && new_config.is_dvd {
682 change_media_disks.push((new_config, controller_id));
683 } else {
684 remove_disks.push((old_config, controller_id));
685 add_disks.push((new_config, controller_id));
686 }
687 }
688 }
689 }
690
691 for (disk, controller_id) in remove_disks {
693 if disk.physical_devices.is_striping() {
694 return Err(Error::StripStorageCannotChangeControllerAtRuntime.into());
695 }
696 todos.push(Vtl2ConfigAcquireResource::RmDisk(
697 controller_id,
698 disk.clone(),
699 ));
700 }
701
702 for (disk, controller_id) in add_disks {
704 if disk.physical_devices.is_striping() {
705 return Err(Error::StripStorageCannotChangeControllerAtRuntime.into());
706 }
707 todos.push(Vtl2ConfigAcquireResource::AddDisk(
708 controller_id,
709 disk.clone(),
710 ));
711 }
712
713 for (disk, controller_id) in change_media_disks {
715 todos.push(Vtl2ConfigAcquireResource::ChangeMedia(
716 controller_id,
717 StorageDisk::Scsi(disk.clone()),
718 ))
719 }
720
721 Ok(())
722}
723
724fn calculate_scsi_disks_change<'a>(
725 old_controller_map: &'a HashMap<Guid, &underhill_config::ScsiController>,
726 new_controller_map: &'a HashMap<Guid, &underhill_config::ScsiController>,
727) -> Result<
728 (
729 Vec<(
730 &'a underhill_config::ScsiDisk,
731 &'a underhill_config::ScsiDisk,
732 Guid,
733 )>,
734 Vec<(&'a underhill_config::ScsiDisk, Guid)>,
735 Vec<(&'a underhill_config::ScsiDisk, Guid)>,
736 ),
737 Vtl2SettingsErrorInfo,
738> {
739 let mut disks_to_modify = Vec::<(
740 &underhill_config::ScsiDisk,
741 &underhill_config::ScsiDisk,
742 Guid,
743 )>::new();
744 let mut disks_to_remove = Vec::<(&underhill_config::ScsiDisk, Guid)>::new();
745 let mut disks_to_add = Vec::<(&underhill_config::ScsiDisk, Guid)>::new();
746
747 for (instance_id, new_controller) in new_controller_map {
748 if let Some(old_controller) = old_controller_map.get(instance_id) {
749 calculate_disks_change_per_scsi_controller(
750 old_controller,
751 new_controller,
752 &mut disks_to_modify,
753 &mut disks_to_remove,
754 &mut disks_to_add,
755 )?;
756 }
757 }
758
759 Ok((disks_to_modify, disks_to_remove, disks_to_add))
760}
761
762fn calculate_disks_change_per_scsi_controller<'a>(
763 old_controller: &'a underhill_config::ScsiController,
764 new_controller: &'a underhill_config::ScsiController,
765 disks_to_modify: &mut Vec<(
766 &'a underhill_config::ScsiDisk,
767 &'a underhill_config::ScsiDisk,
768 Guid,
769 )>,
770 disks_to_remove: &mut Vec<(&'a underhill_config::ScsiDisk, Guid)>,
771 disks_to_add: &mut Vec<(&'a underhill_config::ScsiDisk, Guid)>,
772) -> Result<(), Vtl2SettingsErrorInfo> {
773 let old_slots = get_scsi_controller_slots(old_controller)?;
774 let new_slots = get_scsi_controller_slots(new_controller)?;
775
776 for i in 0..underhill_config::SCSI_LUN_NUM {
777 if old_slots[i] != new_slots[i] {
778 match (old_slots[i], new_slots[i]) {
779 (Some(old_config), Some(new_config)) => {
780 disks_to_modify.push((old_config, new_config, old_controller.instance_id))
781 }
782 (Some(old_config), None) => {
783 disks_to_remove.push((old_config, old_controller.instance_id));
784 }
785 (None, Some(new_config)) => {
786 disks_to_add.push((new_config, new_controller.instance_id));
787 }
788 (None, None) => continue,
789 }
790 }
791 }
792
793 Ok(())
794}
795
796fn get_scsi_controller_slots(
797 controller: &underhill_config::ScsiController,
798) -> Result<
799 [Option<&underhill_config::ScsiDisk>; underhill_config::SCSI_LUN_NUM],
800 Vtl2SettingsErrorInfo,
801> {
802 let mut slots: [Option<&underhill_config::ScsiDisk>; underhill_config::SCSI_LUN_NUM] =
803 [None; underhill_config::SCSI_LUN_NUM];
804
805 for disk in &controller.disks {
806 slots[disk.location as usize] = Some(disk);
807 }
808
809 Ok(slots)
810}
811
812fn storage_path_from_config(
813 disk: &StorageDisk,
814) -> Result<StorageDevicePath, Vtl2SettingsErrorInfo> {
815 match disk {
816 StorageDisk::Ide(disk) => Ok(StorageDevicePath::Ide(ide_path_from_config(disk)?)),
817 StorageDisk::Scsi(disk) => Ok(StorageDevicePath::Scsi(scsi_path_from_config(disk)?)),
818 }
819}
820
821fn ide_path_from_config(
822 disk: &underhill_config::IdeDisk,
823) -> Result<IdePath, Vtl2SettingsErrorInfo> {
824 Ok(IdePath {
825 channel: disk.channel,
826 drive: disk.location,
827 })
828}
829
830fn scsi_path_from_config(
831 disk: &underhill_config::ScsiDisk,
832) -> Result<storvsp_resources::ScsiPath, Vtl2SettingsErrorInfo> {
833 Ok(storvsp_resources::ScsiPath {
834 path: 0,
835 target: 0,
836 lun: disk.location,
837 })
838}
839
840fn modify_network_configuration(
841 old_settings: &Vtl2SettingsDynamic,
842 new_settings: &Vtl2SettingsDynamic,
843 todos: &mut Vec<Vtl2ConfigAcquireResource>,
844) -> Result<(), Vtl2SettingsErrorInfo> {
845 let mut old_settings_map = HashMap::new();
846 for nic_settings in &old_settings.nic_devices {
847 old_settings_map.insert(
848 nic_settings.instance_id,
849 nic_settings.subordinate_instance_id,
850 );
851 }
852 for nic_settings in &new_settings.nic_devices {
853 if let Some(existing_subordinate_id) = old_settings_map.remove(&nic_settings.instance_id) {
854 if existing_subordinate_id.is_some() != nic_settings.subordinate_instance_id.is_some() {
855 todos.push(Vtl2ConfigAcquireResource::ModifyNic((
856 nic_settings.instance_id,
857 nic_settings.subordinate_instance_id,
858 )));
859 }
860 } else {
861 todos.push(Vtl2ConfigAcquireResource::AddNic((
862 nic_settings.instance_id,
863 nic_settings.subordinate_instance_id,
864 nic_settings.max_sub_channels,
865 )));
866 }
867 }
868 if !old_settings_map.is_empty() {
869 for nic_settings in old_settings_map {
870 todos.push(Vtl2ConfigAcquireResource::RemoveNic(nic_settings.0));
871 }
872 }
873 Ok(())
874}
875
876pub struct UhIdeControllerConfig {
877 pub config: IdeControllerConfig,
878 pub dvds: Vec<(IdePath, mesh::Sender<SimpleScsiDvdRequest>)>,
879}
880
881pub struct UhScsiControllerConfig {
882 pub handle: ScsiControllerHandle,
883 pub request: mesh::Sender<ScsiControllerRequest>,
884 pub dvds: Vec<(
885 storvsp_resources::ScsiPath,
886 mesh::Sender<SimpleScsiDvdRequest>,
887 )>,
888}
889
890#[cfg_attr(not(feature = "vpci"), expect(dead_code))]
891pub struct UhVpciDeviceConfig {
892 pub instance_id: Guid,
893 pub resource: Resource<PciDeviceHandleKind>,
894}
895
896#[derive(Debug, Eq, PartialEq, Hash)]
897pub enum StorageDevicePath {
898 Ide(IdePath),
899 Scsi(storvsp_resources::ScsiPath),
900}
901
902async fn make_disk_type_from_physical_devices(
903 ctx: &mut CancelContext,
904 storage_context: &StorageContext<'_>,
905 physical_devices: &PhysicalDevices,
906 ntfs_guid: Option<Guid>,
907 read_only: bool,
908 is_restoring: bool,
909) -> Result<Resource<DiskHandleKind>, Vtl2SettingsErrorInfo> {
910 let disk_type = match *physical_devices {
911 PhysicalDevices::Single { ref device } => {
912 make_disk_type_from_physical_device(ctx, storage_context, device, read_only).await?
913 }
914 PhysicalDevices::Striped {
915 ref devices,
916 chunk_size_in_kb,
917 } => {
918 let mut physical_disk_configs = Vec::new();
919 for single_device in devices {
920 let disk = make_disk_type_from_physical_device(
921 ctx,
922 storage_context,
923 single_device,
924 read_only,
925 )
926 .await?;
927
928 physical_disk_configs.push(disk);
929 }
930 Resource::new(disk_backend_resources::StripedDiskHandle {
931 devices: physical_disk_configs,
932 chunk_size_in_bytes: Some(chunk_size_in_kb * 1024),
933 logic_sector_count: None,
934 })
935 }
936 PhysicalDevices::EmptyDrive => unreachable!("handled in calling function"),
937 };
938
939 if let Some(ntfs_guid) = ntfs_guid {
940 if is_restoring {
941 tracing::debug!(disk_guid = ?ntfs_guid, "restoring - disk not candidate for auto format");
942 } else {
943 return Ok(Resource::new(AutoFormattedDiskHandle {
946 disk: disk_type,
947 guid: ntfs_guid.into(),
948 }));
949 }
950 }
951
952 Ok(disk_type)
953}
954
955struct StorageContext<'a> {
956 uevent_listener: &'a UeventListener,
957 use_nvme_vfio: bool,
958}
959
960#[instrument(skip_all, fields(CVM_ALLOWED))]
961async fn make_disk_type_from_physical_device(
962 ctx: &mut CancelContext,
963 storage_context: &StorageContext<'_>,
964 single_device: &PhysicalDevice,
965 read_only: bool,
966) -> Result<Resource<DiskHandleKind>, Vtl2SettingsErrorInfo> {
967 let controller_instance_id = single_device.vmbus_instance_id;
968 let sub_device_path = single_device.sub_device_path;
969
970 if storage_context.use_nvme_vfio
972 && matches!(
973 single_device.device_type,
974 underhill_config::DeviceType::NVMe
975 )
976 {
977 let (pci_id, devpath) = vpci_path(&controller_instance_id);
979 async {
980 ctx.until_cancelled(storage_context.uevent_listener.wait_for_devpath(&devpath))
981 .await??;
982 ctx.until_cancelled(wait_for_pci_path(&pci_id)).await?;
983 Ok(())
984 }
985 .await
986 .map_err(|err| Error::StorageCannotFindVtl2Device {
987 device_type: single_device.device_type,
988 instance_id: controller_instance_id,
989 sub_device_path,
990 source: err,
991 })?;
992
993 return Ok(Resource::new(NvmeDiskConfig {
996 pci_id,
997 nsid: sub_device_path,
998 }));
999 }
1000
1001 let devname = async {
1003 let devname = ctx
1004 .until_cancelled(async {
1005 match single_device.device_type {
1006 underhill_config::DeviceType::NVMe => {
1007 get_nvme_namespace_devname(
1008 storage_context.uevent_listener,
1009 &controller_instance_id,
1010 sub_device_path,
1011 )
1012 .await
1013 }
1014 underhill_config::DeviceType::VScsi => {
1015 get_vscsi_devname(
1016 storage_context.uevent_listener,
1017 &controller_instance_id,
1018 sub_device_path,
1019 )
1020 .await
1021 }
1022 }
1023 })
1024 .instrument(tracing::info_span!("get_devname", CVM_ALLOWED))
1025 .await??;
1026 Ok(devname)
1027 }
1028 .await
1029 .map_err(|err| Error::StorageCannotFindVtl2Device {
1030 device_type: single_device.device_type,
1031 instance_id: controller_instance_id,
1032 sub_device_path,
1033 source: err,
1034 })?;
1035
1036 async {
1042 ctx.until_cancelled(
1043 storage_context
1044 .uevent_listener
1045 .wait_for_devpath(&PathBuf::from_iter([
1046 Path::new("/sys/block"),
1047 devname.as_ref(),
1048 "bdi".as_ref(),
1049 ]))
1050 .instrument(tracing::info_span!("wait_for_bdi", CVM_ALLOWED)),
1051 )
1052 .await??;
1053 Ok(())
1054 }
1055 .await
1056 .map_err(|err| Error::StorageCannotFindVtl2Device {
1057 device_type: single_device.device_type,
1058 instance_id: controller_instance_id,
1059 sub_device_path,
1060 source: err,
1061 })?;
1062
1063 {
1066 let path =
1067 PathBuf::from_iter([Path::new("/sys/block"), devname.as_ref(), "queue".as_ref()]);
1068
1069 fs_err::write(path.join("scheduler"), "none")
1070 .map_err(Error::StorageCannotDisableIoScheduling)?;
1071
1072 fs_err::write(path.join("nomerges"), "2")
1074 .map_err(Error::StorageCannotDisableIoScheduling)?;
1075 }
1076
1077 let disk_path = Path::new("/dev").join(devname);
1078
1079 let file = disk_blockdevice::open_file_for_block(&disk_path, read_only).map_err(|e| {
1080 Error::StorageCannotOpenVtl2Device {
1081 device_type: single_device.device_type,
1082 instance_id: controller_instance_id,
1083 sub_device_path,
1084 path: &disk_path,
1085 source: e,
1086 }
1087 })?;
1088
1089 Ok(Resource::new(OpenBlockDeviceConfig { file }))
1090}
1091
1092fn make_disk_config_inner(
1093 location: u8,
1094 disk_params: &DiskParameters,
1095) -> Result<scsidisk_resources::DiskParameters, Vtl2SettingsErrorInfo> {
1096 let vendor_id = disk_params
1097 .vendor_id
1098 .parse()
1099 .map_err(|source| Error::StorageBadVendorId {
1100 lun: location,
1101 vendor_id: &disk_params.vendor_id,
1102 source,
1103 })?;
1104
1105 let product_id =
1106 disk_params
1107 .product_id
1108 .parse()
1109 .map_err(|source| Error::StorageBadProductId {
1110 lun: location,
1111 product_id: &disk_params.product_id,
1112 source,
1113 })?;
1114
1115 let product_revision_level = disk_params
1116 .product_revision_level
1117 .parse()
1118 .map_err(|source| Error::StorageBadProductRevisionLevel {
1119 lun: location,
1120 product_revision_level: &disk_params.product_revision_level,
1121 source,
1122 })?;
1123
1124 let disk_id = disk_params
1125 .device_id
1126 .parse::<Guid>()
1127 .map_err(|_| Error::StorageBadDeviceId {
1128 lun: location,
1129 device_id: &disk_params.device_id,
1130 })?;
1131
1132 if disk_id == Guid::ZERO {
1133 return Err(Error::StorageBadDeviceId {
1134 lun: location,
1135 device_id: &disk_params.device_id,
1136 }
1137 .into());
1138 }
1139
1140 Ok(scsidisk_resources::DiskParameters {
1141 disk_id: Some(disk_id.into()),
1142 identity: Some(scsidisk_resources::DiskIdentity {
1143 vendor_id,
1144 product_id,
1145 product_revision_level,
1146 model_number: disk_params.model_number.clone().into_bytes(),
1147 }),
1148 serial_number: disk_params.serial_number.clone().into_bytes(),
1149 medium_rotation_rate: Some(disk_params.medium_rotation_rate),
1150 physical_sector_size: disk_params.physical_sector_size,
1151 fua: disk_params.fua,
1152 write_cache: disk_params.write_cache,
1153 scsi_disk_size_in_bytes: disk_params.scsi_disk_size_in_bytes,
1154 odx: disk_params.odx,
1155 unmap: disk_params.unmap,
1156 get_lba_status: true,
1157 max_transfer_length: disk_params.max_transfer_length,
1158 optimal_unmap_sectors: None, })
1160}
1161
1162async fn make_ide_disk_config(
1163 ctx: &mut CancelContext,
1164 storage_context: &StorageContext<'_>,
1165 disk: &underhill_config::IdeDisk,
1166 is_restoring: bool,
1167) -> Result<(IdeDeviceConfig, Option<mesh::Sender<SimpleScsiDvdRequest>>), Vtl2SettingsErrorInfo> {
1168 let disk_type = make_disk_type(
1169 ctx,
1170 storage_context,
1171 &StorageDisk::Ide(disk.clone()),
1172 is_restoring,
1173 )
1174 .await?;
1175 if disk.is_dvd {
1176 let (send, recv) = mesh::channel();
1177 Ok((
1178 IdeDeviceConfig {
1179 path: ide_path_from_config(disk)?,
1180 guest_media: GuestMedia::Dvd(
1181 SimpleScsiDvdHandle {
1182 media: disk_type,
1183 requests: Some(recv),
1184 }
1185 .into_resource(),
1186 ),
1187 },
1188 Some(send),
1189 ))
1190 } else {
1191 Ok((
1192 IdeDeviceConfig {
1193 path: ide_path_from_config(disk)?,
1194 guest_media: GuestMedia::Disk {
1195 disk_type: disk_type.unwrap(),
1196 read_only: false,
1197 disk_parameters: Some(make_disk_config_inner(
1198 disk.location,
1199 &disk.disk_params,
1200 )?),
1201 },
1202 },
1203 None,
1204 ))
1205 }
1206}
1207
1208async fn make_scsi_disk_config(
1209 ctx: &mut CancelContext,
1210 storage_context: &StorageContext<'_>,
1211 disk: &underhill_config::ScsiDisk,
1212 is_restoring: bool,
1213) -> Result<
1214 (
1215 ScsiDeviceAndPath,
1216 Option<mesh::Sender<SimpleScsiDvdRequest>>,
1217 ),
1218 Vtl2SettingsErrorInfo,
1219> {
1220 let path = scsi_path_from_config(disk)?;
1221 let disk_type = make_disk_type(
1222 ctx,
1223 storage_context,
1224 &StorageDisk::Scsi(disk.clone()),
1225 is_restoring,
1226 )
1227 .await?;
1228 let (device, dvd) = if disk.is_dvd {
1229 let (send, recv) = mesh::channel();
1230 (
1231 SimpleScsiDvdHandle {
1232 media: disk_type,
1233 requests: Some(recv),
1234 }
1235 .into_resource(),
1236 Some(send),
1237 )
1238 } else {
1239 (
1240 SimpleScsiDiskHandle {
1241 disk: disk_type.unwrap(),
1242 read_only: false,
1243 parameters: make_disk_config_inner(disk.location, &disk.disk_params)?,
1244 }
1245 .into_resource(),
1246 None,
1247 )
1248 };
1249 Ok((ScsiDeviceAndPath { path, device }, dvd))
1250}
1251
1252async fn make_disk_type(
1253 ctx: &mut CancelContext,
1254 storage_context: &StorageContext<'_>,
1255 disk: &StorageDisk,
1256 is_restoring: bool,
1257) -> Result<Option<Resource<DiskHandleKind>>, Vtl2SettingsErrorInfo> {
1258 let disk_type = match (disk.is_dvd(), disk.physical_devices()) {
1259 (true, PhysicalDevices::EmptyDrive) => None,
1260 (false, PhysicalDevices::EmptyDrive) => unreachable!("schema validated"),
1261 (false, physical_devices) => Some(
1262 make_disk_type_from_physical_devices(
1263 ctx,
1264 storage_context,
1265 physical_devices,
1266 disk.ntfs_guid(),
1267 false,
1268 is_restoring,
1269 )
1270 .await?,
1271 ),
1272 (true, physical_devices) => {
1273 let disk_type = make_disk_type_from_physical_devices(
1274 ctx,
1275 storage_context,
1276 physical_devices,
1277 disk.ntfs_guid(),
1278 true,
1279 is_restoring,
1280 )
1281 .await;
1282 match disk_type {
1283 Ok(disk_type) => Some(disk_type),
1284 Err(err_info) => match err_info.code() {
1285 Vtl2SettingsErrorCode::StorageCannotOpenVtl2Device => {
1286 tracing::warn!(CVM_ALLOWED, ?err_info, "Check if ISO is present on drive");
1287 None
1288 }
1289 _ => Err(err_info)?,
1290 },
1291 }
1292 }
1293 };
1294 Ok(disk_type)
1295}
1296
1297async fn make_nvme_disk_config(
1298 ctx: &mut CancelContext,
1299 storage_context: &StorageContext<'_>,
1300 namespace: &underhill_config::NvmeNamespace,
1301 is_restoring: bool,
1302) -> Result<NamespaceDefinition, Vtl2SettingsErrorInfo> {
1303 let disk_type = make_disk_type_from_physical_devices(
1304 ctx,
1305 storage_context,
1306 &namespace.physical_devices,
1307 None,
1308 false,
1309 is_restoring,
1310 )
1311 .await?;
1312 Ok(NamespaceDefinition {
1313 nsid: namespace.nsid,
1314 disk: disk_type,
1315 read_only: false,
1316 })
1317}
1318
1319#[instrument(skip_all, fields(CVM_ALLOWED))]
1320async fn make_ide_controller_config(
1321 ctx: &mut CancelContext,
1322 storage_context: &StorageContext<'_>,
1323 settings: &Vtl2SettingsDynamic,
1324 is_restoring: bool,
1325) -> Result<Option<UhIdeControllerConfig>, Vtl2SettingsErrorInfo> {
1326 let mut primary_channel_disks = Vec::new();
1327 let mut secondary_channel_disks = Vec::new();
1328 let mut io_queue_depth = None;
1329
1330 let mut dvds = Vec::new();
1331 if let Some(ide_controller) = &settings.ide_controller {
1332 io_queue_depth = ide_controller.io_queue_depth;
1333 for disk in &ide_controller.disks {
1334 let (config, dvd) =
1335 make_ide_disk_config(ctx, storage_context, disk, is_restoring).await?;
1336 if let Some(dvd) = dvd {
1337 dvds.push((config.path, dvd));
1338 }
1339 if disk.channel == 0 {
1340 primary_channel_disks.push(config);
1341 } else {
1342 secondary_channel_disks.push(config);
1343 }
1344 }
1345 }
1346
1347 if primary_channel_disks.is_empty() && secondary_channel_disks.is_empty() {
1348 Ok(None)
1349 } else {
1350 Ok(Some(UhIdeControllerConfig {
1351 config: IdeControllerConfig {
1352 primary_channel_disks,
1353 secondary_channel_disks,
1354 io_queue_depth,
1355 },
1356 dvds,
1357 }))
1358 }
1359}
1360
1361#[instrument(skip_all, fields(CVM_ALLOWED))]
1362async fn make_scsi_controller_config(
1363 ctx: &mut CancelContext,
1364 storage_context: &StorageContext<'_>,
1365 controller: &underhill_config::ScsiController,
1366 scsi_sub_channels: u16,
1367 is_restoring: bool,
1368 default_io_queue_depth: u32,
1369) -> Result<UhScsiControllerConfig, Vtl2SettingsErrorInfo> {
1370 let instance_id = controller.instance_id;
1371 let mut scsi_disks = Vec::new();
1372
1373 let mut dvds = Vec::new();
1374 for disk in &controller.disks {
1375 let (disk_cfg, dvd) =
1376 make_scsi_disk_config(ctx, storage_context, disk, is_restoring).await?;
1377 if let Some(dvd) = dvd {
1378 dvds.push((disk_cfg.path, dvd));
1379 }
1380 scsi_disks.push(disk_cfg);
1381 }
1382
1383 let (send, recv) = mesh::channel();
1384 Ok(UhScsiControllerConfig {
1388 handle: ScsiControllerHandle {
1389 instance_id,
1390 max_sub_channel_count: scsi_sub_channels.min(256),
1391 devices: scsi_disks,
1392 io_queue_depth: Some(controller.io_queue_depth.unwrap_or(default_io_queue_depth)),
1393 requests: Some(recv),
1394 },
1395 request: send,
1396 dvds,
1397 })
1398}
1399
1400#[instrument(skip_all, fields(CVM_ALLOWED))]
1401async fn make_nvme_controller_config(
1402 ctx: &mut CancelContext,
1403 storage_context: &StorageContext<'_>,
1404 controller: &underhill_config::NvmeController,
1405 is_restoring: bool,
1406) -> Result<UhVpciDeviceConfig, Vtl2SettingsErrorInfo> {
1407 let mut namespaces = Vec::new();
1408 for namespace in &controller.namespaces {
1409 namespaces
1410 .push(make_nvme_disk_config(ctx, storage_context, namespace, is_restoring).await?);
1411 }
1412
1413 Ok(UhVpciDeviceConfig {
1414 instance_id: controller.instance_id,
1415 resource: NvmeControllerHandle {
1416 subsystem_id: controller.instance_id,
1417 namespaces,
1418 max_io_queues: 64,
1419 msix_count: 64,
1420 }
1421 .into_resource(),
1422 })
1423}
1424
1425pub async fn create_storage_controllers_from_vtl2_settings(
1426 ctx: &mut CancelContext,
1427 uevent_listener: &UeventListener,
1428 use_nvme_vfio: bool,
1429 settings: &Vtl2SettingsDynamic,
1430 sub_channels: u16,
1431 is_restoring: bool,
1432 default_io_queue_depth: u32,
1433) -> Result<
1434 (
1435 Option<UhIdeControllerConfig>,
1436 Vec<UhScsiControllerConfig>,
1437 Vec<UhVpciDeviceConfig>,
1438 ),
1439 Vtl2SettingsErrorInfo,
1440> {
1441 let storage_context = StorageContext {
1442 uevent_listener,
1443 use_nvme_vfio,
1444 };
1445 let ide_controller =
1446 make_ide_controller_config(ctx, &storage_context, settings, is_restoring).await?;
1447
1448 let mut scsi_controllers = Vec::new();
1449 for controller in &settings.scsi_controllers {
1450 scsi_controllers.push(
1451 make_scsi_controller_config(
1452 ctx,
1453 &storage_context,
1454 controller,
1455 sub_channels,
1456 is_restoring,
1457 default_io_queue_depth,
1458 )
1459 .await?,
1460 );
1461 }
1462
1463 let mut nvme_controllers = Vec::new();
1464 for controller in &settings.nvme_controllers {
1465 nvme_controllers.push(
1466 make_nvme_controller_config(ctx, &storage_context, controller, is_restoring).await?,
1467 );
1468 }
1469
1470 Ok((ide_controller, scsi_controllers, nvme_controllers))
1471}
1472
1473fn nsid_matches(devpath: &Path, nsid: u32) -> anyhow::Result<bool> {
1474 match fs_err::read_to_string(devpath.join("nsid")) {
1475 Ok(this_nsid) => {
1476 let this_nsid: u32 = this_nsid.trim().parse().context("failed to parse nsid")?;
1477 Ok(this_nsid == nsid)
1478 }
1479 Err(err) if err.kind() == std::io::ErrorKind::NotFound => Ok(false),
1482 Err(err) => Err(err.into()),
1483 }
1484}
1485
1486async fn check_block_sysfs_ready(devname: &str) -> bool {
1498 let devname = devname.to_string();
1499 let dev_clone = devname.clone();
1500 match blocking::unblock(move || fs_err::File::open(Path::new("/dev").join(dev_clone))).await {
1501 Ok(_) => true,
1502 Err(err) if err.raw_os_error() == Some(libc::ENXIO) => {
1503 tracing::info!(
1504 CVM_ALLOWED,
1505 devname,
1506 "block device not ready, waiting for uevent"
1507 );
1508 false
1509 }
1510 Err(err) => {
1511 tracing::warn!(
1512 CVM_ALLOWED,
1513 error = &err as &dyn std::error::Error,
1514 devname,
1515 "block device failed to open during scan"
1516 );
1517 true
1520 }
1521 }
1522}
1523
1524async fn get_nvme_namespace_devname(
1526 uevent_listener: &UeventListener,
1527 controller_instance_id: &Guid,
1528 nsid: u32,
1529) -> anyhow::Result<String> {
1530 let nvme_devpath =
1532 nvme_controller_path_from_vmbus_instance_id(uevent_listener, controller_instance_id)
1533 .await?;
1534
1535 let prefix = nvme_devpath
1537 .file_name()
1538 .unwrap()
1539 .to_str()
1540 .unwrap()
1541 .to_owned()
1542 + "n";
1543
1544 let devname = uevent_listener
1546 .wait_for_matching_child(&nvme_devpath, async |path, uevent| {
1547 let name = path.file_name()?.to_str()?;
1548 if !name.starts_with(&prefix) {
1549 return None;
1550 }
1551 match nsid_matches(&path, nsid) {
1552 Ok(true) => {
1553 if uevent || check_block_sysfs_ready(name).await {
1554 Some(Ok(name.to_string()))
1555 } else {
1556 None
1557 }
1558 }
1559 Ok(false) => None,
1560 Err(err) => Some(Err(err)),
1561 }
1562 })
1563 .await??;
1564
1565 Ok(devname)
1566}
1567
1568async fn get_scsi_host_number(
1569 uevent_listener: &UeventListener,
1570 instance_id: &Guid,
1571) -> anyhow::Result<u32> {
1572 let controller_path = PathBuf::from(format!("/sys/bus/vmbus/devices/{instance_id}"));
1574 let host_number = uevent_listener
1575 .wait_for_matching_child(&controller_path, async |name, _uevent| {
1576 name.file_name()?
1577 .to_str()?
1578 .strip_prefix("host")?
1579 .parse()
1580 .ok()
1581 })
1582 .await?;
1583
1584 Ok(host_number)
1585}
1586
1587async fn get_vscsi_devname(
1588 uevent_listener: &UeventListener,
1589 controller_instance_id: &Guid,
1590 lun: u32,
1591) -> anyhow::Result<String> {
1592 let host_number = get_scsi_host_number(uevent_listener, controller_instance_id).await?;
1594
1595 let block_devpath = PathBuf::from(format!(
1597 "/sys/bus/scsi/devices/{host_number}:0:0:{lun}/block"
1598 ));
1599
1600 uevent_listener
1601 .wait_for_matching_child(&block_devpath, async |path, uevent| {
1602 let name = path.file_name()?.to_str()?;
1603 if uevent || check_block_sysfs_ready(name).await {
1604 Some(name.to_string())
1605 } else {
1606 None
1607 }
1608 })
1609 .await
1610 .context("failed to wait for block path")
1611}
1612
1613async fn nvme_controller_path_from_vmbus_instance_id(
1616 uevent_listener: &UeventListener,
1617 instance_id: &Guid,
1618) -> anyhow::Result<PathBuf> {
1619 let (pci_id, mut devpath) = vpci_path(instance_id);
1620 devpath.push("nvme");
1621 let devpath = uevent_listener
1622 .wait_for_matching_child(&devpath, async |path, _uevent| Some(path))
1623 .await?;
1624 wait_for_pci_path(&pci_id).await;
1625 Ok(devpath)
1626}
1627
1628fn vpci_path(instance_id: &Guid) -> (String, PathBuf) {
1629 let pci_id = format!("{:04x}:00:00.0", instance_id.data2);
1632 let pci_bus_id = &pci_id[..7];
1633
1634 let devpath = PathBuf::from(format!(
1638 "/sys/bus/vmbus/devices/{instance_id}/pci{pci_bus_id}/{pci_id}"
1639 ));
1640
1641 (pci_id, devpath)
1642}
1643
1644pub async fn wait_for_pci_path(pci_id: &String) {
1647 let pci_path = PathBuf::from(format!("/sys/bus/pci/devices/{pci_id}"));
1648 loop {
1649 if pci_path.exists() {
1650 return;
1651 }
1652
1653 let mut context = CancelContext::new().with_timeout(Duration::from_millis(10));
1654 let _ = context
1655 .until_cancelled({
1656 async {
1657 futures::pending!();
1658 }
1659 })
1660 .await;
1661 }
1662}
1663
1664pub async fn wait_for_mana(
1665 uevent_listener: &UeventListener,
1666 instance_id: &Guid,
1667) -> anyhow::Result<String> {
1668 let (pci_id, devpath) = vpci_path(instance_id);
1669
1670 uevent_listener.wait_for_devpath(&devpath).await?;
1672 wait_for_pci_path(&pci_id).await;
1673
1674 let vendor = fs_err::read_to_string(devpath.join("vendor"))?;
1676 let device = fs_err::read_to_string(devpath.join("device"))?;
1677 if vendor.trim_end() != "0x1414" {
1678 anyhow::bail!("invalid mana vendor {vendor}");
1679 }
1680 if device.trim_end() != "0x00ba" {
1681 anyhow::bail!("invalid mana device {device}");
1682 }
1683
1684 Ok(pci_id)
1685}
1686
1687pub async fn get_mana_config_from_vtl2_settings(
1688 ctx: &mut CancelContext,
1689 uevent_listener: &UeventListener,
1690 settings: &Vtl2SettingsDynamic,
1691) -> anyhow::Result<Vec<NicConfig>> {
1692 let mut mana = Vec::<NicConfig>::new();
1693 for config in &settings.nic_devices {
1694 let pci_id = ctx
1695 .until_cancelled(wait_for_mana(uevent_listener, &config.instance_id))
1696 .await
1697 .context("cancelled waiting for mana devices")??;
1698
1699 mana.push(NicConfig {
1700 pci_id,
1701 instance_id: config.instance_id,
1702 subordinate_instance_id: config.subordinate_instance_id,
1703 max_sub_channels: config.max_sub_channels,
1704 });
1705 }
1706 Ok(mana)
1707}
1708
1709trait HasInstanceId {
1710 fn instance_id(&self) -> Guid;
1711}
1712
1713impl HasInstanceId for underhill_config::ScsiController {
1714 fn instance_id(&self) -> Guid {
1715 self.instance_id
1716 }
1717}
1718
1719impl HasInstanceId for underhill_config::IdeController {
1720 fn instance_id(&self) -> Guid {
1721 self.instance_id
1722 }
1723}
1724
1725impl HasInstanceId for NicDevice {
1726 fn instance_id(&self) -> Guid {
1727 self.instance_id
1728 }
1729}
1730
1731fn create_device_map_from_settings<T: HasInstanceId>(devices: &Vec<T>) -> HashMap<Guid, &T> {
1732 let mut map = HashMap::new();
1733
1734 for device in devices {
1735 map.insert(device.instance_id(), device);
1736 }
1737
1738 map
1739}
1740
1741fn calculate_device_change_from_map<'a, T>(
1742 old_device_map: &'a HashMap<Guid, &T>,
1743 new_device_map: &'a HashMap<Guid, &T>,
1744) -> (Vec<(&'a T, &'a T)>, Vec<&'a T>, Vec<&'a T>) {
1745 let mut devices_to_check = Vec::new();
1746 let mut devices_to_remove = Vec::new();
1747 let mut devices_to_add = Vec::new();
1748
1749 for (instance_id, old_config) in old_device_map.iter() {
1750 match new_device_map.get(instance_id) {
1751 Some(new_config) => {
1752 devices_to_check.push((*old_config, *new_config));
1754 }
1755 _ => devices_to_remove.push(*old_config),
1757 }
1758 }
1759
1760 for (instance_id, config) in new_device_map.iter() {
1761 if old_device_map.get(instance_id).is_none() {
1762 devices_to_add.push(*config)
1763 }
1764 }
1765
1766 (devices_to_check, devices_to_remove, devices_to_add)
1767}
1768
1769pub struct InitialControllers {
1770 pub ide_controller: Option<IdeControllerConfig>,
1771 pub vmbus_devices: Vec<Resource<VmbusDeviceHandleKind>>,
1772 pub vpci_devices: Vec<UhVpciDeviceConfig>,
1773 pub mana: Vec<NicConfig>,
1774 pub device_interfaces: DeviceInterfaces,
1775}
1776
1777impl InitialControllers {
1778 pub async fn new(
1780 uevent_listener: &UeventListener,
1781 dps: &DevicePlatformSettings,
1782 use_nvme_vfio: bool,
1783 is_restoring: bool,
1784 default_io_queue_depth: u32,
1785 ) -> anyhow::Result<Self> {
1786 const VM_CONFIG_TIME_OUT_IN_SECONDS: u64 = 5;
1787 let mut context =
1788 CancelContext::new().with_timeout(Duration::from_secs(VM_CONFIG_TIME_OUT_IN_SECONDS));
1789
1790 let vtl2_settings = dps.general.vtl2_settings.as_ref();
1791
1792 tracing::info!(CVM_ALLOWED, ?vtl2_settings, "Initial VTL2 settings");
1793
1794 let fixed = vtl2_settings.map_or_else(Default::default, |s| s.fixed.clone());
1795 let dynamic = vtl2_settings.map(|s| &s.dynamic);
1796
1797 let (ide_controller, scsi_controllers, vpci_devices) = if let Some(dynamic) = &dynamic {
1798 create_storage_controllers_from_vtl2_settings(
1799 &mut context,
1800 uevent_listener,
1801 use_nvme_vfio,
1802 dynamic,
1803 fixed.scsi_sub_channels,
1804 is_restoring,
1805 default_io_queue_depth,
1806 )
1807 .await?
1808 } else {
1809 (None, Vec::new(), Vec::new())
1810 };
1811
1812 let mana = if let Some(dynamic) = &dynamic {
1813 get_mana_config_from_vtl2_settings(&mut context, uevent_listener, dynamic).await?
1814 } else {
1815 Vec::new()
1816 };
1817
1818 let mut scsi_dvds = HashMap::new();
1819 let mut scsi_request = HashMap::new();
1820
1821 let ide_controller = ide_controller.map(|c| {
1822 scsi_dvds.extend(
1823 c.dvds
1824 .into_iter()
1825 .map(|(path, send)| (StorageDevicePath::Ide(path), send)),
1826 );
1827 c.config
1828 });
1829
1830 let vmbus_devices = scsi_controllers
1831 .into_iter()
1832 .map(|c| {
1833 scsi_dvds.extend(
1834 c.dvds
1835 .into_iter()
1836 .map(|(path, send)| (StorageDevicePath::Scsi(path), send)),
1837 );
1838 scsi_request.insert(c.handle.instance_id, c.request);
1839 c.handle.into_resource()
1840 })
1841 .collect();
1842
1843 let cfg = InitialControllers {
1844 ide_controller,
1845 vmbus_devices,
1846 mana,
1847 vpci_devices,
1848 device_interfaces: DeviceInterfaces {
1849 scsi_dvds,
1850 scsi_request,
1851 use_nvme_vfio,
1852 },
1853 };
1854
1855 Ok(cfg)
1856 }
1857}