underhill_core/dispatch/
vtl2_settings_worker.rs

1// Copyright (c) Microsoft Corporation.
2// Licensed under the MIT License.
3
4//! Implements VTL2 settings worker
5
6use super::LoadedVm;
7use crate::nvme_manager::NvmeDiskConfig;
8use crate::worker::NicConfig;
9use anyhow::Context;
10use cvm_tracing::CVM_ALLOWED;
11use disk_backend::Disk;
12use disk_backend::resolve::ResolveDiskParameters;
13use disk_backend_resources::AutoFormattedDiskHandle;
14use disk_blockdevice::OpenBlockDeviceConfig;
15use futures::StreamExt;
16use guest_emulation_transport::api::platform_settings::DevicePlatformSettings;
17use guid::Guid;
18use ide_resources::GuestMedia;
19use ide_resources::IdeControllerConfig;
20use ide_resources::IdeDeviceConfig;
21use ide_resources::IdePath;
22use mesh::CancelContext;
23use mesh::rpc::Rpc;
24use mesh::rpc::RpcError;
25use mesh::rpc::RpcSend;
26use nvme_resources::NamespaceDefinition;
27use nvme_resources::NvmeControllerHandle;
28use scsidisk_resources::SimpleScsiDiskHandle;
29use scsidisk_resources::SimpleScsiDvdHandle;
30use scsidisk_resources::SimpleScsiDvdRequest;
31use std::collections::HashMap;
32use std::error::Error as _;
33use std::fmt::Write;
34use std::path::Path;
35use std::path::PathBuf;
36use std::time::Duration;
37use storage_string::InvalidAsciiString;
38use storvsp_resources::ScsiControllerHandle;
39use storvsp_resources::ScsiControllerRequest;
40use storvsp_resources::ScsiDeviceAndPath;
41use thiserror::Error;
42use tracing::Instrument;
43use tracing::instrument;
44use uevent::UeventListener;
45use underhill_config::DiskParameters;
46use underhill_config::NicDevice;
47use underhill_config::PhysicalDevice;
48use underhill_config::PhysicalDevices;
49use underhill_config::StorageDisk;
50use underhill_config::Vtl2Settings;
51use underhill_config::Vtl2SettingsDynamic;
52use underhill_config::Vtl2SettingsErrorCode;
53use underhill_config::Vtl2SettingsErrorInfo;
54use underhill_threadpool::AffinitizedThreadpool;
55use vm_resource::IntoResource;
56use vm_resource::ResolveError;
57use vm_resource::Resource;
58use vm_resource::ResourceResolver;
59use vm_resource::kind::DiskHandleKind;
60use vm_resource::kind::PciDeviceHandleKind;
61use vm_resource::kind::VmbusDeviceHandleKind;
62
63#[derive(Error, Debug)]
64enum Error<'a> {
65    #[error("RPC error")]
66    Rpc(#[source] RpcError),
67    #[error("cannot add/remove storage controllers at runtime")]
68    StorageCannotAddRemoveControllerAtRuntime,
69    #[error("Striping devices don't support runtime change")]
70    StripStorageCannotChangeControllerAtRuntime,
71    #[error("failed to open disk")]
72    StorageCannotOpenDisk(#[source] ResolveError),
73    #[error("could not disable io scheduling")]
74    StorageCannotDisableIoScheduling(#[source] std::io::Error),
75    #[error("failed to open {device_type:?} disk {path} at {instance_id}/{sub_device_path}")]
76    StorageCannotOpenVtl2Device {
77        device_type: underhill_config::DeviceType,
78        instance_id: Guid,
79        sub_device_path: u32,
80        path: &'a PathBuf,
81        #[source]
82        source: std::io::Error,
83    },
84    #[error("failed to find {device_type:?} disk at {instance_id}/{sub_device_path}")]
85    StorageCannotFindVtl2Device {
86        device_type: underhill_config::DeviceType,
87        instance_id: Guid,
88        sub_device_path: u32,
89        #[source]
90        source: anyhow::Error,
91    },
92    #[error("no SCSI controller {0}")]
93    StorageScsiControllerNotFound(Guid),
94    #[error("failed to add disk")]
95    StorageScsiPathInUse(#[source] anyhow::Error),
96    #[error("failed to add disk at lun {0}")]
97    StorageAttachDiskFailed(u8, #[source] anyhow::Error),
98    #[error("failed to remove disk")]
99    StorageScsiPathNotInUse(#[source] anyhow::Error),
100    #[error("failed to remove disk at lun {0}")]
101    StorageRemoveDiskFailed(u8, #[source] anyhow::Error),
102    #[error("failed to change media at lun {0}")]
103    StorageChangeMediaFailed(u8, #[source] anyhow::Error),
104    #[error("failed to modify networking instance {0}")]
105    NetworkingModifyNicFailed(Guid, #[source] anyhow::Error),
106    #[error("failed to add network interface {0}")]
107    NetworkingAddNicFailed(Guid, #[source] anyhow::Error),
108    #[error("failed to remove network interface {0}")]
109    NetworkingRemoveNicFailed(Guid, #[source] anyhow::Error),
110    #[error("failed to parse Vendor ID: LUN = {lun:?}, vendor_id = {vendor_id:?}")]
111    StorageBadVendorId {
112        lun: u8,
113        vendor_id: &'a str,
114        #[source]
115        source: InvalidAsciiString,
116    },
117    #[error("failed to parse Product ID: LUN = {lun:?}, product_id = {product_id:?}")]
118    StorageBadProductId {
119        lun: u8,
120        product_id: &'a str,
121        #[source]
122        source: InvalidAsciiString,
123    },
124    #[error("failed to parse Device ID: LUN = {lun:?}, device_id = {device_id:?}")]
125    StorageBadDeviceId { lun: u8, device_id: &'a str },
126    #[error(
127        "failed to parse Product Revision Level: LUN = {lun:?}, product_revision_level = {product_revision_level:?}"
128    )]
129    StorageBadProductRevisionLevel {
130        lun: u8,
131        product_revision_level: &'a str,
132        #[source]
133        source: InvalidAsciiString,
134    },
135    #[error("cannot modify IDE configuration at runtime")]
136    StorageCannotModifyIdeAtRuntime,
137}
138
139impl Error<'_> {
140    fn code(&self) -> Vtl2SettingsErrorCode {
141        match self {
142            Error::Rpc(_) => Vtl2SettingsErrorCode::InternalFailure,
143            Error::StorageCannotAddRemoveControllerAtRuntime => {
144                Vtl2SettingsErrorCode::StorageCannotAddRemoveControllerAtRuntime
145            }
146            Error::StripStorageCannotChangeControllerAtRuntime => {
147                Vtl2SettingsErrorCode::StripedStorageCannotChangeControllerAtRuntime
148            }
149            Error::StorageCannotOpenDisk(_) => Vtl2SettingsErrorCode::StorageCannotOpenVtl2Device,
150            Error::StorageCannotDisableIoScheduling(_) => {
151                Vtl2SettingsErrorCode::StorageCannotOpenVtl2Device
152            }
153            Error::StorageCannotOpenVtl2Device { .. } => {
154                Vtl2SettingsErrorCode::StorageCannotOpenVtl2Device
155            }
156            Error::StorageCannotFindVtl2Device { .. } => {
157                Vtl2SettingsErrorCode::StorageCannotFindVtl2Device
158            }
159            Error::StorageScsiControllerNotFound(_) => {
160                Vtl2SettingsErrorCode::StorageScsiControllerNotFound
161            }
162            Error::StorageScsiPathInUse(_) => Vtl2SettingsErrorCode::StorageAttachDiskFailed,
163            Error::StorageAttachDiskFailed(..) => Vtl2SettingsErrorCode::StorageAttachDiskFailed,
164            Error::StorageScsiPathNotInUse(_) => Vtl2SettingsErrorCode::StorageRmDiskFailed,
165            Error::StorageRemoveDiskFailed(..) => Vtl2SettingsErrorCode::StorageRmDiskFailed,
166            Error::NetworkingModifyNicFailed(..) => {
167                Vtl2SettingsErrorCode::NetworkingModifyNicFailed
168            }
169            Error::NetworkingAddNicFailed(..) => Vtl2SettingsErrorCode::NetworkingAddNicFailed,
170            Error::NetworkingRemoveNicFailed(..) => {
171                Vtl2SettingsErrorCode::NetworkingRemoveNicFailed
172            }
173            Error::StorageBadVendorId { .. } => Vtl2SettingsErrorCode::StorageInvalidVendorId,
174            Error::StorageBadProductId { .. } => Vtl2SettingsErrorCode::StorageInvalidProductId,
175            Error::StorageBadProductRevisionLevel { .. } => {
176                Vtl2SettingsErrorCode::StorageInvalidProductRevisionLevel
177            }
178            Error::StorageCannotModifyIdeAtRuntime => {
179                Vtl2SettingsErrorCode::StorageCannotModifyIdeAtRuntime
180            }
181            Error::StorageBadDeviceId { .. } => Vtl2SettingsErrorCode::StorageInvalidDeviceId,
182            Error::StorageChangeMediaFailed { .. } => {
183                Vtl2SettingsErrorCode::StorageChangeMediaFailed
184            }
185        }
186    }
187}
188
189impl From<Error<'_>> for Vtl2SettingsErrorInfo {
190    #[track_caller]
191    fn from(e: Error<'_>) -> Vtl2SettingsErrorInfo {
192        // Format the message manually to get the full error string (including
193        // error sources).
194        let mut message = e.to_string();
195        let mut source = e.source();
196        while let Some(inner) = source {
197            write!(&mut message, ": {}", inner).unwrap();
198            source = inner.source();
199        }
200
201        Vtl2SettingsErrorInfo::new(e.code(), message)
202    }
203}
204
205pub enum Vtl2ConfigNicRpc {
206    Modify(Rpc<(Guid, Option<Guid>), anyhow::Result<()>>),
207    Add(Rpc<(Guid, Option<Guid>, Option<u16>), anyhow::Result<()>>),
208    Remove(Rpc<Guid, anyhow::Result<()>>),
209}
210
211pub enum Vtl2ConfigAcquireResource {
212    AddDisk(Guid, underhill_config::ScsiDisk),
213    RmDisk(Guid, underhill_config::ScsiDisk),
214    ChangeMedia(Guid, StorageDisk),
215    ModifyNic((Guid, Option<Guid>)),
216    AddNic((Guid, Option<Guid>, Option<u16>)),
217    RemoveNic(Guid),
218}
219
220pub enum Vtl2ConfigCommit {
221    AddDisk(
222        Guid,
223        ScsiDeviceAndPath,
224        Option<mesh::Sender<SimpleScsiDvdRequest>>,
225    ),
226    RmDisk(Guid, storvsp_resources::ScsiPath),
227    ChangeMedia(Guid, StorageDevicePath, Option<Resource<DiskHandleKind>>),
228    ModifyNic((Guid, Option<Guid>)),
229    AddNic((Guid, Option<Guid>, Option<u16>)),
230    RemoveNic(Guid),
231}
232
233/// VTL2 settings worker
234pub struct Vtl2SettingsWorker {
235    old_settings: Vtl2SettingsDynamic,
236    device_config_send: mesh::Sender<Vtl2ConfigNicRpc>,
237    get_client: guest_emulation_transport::GuestEmulationTransportClient,
238    interfaces: DeviceInterfaces,
239}
240
241pub struct DeviceInterfaces {
242    scsi_dvds: HashMap<StorageDevicePath, mesh::Sender<SimpleScsiDvdRequest>>,
243    scsi_request: HashMap<Guid, mesh::Sender<ScsiControllerRequest>>,
244    use_nvme_vfio: bool,
245}
246
247impl Vtl2SettingsWorker {
248    pub fn new(
249        initial_settings: Vtl2SettingsDynamic,
250        device_config_send: mesh::Sender<Vtl2ConfigNicRpc>,
251        get_client: guest_emulation_transport::GuestEmulationTransportClient,
252        interfaces: DeviceInterfaces,
253    ) -> Vtl2SettingsWorker {
254        Vtl2SettingsWorker {
255            old_settings: initial_settings,
256            device_config_send,
257            get_client,
258            interfaces,
259        }
260    }
261
262    pub async fn run(&mut self, uevent_listener: &UeventListener) {
263        let mut settings_recv = self.get_client.take_vtl2_settings_recv().await.unwrap();
264
265        while let Some(req) = settings_recv.next().await {
266            req.0
267                .handle(async |buf| {
268                    self.handle_modify_vtl2_settings(uevent_listener, &{ buf })
269                        .await
270                })
271                .await
272        }
273    }
274
275    async fn handle_modify_vtl2_settings(
276        &mut self,
277        uevent_listener: &UeventListener,
278        buf: &[u8],
279    ) -> Result<(), Vec<Vtl2SettingsErrorInfo>> {
280        const MODIFY_VTL2_SETTINGS_TIMEOUT_IN_SECONDS: u64 = 5;
281        let mut context = CancelContext::new()
282            .with_timeout(Duration::from_secs(MODIFY_VTL2_SETTINGS_TIMEOUT_IN_SECONDS));
283
284        let old_settings = Vtl2Settings {
285            fixed: Default::default(),
286            dynamic: self.old_settings.clone(),
287        };
288        let vtl2_settings =
289            Vtl2Settings::read_from(buf, old_settings).map_err(|err| match err {
290                underhill_config::schema::ParseError::Json(err) => {
291                    vec![Vtl2SettingsErrorInfo::new(
292                        Vtl2SettingsErrorCode::JsonFormatError,
293                        err.to_string(),
294                    )]
295                }
296                underhill_config::schema::ParseError::Protobuf(err) => {
297                    vec![Vtl2SettingsErrorInfo::new(
298                        Vtl2SettingsErrorCode::ProtobufFormatError,
299                        err.to_string(),
300                    )]
301                }
302                underhill_config::schema::ParseError::Validation(err) => err.errors,
303            })?;
304
305        let new_settings = vtl2_settings.dynamic;
306
307        tracing::info!(CVM_ALLOWED, ?new_settings, "Received VTL2 settings");
308
309        let mut todos: Vec<Vtl2ConfigAcquireResource> = Vec::new();
310
311        let mut errors = Vec::new();
312
313        modify_storage_configuration(&self.old_settings, &new_settings, &mut todos, &mut errors);
314        if let Err(err) =
315            modify_network_configuration(&self.old_settings, &new_settings, &mut todos)
316        {
317            errors.push(err);
318        }
319
320        if !errors.is_empty() {
321            return Err(errors);
322        }
323
324        if errors.is_empty() {
325            let mut to_commits: Vec<Vtl2ConfigCommit> = Vec::new();
326            match self
327                .acquire_configuration_resources(
328                    &mut context,
329                    uevent_listener,
330                    todos,
331                    &mut to_commits,
332                )
333                .await
334            {
335                Err(e) => {
336                    tracing::error!(
337                        CVM_ALLOWED,
338                        ?e,
339                        "Error acquiring VTL2 configuration resources"
340                    );
341                    errors.push(e);
342                }
343                Ok(()) => {
344                    // NOTHING BEYOND CAN FAIL
345                    // We assume recover action for commit is to re-create the VM
346                    if let Err(e) = self.commit_configuration_changes(to_commits).await {
347                        tracing::error!(CVM_ALLOWED, ?e, "Error commit VTL2 configuration changes");
348                        errors.push(e);
349                    }
350                }
351            }
352        }
353
354        if !errors.is_empty() {
355            return Err(errors);
356        }
357
358        tracing::info!(CVM_ALLOWED, "VTL2 settings modified");
359        self.old_settings = new_settings;
360        Ok(())
361    }
362
363    async fn acquire_configuration_resources(
364        &mut self,
365        ctx: &mut CancelContext,
366        uevent_listener: &UeventListener,
367        todos: Vec<Vtl2ConfigAcquireResource>,
368        to_commits: &mut Vec<Vtl2ConfigCommit>,
369    ) -> Result<(), Vtl2SettingsErrorInfo> {
370        for todo in todos {
371            match todo {
372                Vtl2ConfigAcquireResource::AddDisk(guid, disk) => {
373                    let (disk_cfg, dvd) = make_scsi_disk_config(
374                        ctx,
375                        &StorageContext {
376                            uevent_listener,
377                            use_nvme_vfio: self.interfaces.use_nvme_vfio,
378                        },
379                        &disk,
380                        false,
381                    )
382                    .await?;
383                    to_commits.push(Vtl2ConfigCommit::AddDisk(guid, disk_cfg, dvd));
384                }
385                Vtl2ConfigAcquireResource::RmDisk(guid, disk) => {
386                    let scsi_path = scsi_path_from_config(&disk)?;
387                    to_commits.push(Vtl2ConfigCommit::RmDisk(guid, scsi_path));
388                }
389                Vtl2ConfigAcquireResource::ChangeMedia(guid, disk) => {
390                    let path = storage_path_from_config(&disk)?;
391                    let disk_type = make_disk_type(
392                        ctx,
393                        &StorageContext {
394                            uevent_listener,
395                            use_nvme_vfio: self.interfaces.use_nvme_vfio,
396                        },
397                        &disk,
398                        false,
399                    )
400                    .await?;
401                    to_commits.push(Vtl2ConfigCommit::ChangeMedia(guid, path, disk_type));
402                }
403                Vtl2ConfigAcquireResource::ModifyNic(nic_settings) => {
404                    to_commits.push(Vtl2ConfigCommit::ModifyNic(nic_settings));
405                }
406                Vtl2ConfigAcquireResource::AddNic(nic_settings) => {
407                    to_commits.push(Vtl2ConfigCommit::AddNic(nic_settings));
408                }
409                Vtl2ConfigAcquireResource::RemoveNic(instance_id) => {
410                    to_commits.push(Vtl2ConfigCommit::RemoveNic(instance_id));
411                }
412            }
413        }
414        Ok(())
415    }
416
417    async fn commit_configuration_changes(
418        &mut self,
419        to_commits: Vec<Vtl2ConfigCommit>,
420    ) -> Result<(), Vtl2SettingsErrorInfo> {
421        for commit in to_commits {
422            match commit {
423                Vtl2ConfigCommit::AddDisk(controller_id, disk_cfg, dvd) => {
424                    let scsi_path = disk_cfg.path;
425                    self.interfaces
426                        .scsi_request
427                        .get(&controller_id)
428                        .ok_or(Error::StorageScsiControllerNotFound(controller_id))?
429                        .call_failable(ScsiControllerRequest::AddDevice, disk_cfg)
430                        .await
431                        .map_err(|err| {
432                            Error::StorageAttachDiskFailed(
433                                scsi_path.lun,
434                                Error::StorageScsiPathInUse(err.into()).into(),
435                            )
436                        })?;
437
438                    if let Some(dvd) = dvd {
439                        assert!(
440                            self.interfaces
441                                .scsi_dvds
442                                .insert(StorageDevicePath::Scsi(scsi_path), dvd)
443                                .is_none()
444                        );
445                    }
446                }
447                Vtl2ConfigCommit::RmDisk(controller_id, scsi_path) => {
448                    self.interfaces
449                        .scsi_request
450                        .get(&controller_id)
451                        .ok_or(Error::StorageScsiControllerNotFound(controller_id))?
452                        .call_failable(ScsiControllerRequest::RemoveDevice, scsi_path)
453                        .await
454                        .map_err(|err| {
455                            Error::StorageRemoveDiskFailed(
456                                scsi_path.lun,
457                                Error::StorageScsiPathNotInUse(err.into()).into(),
458                            )
459                        })?;
460
461                    let _ = self
462                        .interfaces
463                        .scsi_dvds
464                        .remove(&StorageDevicePath::Scsi(scsi_path));
465                }
466                Vtl2ConfigCommit::ChangeMedia(controller_id, path, disk_cfg) => {
467                    // TODO: Improve error handling to work with both storage types (IDE/SCSI)
468                    let lun = if let StorageDevicePath::Scsi(scsi_path) = path {
469                        scsi_path.lun
470                    } else {
471                        0
472                    };
473
474                    async {
475                        let target = self
476                            .interfaces
477                            .scsi_dvds
478                            .get(&path)
479                            .ok_or(Error::StorageScsiControllerNotFound(controller_id))?;
480
481                        target
482                            .call_failable(SimpleScsiDvdRequest::ChangeMedia, disk_cfg)
483                            .await?;
484
485                        anyhow::Ok(())
486                    }
487                    .await
488                    .map_err(|e| Error::StorageChangeMediaFailed(lun, e))?;
489                }
490                Vtl2ConfigCommit::ModifyNic(nic_settings) => {
491                    let instance_id = nic_settings.0;
492                    self.device_config_send
493                        .call(Vtl2ConfigNicRpc::Modify, nic_settings)
494                        .await
495                        .map_err(Error::Rpc)?
496                        .map_err(|e| Error::NetworkingModifyNicFailed(instance_id, e))?;
497                }
498                Vtl2ConfigCommit::AddNic(nic_settings) => {
499                    let instance_id = nic_settings.0;
500                    self.device_config_send
501                        .call(Vtl2ConfigNicRpc::Add, nic_settings)
502                        .await
503                        .map_err(Error::Rpc)?
504                        .map_err(|e| Error::NetworkingAddNicFailed(instance_id, e))?;
505                }
506                Vtl2ConfigCommit::RemoveNic(instance_id) => {
507                    self.device_config_send
508                        .call(Vtl2ConfigNicRpc::Remove, instance_id)
509                        .await
510                        .map_err(Error::Rpc)?
511                        .map_err(|e| Error::NetworkingRemoveNicFailed(instance_id, e))?;
512                }
513            }
514        }
515
516        Ok(())
517    }
518}
519
520pub(crate) async fn handle_vtl2_config_rpc(
521    message: Vtl2ConfigNicRpc,
522    vm: &mut LoadedVm,
523    threadpool: &AffinitizedThreadpool,
524) {
525    match message {
526        Vtl2ConfigNicRpc::Modify(rpc) => {
527            rpc.handle(async |nic_settings| {
528                let modify_settings = vm.network_settings.as_mut().map(|settings| {
529                    settings.modify_network_settings(nic_settings.0, nic_settings.1)
530                });
531                modify_settings
532                    .context("network modifications not supported for this VM")?
533                    .await
534            })
535            .await
536        }
537        Vtl2ConfigNicRpc::Add(rpc) => {
538            rpc.handle(async |nic_settings| {
539                vm.add_vf_manager(threadpool, nic_settings.0, nic_settings.1, nic_settings.2)
540                    .await
541            })
542            .await
543        }
544        Vtl2ConfigNicRpc::Remove(rpc) => {
545            rpc.handle(async |instance_id| vm.remove_vf_manager(instance_id).await)
546                .await
547        }
548    }
549}
550
551pub async fn disk_from_disk_type(
552    disk_type: Resource<DiskHandleKind>,
553    read_only: bool,
554    resolver: &ResourceResolver,
555) -> Result<Disk, Vtl2SettingsErrorInfo> {
556    let disk = resolver
557        .resolve(
558            disk_type,
559            ResolveDiskParameters {
560                read_only,
561                _async_trait_workaround: &(),
562            },
563        )
564        .await
565        .map_err(Error::StorageCannotOpenDisk)?;
566    Ok(disk.0)
567}
568
569fn modify_storage_configuration(
570    old_settings: &Vtl2SettingsDynamic,
571    new_settings: &Vtl2SettingsDynamic,
572    todos: &mut Vec<Vtl2ConfigAcquireResource>,
573    errors: &mut Vec<Vtl2SettingsErrorInfo>,
574) {
575    if let Err(e) = modify_ide_configuration(old_settings, new_settings, todos) {
576        errors.push(e);
577    }
578    if let Err(e) = modify_scsi_configuration(old_settings, new_settings, todos) {
579        errors.push(e);
580    }
581}
582
583fn modify_ide_configuration(
584    old_settings: &Vtl2SettingsDynamic,
585    new_settings: &Vtl2SettingsDynamic,
586    todos: &mut Vec<Vtl2ConfigAcquireResource>,
587) -> Result<(), Vtl2SettingsErrorInfo> {
588    if old_settings.ide_controller != new_settings.ide_controller {
589        if let (Some(old_ide), Some(new_ide)) =
590            (&old_settings.ide_controller, &new_settings.ide_controller)
591        {
592            if old_ide.instance_id == new_ide.instance_id {
593                let instance_id = old_ide.instance_id;
594                if old_ide.disks.len() != new_ide.disks.len() {
595                    return Err(Error::StorageCannotModifyIdeAtRuntime.into());
596                }
597                for (old_disk, new_disk) in old_ide.disks.iter().zip(new_ide.disks.iter()) {
598                    if old_disk.is_dvd
599                        && new_disk.is_dvd
600                        && old_disk.channel == new_disk.channel
601                        && old_disk.location == new_disk.location
602                        && old_disk.disk_params == new_disk.disk_params
603                        && old_disk.physical_devices != new_disk.physical_devices
604                    {
605                        match (
606                            old_disk.physical_devices.is_empty(),
607                            new_disk.physical_devices.is_empty(),
608                        ) {
609                            (true, true) | (false, false) => {
610                                return Err(Error::StorageCannotModifyIdeAtRuntime.into());
611                            }
612                            (true, false) | (false, true) => {
613                                // (true, false) => eject
614                                // (false, true) => insert
615
616                                todos.push(Vtl2ConfigAcquireResource::ChangeMedia(
617                                    instance_id,
618                                    StorageDisk::Ide(new_disk.clone()),
619                                ));
620                                return Ok(());
621                            }
622                        }
623                    }
624                }
625            }
626        }
627
628        return Err(Error::StorageCannotModifyIdeAtRuntime.into());
629    }
630
631    Ok(())
632}
633
634fn modify_scsi_configuration(
635    old_settings: &Vtl2SettingsDynamic,
636    new_settings: &Vtl2SettingsDynamic,
637    todos: &mut Vec<Vtl2ConfigAcquireResource>,
638) -> Result<(), Vtl2SettingsErrorInfo> {
639    let old_controller_map = create_device_map_from_settings(&old_settings.scsi_controllers);
640    let new_controller_map = create_device_map_from_settings(&new_settings.scsi_controllers);
641
642    let (_, controllers_to_remove, controllers_to_add) =
643        calculate_device_change_from_map(&old_controller_map, &new_controller_map);
644
645    if !controllers_to_add.is_empty() || !controllers_to_remove.is_empty() {
646        return Err(Error::StorageCannotAddRemoveControllerAtRuntime.into());
647    }
648
649    modify_scsi_disks_configuration(&old_controller_map, &new_controller_map, todos)?;
650
651    Ok(())
652}
653
654fn modify_scsi_disks_configuration(
655    old_controller_map: &HashMap<Guid, &underhill_config::ScsiController>,
656    new_controller_map: &HashMap<Guid, &underhill_config::ScsiController>,
657    todos: &mut Vec<Vtl2ConfigAcquireResource>,
658) -> Result<(), Vtl2SettingsErrorInfo> {
659    let (modify_disks, mut remove_disks, mut add_disks) =
660        calculate_scsi_disks_change(old_controller_map, new_controller_map)?;
661    let mut change_media_disks = Vec::new();
662
663    for (old_config, new_config, controller_id) in modify_disks {
664        if (old_config.physical_devices.is_striping()) || new_config.physical_devices.is_striping()
665        {
666            return Err(Error::StripStorageCannotChangeControllerAtRuntime.into());
667        }
668
669        match (
670            old_config.physical_devices.is_empty(),
671            new_config.physical_devices.is_empty(),
672        ) {
673            (true, true) | (false, false) => {
674                remove_disks.push((old_config, controller_id));
675                add_disks.push((new_config, controller_id));
676            }
677            (true, false) | (false, true) => {
678                // (true, false) => eject
679                // (false, true) => insert
680
681                if old_config.is_dvd && new_config.is_dvd {
682                    change_media_disks.push((new_config, controller_id));
683                } else {
684                    remove_disks.push((old_config, controller_id));
685                    add_disks.push((new_config, controller_id));
686                }
687            }
688        }
689    }
690
691    // Remove devices
692    for (disk, controller_id) in remove_disks {
693        if disk.physical_devices.is_striping() {
694            return Err(Error::StripStorageCannotChangeControllerAtRuntime.into());
695        }
696        todos.push(Vtl2ConfigAcquireResource::RmDisk(
697            controller_id,
698            disk.clone(),
699        ));
700    }
701
702    // Add devices
703    for (disk, controller_id) in add_disks {
704        if disk.physical_devices.is_striping() {
705            return Err(Error::StripStorageCannotChangeControllerAtRuntime.into());
706        }
707        todos.push(Vtl2ConfigAcquireResource::AddDisk(
708            controller_id,
709            disk.clone(),
710        ));
711    }
712
713    // Change multimedia disks (eject or insert)
714    for (disk, controller_id) in change_media_disks {
715        todos.push(Vtl2ConfigAcquireResource::ChangeMedia(
716            controller_id,
717            StorageDisk::Scsi(disk.clone()),
718        ))
719    }
720
721    Ok(())
722}
723
724fn calculate_scsi_disks_change<'a>(
725    old_controller_map: &'a HashMap<Guid, &underhill_config::ScsiController>,
726    new_controller_map: &'a HashMap<Guid, &underhill_config::ScsiController>,
727) -> Result<
728    (
729        Vec<(
730            &'a underhill_config::ScsiDisk,
731            &'a underhill_config::ScsiDisk,
732            Guid,
733        )>,
734        Vec<(&'a underhill_config::ScsiDisk, Guid)>,
735        Vec<(&'a underhill_config::ScsiDisk, Guid)>,
736    ),
737    Vtl2SettingsErrorInfo,
738> {
739    let mut disks_to_modify = Vec::<(
740        &underhill_config::ScsiDisk,
741        &underhill_config::ScsiDisk,
742        Guid,
743    )>::new();
744    let mut disks_to_remove = Vec::<(&underhill_config::ScsiDisk, Guid)>::new();
745    let mut disks_to_add = Vec::<(&underhill_config::ScsiDisk, Guid)>::new();
746
747    for (instance_id, new_controller) in new_controller_map {
748        if let Some(old_controller) = old_controller_map.get(instance_id) {
749            calculate_disks_change_per_scsi_controller(
750                old_controller,
751                new_controller,
752                &mut disks_to_modify,
753                &mut disks_to_remove,
754                &mut disks_to_add,
755            )?;
756        }
757    }
758
759    Ok((disks_to_modify, disks_to_remove, disks_to_add))
760}
761
762fn calculate_disks_change_per_scsi_controller<'a>(
763    old_controller: &'a underhill_config::ScsiController,
764    new_controller: &'a underhill_config::ScsiController,
765    disks_to_modify: &mut Vec<(
766        &'a underhill_config::ScsiDisk,
767        &'a underhill_config::ScsiDisk,
768        Guid,
769    )>,
770    disks_to_remove: &mut Vec<(&'a underhill_config::ScsiDisk, Guid)>,
771    disks_to_add: &mut Vec<(&'a underhill_config::ScsiDisk, Guid)>,
772) -> Result<(), Vtl2SettingsErrorInfo> {
773    let old_slots = get_scsi_controller_slots(old_controller)?;
774    let new_slots = get_scsi_controller_slots(new_controller)?;
775
776    for i in 0..underhill_config::SCSI_LUN_NUM {
777        if old_slots[i] != new_slots[i] {
778            match (old_slots[i], new_slots[i]) {
779                (Some(old_config), Some(new_config)) => {
780                    disks_to_modify.push((old_config, new_config, old_controller.instance_id))
781                }
782                (Some(old_config), None) => {
783                    disks_to_remove.push((old_config, old_controller.instance_id));
784                }
785                (None, Some(new_config)) => {
786                    disks_to_add.push((new_config, new_controller.instance_id));
787                }
788                (None, None) => continue,
789            }
790        }
791    }
792
793    Ok(())
794}
795
796fn get_scsi_controller_slots(
797    controller: &underhill_config::ScsiController,
798) -> Result<
799    [Option<&underhill_config::ScsiDisk>; underhill_config::SCSI_LUN_NUM],
800    Vtl2SettingsErrorInfo,
801> {
802    let mut slots: [Option<&underhill_config::ScsiDisk>; underhill_config::SCSI_LUN_NUM] =
803        [None; underhill_config::SCSI_LUN_NUM];
804
805    for disk in &controller.disks {
806        slots[disk.location as usize] = Some(disk);
807    }
808
809    Ok(slots)
810}
811
812fn storage_path_from_config(
813    disk: &StorageDisk,
814) -> Result<StorageDevicePath, Vtl2SettingsErrorInfo> {
815    match disk {
816        StorageDisk::Ide(disk) => Ok(StorageDevicePath::Ide(ide_path_from_config(disk)?)),
817        StorageDisk::Scsi(disk) => Ok(StorageDevicePath::Scsi(scsi_path_from_config(disk)?)),
818    }
819}
820
821fn ide_path_from_config(
822    disk: &underhill_config::IdeDisk,
823) -> Result<IdePath, Vtl2SettingsErrorInfo> {
824    Ok(IdePath {
825        channel: disk.channel,
826        drive: disk.location,
827    })
828}
829
830fn scsi_path_from_config(
831    disk: &underhill_config::ScsiDisk,
832) -> Result<storvsp_resources::ScsiPath, Vtl2SettingsErrorInfo> {
833    Ok(storvsp_resources::ScsiPath {
834        path: 0,
835        target: 0,
836        lun: disk.location,
837    })
838}
839
840fn modify_network_configuration(
841    old_settings: &Vtl2SettingsDynamic,
842    new_settings: &Vtl2SettingsDynamic,
843    todos: &mut Vec<Vtl2ConfigAcquireResource>,
844) -> Result<(), Vtl2SettingsErrorInfo> {
845    let mut old_settings_map = HashMap::new();
846    for nic_settings in &old_settings.nic_devices {
847        old_settings_map.insert(
848            nic_settings.instance_id,
849            nic_settings.subordinate_instance_id,
850        );
851    }
852    for nic_settings in &new_settings.nic_devices {
853        if let Some(existing_subordinate_id) = old_settings_map.remove(&nic_settings.instance_id) {
854            if existing_subordinate_id.is_some() != nic_settings.subordinate_instance_id.is_some() {
855                todos.push(Vtl2ConfigAcquireResource::ModifyNic((
856                    nic_settings.instance_id,
857                    nic_settings.subordinate_instance_id,
858                )));
859            }
860        } else {
861            todos.push(Vtl2ConfigAcquireResource::AddNic((
862                nic_settings.instance_id,
863                nic_settings.subordinate_instance_id,
864                nic_settings.max_sub_channels,
865            )));
866        }
867    }
868    if !old_settings_map.is_empty() {
869        for nic_settings in old_settings_map {
870            todos.push(Vtl2ConfigAcquireResource::RemoveNic(nic_settings.0));
871        }
872    }
873    Ok(())
874}
875
876pub struct UhIdeControllerConfig {
877    pub config: IdeControllerConfig,
878    pub dvds: Vec<(IdePath, mesh::Sender<SimpleScsiDvdRequest>)>,
879}
880
881pub struct UhScsiControllerConfig {
882    pub handle: ScsiControllerHandle,
883    pub request: mesh::Sender<ScsiControllerRequest>,
884    pub dvds: Vec<(
885        storvsp_resources::ScsiPath,
886        mesh::Sender<SimpleScsiDvdRequest>,
887    )>,
888}
889
890#[cfg_attr(not(feature = "vpci"), expect(dead_code))]
891pub struct UhVpciDeviceConfig {
892    pub instance_id: Guid,
893    pub resource: Resource<PciDeviceHandleKind>,
894}
895
896#[derive(Debug, Eq, PartialEq, Hash)]
897pub enum StorageDevicePath {
898    Ide(IdePath),
899    Scsi(storvsp_resources::ScsiPath),
900}
901
902async fn make_disk_type_from_physical_devices(
903    ctx: &mut CancelContext,
904    storage_context: &StorageContext<'_>,
905    physical_devices: &PhysicalDevices,
906    ntfs_guid: Option<Guid>,
907    read_only: bool,
908    is_restoring: bool,
909) -> Result<Resource<DiskHandleKind>, Vtl2SettingsErrorInfo> {
910    let disk_type = match *physical_devices {
911        PhysicalDevices::Single { ref device } => {
912            make_disk_type_from_physical_device(ctx, storage_context, device, read_only).await?
913        }
914        PhysicalDevices::Striped {
915            ref devices,
916            chunk_size_in_kb,
917        } => {
918            let mut physical_disk_configs = Vec::new();
919            for single_device in devices {
920                let disk = make_disk_type_from_physical_device(
921                    ctx,
922                    storage_context,
923                    single_device,
924                    read_only,
925                )
926                .await?;
927
928                physical_disk_configs.push(disk);
929            }
930            Resource::new(disk_backend_resources::StripedDiskHandle {
931                devices: physical_disk_configs,
932                chunk_size_in_bytes: Some(chunk_size_in_kb * 1024),
933                logic_sector_count: None,
934            })
935        }
936        PhysicalDevices::EmptyDrive => unreachable!("handled in calling function"),
937    };
938
939    if let Some(ntfs_guid) = ntfs_guid {
940        if is_restoring {
941            tracing::debug!(disk_guid = ?ntfs_guid, "restoring - disk not candidate for auto format");
942        } else {
943            // DEVNOTE: open-source OpenHCL does not currently have a resolver
944            // for `AutoFormattedDiskHandle`.
945            return Ok(Resource::new(AutoFormattedDiskHandle {
946                disk: disk_type,
947                guid: ntfs_guid.into(),
948            }));
949        }
950    }
951
952    Ok(disk_type)
953}
954
955struct StorageContext<'a> {
956    uevent_listener: &'a UeventListener,
957    use_nvme_vfio: bool,
958}
959
960#[instrument(skip_all, fields(CVM_ALLOWED))]
961async fn make_disk_type_from_physical_device(
962    ctx: &mut CancelContext,
963    storage_context: &StorageContext<'_>,
964    single_device: &PhysicalDevice,
965    read_only: bool,
966) -> Result<Resource<DiskHandleKind>, Vtl2SettingsErrorInfo> {
967    let controller_instance_id = single_device.vmbus_instance_id;
968    let sub_device_path = single_device.sub_device_path;
969
970    // Special case for NVMe when using VFIO.
971    if storage_context.use_nvme_vfio
972        && matches!(
973            single_device.device_type,
974            underhill_config::DeviceType::NVMe
975        )
976    {
977        // Wait for the NVMe controller to arrive.
978        let (pci_id, devpath) = vpci_path(&controller_instance_id);
979        async {
980            ctx.until_cancelled(storage_context.uevent_listener.wait_for_devpath(&devpath))
981                .await??;
982            ctx.until_cancelled(wait_for_pci_path(&pci_id)).await?;
983            Ok(())
984        }
985        .await
986        .map_err(|err| Error::StorageCannotFindVtl2Device {
987            device_type: single_device.device_type,
988            instance_id: controller_instance_id,
989            sub_device_path,
990            source: err,
991        })?;
992
993        // We can't validate yet that this namespace actually exists. That will
994        // be checked later.
995        return Ok(Resource::new(NvmeDiskConfig {
996            pci_id,
997            nsid: sub_device_path,
998        }));
999    }
1000
1001    // Wait for the device to arrive.
1002    let devname = async {
1003        let devname = ctx
1004            .until_cancelled(async {
1005                match single_device.device_type {
1006                    underhill_config::DeviceType::NVMe => {
1007                        get_nvme_namespace_devname(
1008                            storage_context.uevent_listener,
1009                            &controller_instance_id,
1010                            sub_device_path,
1011                        )
1012                        .await
1013                    }
1014                    underhill_config::DeviceType::VScsi => {
1015                        get_vscsi_devname(
1016                            storage_context.uevent_listener,
1017                            &controller_instance_id,
1018                            sub_device_path,
1019                        )
1020                        .await
1021                    }
1022                }
1023            })
1024            .instrument(tracing::info_span!("get_devname", CVM_ALLOWED))
1025            .await??;
1026        Ok(devname)
1027    }
1028    .await
1029    .map_err(|err| Error::StorageCannotFindVtl2Device {
1030        device_type: single_device.device_type,
1031        instance_id: controller_instance_id,
1032        sub_device_path,
1033        source: err,
1034    })?;
1035
1036    // Wait for bdi, which is the last thing before the block device actually
1037    // gets enabled. Per the above, the block device should already be ready (we
1038    // successfully opened it, or we got the uevent that indicates the disk is
1039    // ready), but it's possible that some times of devices are openable even
1040    // when the block device infrastructure is not fully ready.
1041    async {
1042        ctx.until_cancelled(
1043            storage_context
1044                .uevent_listener
1045                .wait_for_devpath(&PathBuf::from_iter([
1046                    Path::new("/sys/block"),
1047                    devname.as_ref(),
1048                    "bdi".as_ref(),
1049                ]))
1050                .instrument(tracing::info_span!("wait_for_bdi", CVM_ALLOWED)),
1051        )
1052        .await??;
1053        Ok(())
1054    }
1055    .await
1056    .map_err(|err| Error::StorageCannotFindVtl2Device {
1057        device_type: single_device.device_type,
1058        instance_id: controller_instance_id,
1059        sub_device_path,
1060        source: err,
1061    })?;
1062
1063    // Disable any IO scheduler for the device, and disable blk layer merging,
1064    // so that Underhill is a straight passhthrough.
1065    {
1066        let path =
1067            PathBuf::from_iter([Path::new("/sys/block"), devname.as_ref(), "queue".as_ref()]);
1068
1069        fs_err::write(path.join("scheduler"), "none")
1070            .map_err(Error::StorageCannotDisableIoScheduling)?;
1071
1072        // 2 means disable all merges.
1073        fs_err::write(path.join("nomerges"), "2")
1074            .map_err(Error::StorageCannotDisableIoScheduling)?;
1075    }
1076
1077    let disk_path = Path::new("/dev").join(devname);
1078
1079    let file = disk_blockdevice::open_file_for_block(&disk_path, read_only).map_err(|e| {
1080        Error::StorageCannotOpenVtl2Device {
1081            device_type: single_device.device_type,
1082            instance_id: controller_instance_id,
1083            sub_device_path,
1084            path: &disk_path,
1085            source: e,
1086        }
1087    })?;
1088
1089    Ok(Resource::new(OpenBlockDeviceConfig { file }))
1090}
1091
1092fn make_disk_config_inner(
1093    location: u8,
1094    disk_params: &DiskParameters,
1095) -> Result<scsidisk_resources::DiskParameters, Vtl2SettingsErrorInfo> {
1096    let vendor_id = disk_params
1097        .vendor_id
1098        .parse()
1099        .map_err(|source| Error::StorageBadVendorId {
1100            lun: location,
1101            vendor_id: &disk_params.vendor_id,
1102            source,
1103        })?;
1104
1105    let product_id =
1106        disk_params
1107            .product_id
1108            .parse()
1109            .map_err(|source| Error::StorageBadProductId {
1110                lun: location,
1111                product_id: &disk_params.product_id,
1112                source,
1113            })?;
1114
1115    let product_revision_level = disk_params
1116        .product_revision_level
1117        .parse()
1118        .map_err(|source| Error::StorageBadProductRevisionLevel {
1119            lun: location,
1120            product_revision_level: &disk_params.product_revision_level,
1121            source,
1122        })?;
1123
1124    let disk_id = disk_params
1125        .device_id
1126        .parse::<Guid>()
1127        .map_err(|_| Error::StorageBadDeviceId {
1128            lun: location,
1129            device_id: &disk_params.device_id,
1130        })?;
1131
1132    if disk_id == Guid::ZERO {
1133        return Err(Error::StorageBadDeviceId {
1134            lun: location,
1135            device_id: &disk_params.device_id,
1136        }
1137        .into());
1138    }
1139
1140    Ok(scsidisk_resources::DiskParameters {
1141        disk_id: Some(disk_id.into()),
1142        identity: Some(scsidisk_resources::DiskIdentity {
1143            vendor_id,
1144            product_id,
1145            product_revision_level,
1146            model_number: disk_params.model_number.clone().into_bytes(),
1147        }),
1148        serial_number: disk_params.serial_number.clone().into_bytes(),
1149        medium_rotation_rate: Some(disk_params.medium_rotation_rate),
1150        physical_sector_size: disk_params.physical_sector_size,
1151        fua: disk_params.fua,
1152        write_cache: disk_params.write_cache,
1153        scsi_disk_size_in_bytes: disk_params.scsi_disk_size_in_bytes,
1154        odx: disk_params.odx,
1155        unmap: disk_params.unmap,
1156        get_lba_status: true,
1157        max_transfer_length: disk_params.max_transfer_length,
1158        optimal_unmap_sectors: None, // TODO
1159    })
1160}
1161
1162async fn make_ide_disk_config(
1163    ctx: &mut CancelContext,
1164    storage_context: &StorageContext<'_>,
1165    disk: &underhill_config::IdeDisk,
1166    is_restoring: bool,
1167) -> Result<(IdeDeviceConfig, Option<mesh::Sender<SimpleScsiDvdRequest>>), Vtl2SettingsErrorInfo> {
1168    let disk_type = make_disk_type(
1169        ctx,
1170        storage_context,
1171        &StorageDisk::Ide(disk.clone()),
1172        is_restoring,
1173    )
1174    .await?;
1175    if disk.is_dvd {
1176        let (send, recv) = mesh::channel();
1177        Ok((
1178            IdeDeviceConfig {
1179                path: ide_path_from_config(disk)?,
1180                guest_media: GuestMedia::Dvd(
1181                    SimpleScsiDvdHandle {
1182                        media: disk_type,
1183                        requests: Some(recv),
1184                    }
1185                    .into_resource(),
1186                ),
1187            },
1188            Some(send),
1189        ))
1190    } else {
1191        Ok((
1192            IdeDeviceConfig {
1193                path: ide_path_from_config(disk)?,
1194                guest_media: GuestMedia::Disk {
1195                    disk_type: disk_type.unwrap(),
1196                    read_only: false,
1197                    disk_parameters: Some(make_disk_config_inner(
1198                        disk.location,
1199                        &disk.disk_params,
1200                    )?),
1201                },
1202            },
1203            None,
1204        ))
1205    }
1206}
1207
1208async fn make_scsi_disk_config(
1209    ctx: &mut CancelContext,
1210    storage_context: &StorageContext<'_>,
1211    disk: &underhill_config::ScsiDisk,
1212    is_restoring: bool,
1213) -> Result<
1214    (
1215        ScsiDeviceAndPath,
1216        Option<mesh::Sender<SimpleScsiDvdRequest>>,
1217    ),
1218    Vtl2SettingsErrorInfo,
1219> {
1220    let path = scsi_path_from_config(disk)?;
1221    let disk_type = make_disk_type(
1222        ctx,
1223        storage_context,
1224        &StorageDisk::Scsi(disk.clone()),
1225        is_restoring,
1226    )
1227    .await?;
1228    let (device, dvd) = if disk.is_dvd {
1229        let (send, recv) = mesh::channel();
1230        (
1231            SimpleScsiDvdHandle {
1232                media: disk_type,
1233                requests: Some(recv),
1234            }
1235            .into_resource(),
1236            Some(send),
1237        )
1238    } else {
1239        (
1240            SimpleScsiDiskHandle {
1241                disk: disk_type.unwrap(),
1242                read_only: false,
1243                parameters: make_disk_config_inner(disk.location, &disk.disk_params)?,
1244            }
1245            .into_resource(),
1246            None,
1247        )
1248    };
1249    Ok((ScsiDeviceAndPath { path, device }, dvd))
1250}
1251
1252async fn make_disk_type(
1253    ctx: &mut CancelContext,
1254    storage_context: &StorageContext<'_>,
1255    disk: &StorageDisk,
1256    is_restoring: bool,
1257) -> Result<Option<Resource<DiskHandleKind>>, Vtl2SettingsErrorInfo> {
1258    let disk_type = match (disk.is_dvd(), disk.physical_devices()) {
1259        (true, PhysicalDevices::EmptyDrive) => None,
1260        (false, PhysicalDevices::EmptyDrive) => unreachable!("schema validated"),
1261        (false, physical_devices) => Some(
1262            make_disk_type_from_physical_devices(
1263                ctx,
1264                storage_context,
1265                physical_devices,
1266                disk.ntfs_guid(),
1267                false,
1268                is_restoring,
1269            )
1270            .await?,
1271        ),
1272        (true, physical_devices) => {
1273            let disk_type = make_disk_type_from_physical_devices(
1274                ctx,
1275                storage_context,
1276                physical_devices,
1277                disk.ntfs_guid(),
1278                true,
1279                is_restoring,
1280            )
1281            .await;
1282            match disk_type {
1283                Ok(disk_type) => Some(disk_type),
1284                Err(err_info) => match err_info.code() {
1285                    Vtl2SettingsErrorCode::StorageCannotOpenVtl2Device => {
1286                        tracing::warn!(CVM_ALLOWED, ?err_info, "Check if ISO is present on drive");
1287                        None
1288                    }
1289                    _ => Err(err_info)?,
1290                },
1291            }
1292        }
1293    };
1294    Ok(disk_type)
1295}
1296
1297async fn make_nvme_disk_config(
1298    ctx: &mut CancelContext,
1299    storage_context: &StorageContext<'_>,
1300    namespace: &underhill_config::NvmeNamespace,
1301    is_restoring: bool,
1302) -> Result<NamespaceDefinition, Vtl2SettingsErrorInfo> {
1303    let disk_type = make_disk_type_from_physical_devices(
1304        ctx,
1305        storage_context,
1306        &namespace.physical_devices,
1307        None,
1308        false,
1309        is_restoring,
1310    )
1311    .await?;
1312    Ok(NamespaceDefinition {
1313        nsid: namespace.nsid,
1314        disk: disk_type,
1315        read_only: false,
1316    })
1317}
1318
1319#[instrument(skip_all, fields(CVM_ALLOWED))]
1320async fn make_ide_controller_config(
1321    ctx: &mut CancelContext,
1322    storage_context: &StorageContext<'_>,
1323    settings: &Vtl2SettingsDynamic,
1324    is_restoring: bool,
1325) -> Result<Option<UhIdeControllerConfig>, Vtl2SettingsErrorInfo> {
1326    let mut primary_channel_disks = Vec::new();
1327    let mut secondary_channel_disks = Vec::new();
1328    let mut io_queue_depth = None;
1329
1330    let mut dvds = Vec::new();
1331    if let Some(ide_controller) = &settings.ide_controller {
1332        io_queue_depth = ide_controller.io_queue_depth;
1333        for disk in &ide_controller.disks {
1334            let (config, dvd) =
1335                make_ide_disk_config(ctx, storage_context, disk, is_restoring).await?;
1336            if let Some(dvd) = dvd {
1337                dvds.push((config.path, dvd));
1338            }
1339            if disk.channel == 0 {
1340                primary_channel_disks.push(config);
1341            } else {
1342                secondary_channel_disks.push(config);
1343            }
1344        }
1345    }
1346
1347    if primary_channel_disks.is_empty() && secondary_channel_disks.is_empty() {
1348        Ok(None)
1349    } else {
1350        Ok(Some(UhIdeControllerConfig {
1351            config: IdeControllerConfig {
1352                primary_channel_disks,
1353                secondary_channel_disks,
1354                io_queue_depth,
1355            },
1356            dvds,
1357        }))
1358    }
1359}
1360
1361#[instrument(skip_all, fields(CVM_ALLOWED))]
1362async fn make_scsi_controller_config(
1363    ctx: &mut CancelContext,
1364    storage_context: &StorageContext<'_>,
1365    controller: &underhill_config::ScsiController,
1366    scsi_sub_channels: u16,
1367    is_restoring: bool,
1368    default_io_queue_depth: u32,
1369) -> Result<UhScsiControllerConfig, Vtl2SettingsErrorInfo> {
1370    let instance_id = controller.instance_id;
1371    let mut scsi_disks = Vec::new();
1372
1373    let mut dvds = Vec::new();
1374    for disk in &controller.disks {
1375        let (disk_cfg, dvd) =
1376            make_scsi_disk_config(ctx, storage_context, disk, is_restoring).await?;
1377        if let Some(dvd) = dvd {
1378            dvds.push((disk_cfg.path, dvd));
1379        }
1380        scsi_disks.push(disk_cfg);
1381    }
1382
1383    let (send, recv) = mesh::channel();
1384    // The choice of max 256 scsi subchannels is somewhat arbitrary. But
1385    // for now, it provides a decent trade off between unbounded number of
1386    // channels that cause perf issues vs delivering some reasonable perf.
1387    Ok(UhScsiControllerConfig {
1388        handle: ScsiControllerHandle {
1389            instance_id,
1390            max_sub_channel_count: scsi_sub_channels.min(256),
1391            devices: scsi_disks,
1392            io_queue_depth: Some(controller.io_queue_depth.unwrap_or(default_io_queue_depth)),
1393            requests: Some(recv),
1394        },
1395        request: send,
1396        dvds,
1397    })
1398}
1399
1400#[instrument(skip_all, fields(CVM_ALLOWED))]
1401async fn make_nvme_controller_config(
1402    ctx: &mut CancelContext,
1403    storage_context: &StorageContext<'_>,
1404    controller: &underhill_config::NvmeController,
1405    is_restoring: bool,
1406) -> Result<UhVpciDeviceConfig, Vtl2SettingsErrorInfo> {
1407    let mut namespaces = Vec::new();
1408    for namespace in &controller.namespaces {
1409        namespaces
1410            .push(make_nvme_disk_config(ctx, storage_context, namespace, is_restoring).await?);
1411    }
1412
1413    Ok(UhVpciDeviceConfig {
1414        instance_id: controller.instance_id,
1415        resource: NvmeControllerHandle {
1416            subsystem_id: controller.instance_id,
1417            namespaces,
1418            max_io_queues: 64,
1419            msix_count: 64,
1420        }
1421        .into_resource(),
1422    })
1423}
1424
1425pub async fn create_storage_controllers_from_vtl2_settings(
1426    ctx: &mut CancelContext,
1427    uevent_listener: &UeventListener,
1428    use_nvme_vfio: bool,
1429    settings: &Vtl2SettingsDynamic,
1430    sub_channels: u16,
1431    is_restoring: bool,
1432    default_io_queue_depth: u32,
1433) -> Result<
1434    (
1435        Option<UhIdeControllerConfig>,
1436        Vec<UhScsiControllerConfig>,
1437        Vec<UhVpciDeviceConfig>,
1438    ),
1439    Vtl2SettingsErrorInfo,
1440> {
1441    let storage_context = StorageContext {
1442        uevent_listener,
1443        use_nvme_vfio,
1444    };
1445    let ide_controller =
1446        make_ide_controller_config(ctx, &storage_context, settings, is_restoring).await?;
1447
1448    let mut scsi_controllers = Vec::new();
1449    for controller in &settings.scsi_controllers {
1450        scsi_controllers.push(
1451            make_scsi_controller_config(
1452                ctx,
1453                &storage_context,
1454                controller,
1455                sub_channels,
1456                is_restoring,
1457                default_io_queue_depth,
1458            )
1459            .await?,
1460        );
1461    }
1462
1463    let mut nvme_controllers = Vec::new();
1464    for controller in &settings.nvme_controllers {
1465        nvme_controllers.push(
1466            make_nvme_controller_config(ctx, &storage_context, controller, is_restoring).await?,
1467        );
1468    }
1469
1470    Ok((ide_controller, scsi_controllers, nvme_controllers))
1471}
1472
1473fn nsid_matches(devpath: &Path, nsid: u32) -> anyhow::Result<bool> {
1474    match fs_err::read_to_string(devpath.join("nsid")) {
1475        Ok(this_nsid) => {
1476            let this_nsid: u32 = this_nsid.trim().parse().context("failed to parse nsid")?;
1477            Ok(this_nsid == nsid)
1478        }
1479        // If `nsid` is not found, then this sysfs path is not ready yet. Report
1480        // mismatch; we'll try again when the uevent arrives.
1481        Err(err) if err.kind() == std::io::ErrorKind::NotFound => Ok(false),
1482        Err(err) => Err(err.into()),
1483    }
1484}
1485
1486/// Checks if the block device `/dev/devname` is ready.
1487///
1488/// Linux adds the sysfs entry for a block device before the block device can
1489/// actually be opened; opens in this state will fail with ENXIO. But it delays
1490/// the uevent to notify the device's arrival until the device is fully ready
1491/// (and has even scanned the partition table). This function checks to ensure
1492/// the block device can be opened.
1493///
1494/// This should be called only when the block device was discovered by sysfs. If
1495/// it is not ready, then the caller should wait for a uevent indicating its
1496/// readiness.
1497async fn check_block_sysfs_ready(devname: &str) -> bool {
1498    let devname = devname.to_string();
1499    let dev_clone = devname.clone();
1500    match blocking::unblock(move || fs_err::File::open(Path::new("/dev").join(dev_clone))).await {
1501        Ok(_) => true,
1502        Err(err) if err.raw_os_error() == Some(libc::ENXIO) => {
1503            tracing::info!(
1504                CVM_ALLOWED,
1505                devname,
1506                "block device not ready, waiting for uevent"
1507            );
1508            false
1509        }
1510        Err(err) => {
1511            tracing::warn!(
1512                CVM_ALLOWED,
1513                error = &err as &dyn std::error::Error,
1514                devname,
1515                "block device failed to open during scan"
1516            );
1517            // The block device is ready but is failing to open. Let this one
1518            // propagate through so that we don't time out.
1519            true
1520        }
1521    }
1522}
1523
1524/// Returns the device path of an NVMe disk given the disk's namespace ID.
1525async fn get_nvme_namespace_devname(
1526    uevent_listener: &UeventListener,
1527    controller_instance_id: &Guid,
1528    nsid: u32,
1529) -> anyhow::Result<String> {
1530    // Wait for the nvme host to show up.
1531    let nvme_devpath =
1532        nvme_controller_path_from_vmbus_instance_id(uevent_listener, controller_instance_id)
1533            .await?;
1534
1535    // Get a prefix like `nvme0n` to use when evaluating child nodes.
1536    let prefix = nvme_devpath
1537        .file_name()
1538        .unwrap()
1539        .to_str()
1540        .unwrap()
1541        .to_owned()
1542        + "n";
1543
1544    // Look for a child node with the correct namespace ID.
1545    let devname = uevent_listener
1546        .wait_for_matching_child(&nvme_devpath, async |path, uevent| {
1547            let name = path.file_name()?.to_str()?;
1548            if !name.starts_with(&prefix) {
1549                return None;
1550            }
1551            match nsid_matches(&path, nsid) {
1552                Ok(true) => {
1553                    if uevent || check_block_sysfs_ready(name).await {
1554                        Some(Ok(name.to_string()))
1555                    } else {
1556                        None
1557                    }
1558                }
1559                Ok(false) => None,
1560                Err(err) => Some(Err(err)),
1561            }
1562        })
1563        .await??;
1564
1565    Ok(devname)
1566}
1567
1568async fn get_scsi_host_number(
1569    uevent_listener: &UeventListener,
1570    instance_id: &Guid,
1571) -> anyhow::Result<u32> {
1572    // Wait for a node of the name host<n> to show up.
1573    let controller_path = PathBuf::from(format!("/sys/bus/vmbus/devices/{instance_id}"));
1574    let host_number = uevent_listener
1575        .wait_for_matching_child(&controller_path, async |name, _uevent| {
1576            name.file_name()?
1577                .to_str()?
1578                .strip_prefix("host")?
1579                .parse()
1580                .ok()
1581        })
1582        .await?;
1583
1584    Ok(host_number)
1585}
1586
1587async fn get_vscsi_devname(
1588    uevent_listener: &UeventListener,
1589    controller_instance_id: &Guid,
1590    lun: u32,
1591) -> anyhow::Result<String> {
1592    // Wait for the SCSI host.
1593    let host_number = get_scsi_host_number(uevent_listener, controller_instance_id).await?;
1594
1595    // Wait for the block device to show up.
1596    let block_devpath = PathBuf::from(format!(
1597        "/sys/bus/scsi/devices/{host_number}:0:0:{lun}/block"
1598    ));
1599
1600    uevent_listener
1601        .wait_for_matching_child(&block_devpath, async |path, uevent| {
1602            let name = path.file_name()?.to_str()?;
1603            if uevent || check_block_sysfs_ready(name).await {
1604                Some(name.to_string())
1605            } else {
1606                None
1607            }
1608        })
1609        .await
1610        .context("failed to wait for block path")
1611}
1612
1613/// Returns the device path of an NVMe controller given the instance ID of the VMBUS channel
1614/// associated to its parent virtual PCI bus.
1615async fn nvme_controller_path_from_vmbus_instance_id(
1616    uevent_listener: &UeventListener,
1617    instance_id: &Guid,
1618) -> anyhow::Result<PathBuf> {
1619    let (pci_id, mut devpath) = vpci_path(instance_id);
1620    devpath.push("nvme");
1621    let devpath = uevent_listener
1622        .wait_for_matching_child(&devpath, async |path, _uevent| Some(path))
1623        .await?;
1624    wait_for_pci_path(&pci_id).await;
1625    Ok(devpath)
1626}
1627
1628fn vpci_path(instance_id: &Guid) -> (String, PathBuf) {
1629    // The RID of a VPCI device is derived from its vmbus channel instance ID by
1630    // using Data2 as the segment ID and setting bus, device and function number to zero.
1631    let pci_id = format!("{:04x}:00:00.0", instance_id.data2);
1632    let pci_bus_id = &pci_id[..7];
1633
1634    // Wait for the vfio driver to bind.
1635    //
1636    // Example devpath: /sys/bus/vmbus/devices/bf66a3ce-2c8d-4443-a1c2-59f58e4fcb14/pci2c8d:00/2c8d:00:00.0
1637    let devpath = PathBuf::from(format!(
1638        "/sys/bus/vmbus/devices/{instance_id}/pci{pci_bus_id}/{pci_id}"
1639    ));
1640
1641    (pci_id, devpath)
1642}
1643
1644/// Waits for the PCI path to get populated. The PCI path is just a symlink to the actual
1645/// device path. This should be called once the device path is available.
1646pub async fn wait_for_pci_path(pci_id: &String) {
1647    let pci_path = PathBuf::from(format!("/sys/bus/pci/devices/{pci_id}"));
1648    loop {
1649        if pci_path.exists() {
1650            return;
1651        }
1652
1653        let mut context = CancelContext::new().with_timeout(Duration::from_millis(10));
1654        let _ = context
1655            .until_cancelled({
1656                async {
1657                    futures::pending!();
1658                }
1659            })
1660            .await;
1661    }
1662}
1663
1664pub async fn wait_for_mana(
1665    uevent_listener: &UeventListener,
1666    instance_id: &Guid,
1667) -> anyhow::Result<String> {
1668    let (pci_id, devpath) = vpci_path(instance_id);
1669
1670    // Wait for the device to show up.
1671    uevent_listener.wait_for_devpath(&devpath).await?;
1672    wait_for_pci_path(&pci_id).await;
1673
1674    // Validate the device and vendor.
1675    let vendor = fs_err::read_to_string(devpath.join("vendor"))?;
1676    let device = fs_err::read_to_string(devpath.join("device"))?;
1677    if vendor.trim_end() != "0x1414" {
1678        anyhow::bail!("invalid mana vendor {vendor}");
1679    }
1680    if device.trim_end() != "0x00ba" {
1681        anyhow::bail!("invalid mana device {device}");
1682    }
1683
1684    Ok(pci_id)
1685}
1686
1687pub async fn get_mana_config_from_vtl2_settings(
1688    ctx: &mut CancelContext,
1689    uevent_listener: &UeventListener,
1690    settings: &Vtl2SettingsDynamic,
1691) -> anyhow::Result<Vec<NicConfig>> {
1692    let mut mana = Vec::<NicConfig>::new();
1693    for config in &settings.nic_devices {
1694        let pci_id = ctx
1695            .until_cancelled(wait_for_mana(uevent_listener, &config.instance_id))
1696            .await
1697            .context("cancelled waiting for mana devices")??;
1698
1699        mana.push(NicConfig {
1700            pci_id,
1701            instance_id: config.instance_id,
1702            subordinate_instance_id: config.subordinate_instance_id,
1703            max_sub_channels: config.max_sub_channels,
1704        });
1705    }
1706    Ok(mana)
1707}
1708
1709trait HasInstanceId {
1710    fn instance_id(&self) -> Guid;
1711}
1712
1713impl HasInstanceId for underhill_config::ScsiController {
1714    fn instance_id(&self) -> Guid {
1715        self.instance_id
1716    }
1717}
1718
1719impl HasInstanceId for underhill_config::IdeController {
1720    fn instance_id(&self) -> Guid {
1721        self.instance_id
1722    }
1723}
1724
1725impl HasInstanceId for NicDevice {
1726    fn instance_id(&self) -> Guid {
1727        self.instance_id
1728    }
1729}
1730
1731fn create_device_map_from_settings<T: HasInstanceId>(devices: &Vec<T>) -> HashMap<Guid, &T> {
1732    let mut map = HashMap::new();
1733
1734    for device in devices {
1735        map.insert(device.instance_id(), device);
1736    }
1737
1738    map
1739}
1740
1741fn calculate_device_change_from_map<'a, T>(
1742    old_device_map: &'a HashMap<Guid, &T>,
1743    new_device_map: &'a HashMap<Guid, &T>,
1744) -> (Vec<(&'a T, &'a T)>, Vec<&'a T>, Vec<&'a T>) {
1745    let mut devices_to_check = Vec::new();
1746    let mut devices_to_remove = Vec::new();
1747    let mut devices_to_add = Vec::new();
1748
1749    for (instance_id, old_config) in old_device_map.iter() {
1750        match new_device_map.get(instance_id) {
1751            Some(new_config) => {
1752                // add device to check list
1753                devices_to_check.push((*old_config, *new_config));
1754            }
1755            // remove the device
1756            _ => devices_to_remove.push(*old_config),
1757        }
1758    }
1759
1760    for (instance_id, config) in new_device_map.iter() {
1761        if old_device_map.get(instance_id).is_none() {
1762            devices_to_add.push(*config)
1763        }
1764    }
1765
1766    (devices_to_check, devices_to_remove, devices_to_add)
1767}
1768
1769pub struct InitialControllers {
1770    pub ide_controller: Option<IdeControllerConfig>,
1771    pub vmbus_devices: Vec<Resource<VmbusDeviceHandleKind>>,
1772    pub vpci_devices: Vec<UhVpciDeviceConfig>,
1773    pub mana: Vec<NicConfig>,
1774    pub device_interfaces: DeviceInterfaces,
1775}
1776
1777impl InitialControllers {
1778    /// Construct initial network/storage controllers from initial Vtl2Settings
1779    pub async fn new(
1780        uevent_listener: &UeventListener,
1781        dps: &DevicePlatformSettings,
1782        use_nvme_vfio: bool,
1783        is_restoring: bool,
1784        default_io_queue_depth: u32,
1785    ) -> anyhow::Result<Self> {
1786        const VM_CONFIG_TIME_OUT_IN_SECONDS: u64 = 5;
1787        let mut context =
1788            CancelContext::new().with_timeout(Duration::from_secs(VM_CONFIG_TIME_OUT_IN_SECONDS));
1789
1790        let vtl2_settings = dps.general.vtl2_settings.as_ref();
1791
1792        tracing::info!(CVM_ALLOWED, ?vtl2_settings, "Initial VTL2 settings");
1793
1794        let fixed = vtl2_settings.map_or_else(Default::default, |s| s.fixed.clone());
1795        let dynamic = vtl2_settings.map(|s| &s.dynamic);
1796
1797        let (ide_controller, scsi_controllers, vpci_devices) = if let Some(dynamic) = &dynamic {
1798            create_storage_controllers_from_vtl2_settings(
1799                &mut context,
1800                uevent_listener,
1801                use_nvme_vfio,
1802                dynamic,
1803                fixed.scsi_sub_channels,
1804                is_restoring,
1805                default_io_queue_depth,
1806            )
1807            .await?
1808        } else {
1809            (None, Vec::new(), Vec::new())
1810        };
1811
1812        let mana = if let Some(dynamic) = &dynamic {
1813            get_mana_config_from_vtl2_settings(&mut context, uevent_listener, dynamic).await?
1814        } else {
1815            Vec::new()
1816        };
1817
1818        let mut scsi_dvds = HashMap::new();
1819        let mut scsi_request = HashMap::new();
1820
1821        let ide_controller = ide_controller.map(|c| {
1822            scsi_dvds.extend(
1823                c.dvds
1824                    .into_iter()
1825                    .map(|(path, send)| (StorageDevicePath::Ide(path), send)),
1826            );
1827            c.config
1828        });
1829
1830        let vmbus_devices = scsi_controllers
1831            .into_iter()
1832            .map(|c| {
1833                scsi_dvds.extend(
1834                    c.dvds
1835                        .into_iter()
1836                        .map(|(path, send)| (StorageDevicePath::Scsi(path), send)),
1837                );
1838                scsi_request.insert(c.handle.instance_id, c.request);
1839                c.handle.into_resource()
1840            })
1841            .collect();
1842
1843        let cfg = InitialControllers {
1844            ide_controller,
1845            vmbus_devices,
1846            mana,
1847            vpci_devices,
1848            device_interfaces: DeviceInterfaces {
1849                scsi_dvds,
1850                scsi_request,
1851                use_nvme_vfio,
1852            },
1853        };
1854
1855        Ok(cfg)
1856    }
1857}