1#![cfg_attr(any(windows, target_os = "linux"), expect(unsafe_code))]
10
11use inspect::Inspect;
12use pal_async::driver::Driver;
13use pal_async::task::Spawn;
14use pal_async::task::TaskMetadata;
15use std::fmt::Debug;
16use std::pin::Pin;
17use std::sync::Arc;
18
19#[derive(Clone)]
27pub struct VmTaskDriverSource {
28 backend: Arc<dyn DynVmBackend>,
29}
30
31impl VmTaskDriverSource {
32 pub fn new(backend: impl 'static + BuildVmTaskDriver) -> Self {
34 Self {
35 backend: Arc::new(backend),
36 }
37 }
38
39 pub fn simple(&self) -> VmTaskDriver {
43 self.builder().build("")
46 }
47
48 pub fn current(&self) -> VmTaskDriver {
54 VmTaskDriver {
55 inner: self.backend.build_current(),
56 }
57 }
58
59 pub fn builder(&self) -> VmTaskDriverBuilder<'_> {
61 VmTaskDriverBuilder {
62 backend: self.backend.as_ref(),
63 run_on_target: false,
64 target_vp: None,
65 }
66 }
67}
68
69pub trait BuildVmTaskDriver: Send + Sync {
71 type Driver: TargetedDriver;
73 type CurrentDriver: TargetedDriver;
75
76 fn build(&self, name: String, target_vp: Option<u32>, run_on_target: bool) -> Self::Driver;
78
79 fn build_current(&self) -> Self::CurrentDriver;
81}
82
83pub trait TargetedDriver: 'static + Send + Sync + Inspect {
85 fn spawner(&self) -> &dyn Spawn;
87 fn driver(&self) -> &dyn Driver;
89 fn retarget_vp(&self, target_vp: u32);
91 fn is_target_vp_ready(&self) -> bool {
96 true
97 }
98 fn wait_target_vp_ready(&self) -> impl Future<Output = ()> + Send {
100 std::future::ready(())
101 }
102}
103
104trait DynTargetedDriver: 'static + Send + Sync + Inspect {
105 fn spawner(&self) -> &dyn Spawn;
106 fn driver(&self) -> &dyn Driver;
107 fn retarget_vp(&self, target_vp: u32);
108 fn is_ready(&self) -> bool;
109 fn wait_ready(&self) -> Pin<Box<dyn '_ + Future<Output = ()> + Send>>;
110}
111
112impl<T: TargetedDriver> DynTargetedDriver for T {
113 fn spawner(&self) -> &dyn Spawn {
114 self.spawner()
115 }
116
117 fn driver(&self) -> &dyn Driver {
118 self.driver()
119 }
120
121 fn retarget_vp(&self, target_vp: u32) {
122 self.retarget_vp(target_vp)
123 }
124
125 fn is_ready(&self) -> bool {
126 self.is_target_vp_ready()
127 }
128
129 fn wait_ready(&self) -> Pin<Box<dyn '_ + Future<Output = ()> + Send>> {
130 Box::pin(self.wait_target_vp_ready())
131 }
132}
133
134trait DynVmBackend: Send + Sync {
135 fn build(
136 &self,
137 name: String,
138 target_vp: Option<u32>,
139 run_on_target: bool,
140 ) -> Arc<dyn DynTargetedDriver>;
141
142 fn build_current(&self) -> Arc<dyn DynTargetedDriver>;
143}
144
145impl<T: BuildVmTaskDriver> DynVmBackend for T {
146 fn build(
147 &self,
148 name: String,
149 target_vp: Option<u32>,
150 run_on_target: bool,
151 ) -> Arc<dyn DynTargetedDriver> {
152 Arc::new(self.build(name, target_vp, run_on_target))
153 }
154
155 fn build_current(&self) -> Arc<dyn DynTargetedDriver> {
156 Arc::new(BuildVmTaskDriver::build_current(self))
157 }
158}
159
160pub struct VmTaskDriverBuilder<'a> {
162 backend: &'a dyn DynVmBackend,
163 run_on_target: bool,
164 target_vp: Option<u32>,
165}
166
167impl VmTaskDriverBuilder<'_> {
168 pub fn run_on_target(&mut self, run_on_target: bool) -> &mut Self {
180 self.run_on_target = run_on_target;
181 self
182 }
183
184 pub fn target_vp(&mut self, target_vp: u32) -> &mut Self {
195 self.target_vp = Some(target_vp);
196 self
197 }
198
199 pub fn build(&self, name: impl Into<String>) -> VmTaskDriver {
204 VmTaskDriver {
205 inner: self
206 .backend
207 .build(name.into(), self.target_vp, self.run_on_target),
208 }
209 }
210}
211
212#[derive(Clone, Inspect)]
216pub struct VmTaskDriver {
217 #[inspect(flatten)]
218 inner: Arc<dyn DynTargetedDriver>,
219}
220
221impl VmTaskDriver {
222 pub fn retarget_vp(&self, target_vp: u32) {
229 self.inner.retarget_vp(target_vp)
230 }
231
232 pub fn is_target_vp_ready(&self) -> bool {
237 self.inner.is_ready()
238 }
239
240 pub async fn wait_target_vp_ready(&self) {
242 self.inner.wait_ready().await
243 }
244}
245
246impl Driver for VmTaskDriver {
247 fn new_dyn_timer(&self) -> pal_async::driver::PollImpl<dyn pal_async::timer::PollTimer> {
248 self.inner.driver().new_dyn_timer()
249 }
250
251 #[cfg(unix)]
252 fn new_dyn_fd_ready(
253 &self,
254 fd: std::os::fd::RawFd,
255 ) -> std::io::Result<pal_async::driver::PollImpl<dyn pal_async::fd::PollFdReady>> {
256 self.inner.driver().new_dyn_fd_ready(fd)
257 }
258
259 #[cfg(unix)]
260 fn new_dyn_socket_ready(
261 &self,
262 socket: std::os::fd::RawFd,
263 ) -> std::io::Result<pal_async::driver::PollImpl<dyn pal_async::socket::PollSocketReady>> {
264 self.inner.driver().new_dyn_socket_ready(socket)
265 }
266
267 #[cfg(windows)]
268 fn new_dyn_socket_ready(
269 &self,
270 socket: std::os::windows::io::RawSocket,
271 ) -> std::io::Result<pal_async::driver::PollImpl<dyn pal_async::socket::PollSocketReady>> {
272 self.inner.driver().new_dyn_socket_ready(socket)
273 }
274
275 #[cfg(unix)]
276 fn new_dyn_wait(
277 &self,
278 fd: std::os::fd::RawFd,
279 read_size: usize,
280 ) -> std::io::Result<pal_async::driver::PollImpl<dyn pal_async::wait::PollWait>> {
281 self.inner.driver().new_dyn_wait(fd, read_size)
282 }
283
284 #[cfg(windows)]
285 fn new_dyn_wait(
286 &self,
287 handle: std::os::windows::io::RawHandle,
288 ) -> std::io::Result<pal_async::driver::PollImpl<dyn pal_async::wait::PollWait>> {
289 self.inner.driver().new_dyn_wait(handle)
290 }
291
292 #[cfg(windows)]
293 unsafe fn new_dyn_overlapped_file(
294 &self,
295 handle: std::os::windows::io::RawHandle,
296 ) -> std::io::Result<
297 pal_async::driver::PollImpl<dyn pal_async::windows::overlapped::IoOverlapped>,
298 > {
299 unsafe { self.inner.driver().new_dyn_overlapped_file(handle) }
301 }
302
303 #[cfg(target_os = "linux")]
304 fn io_uring_probe(&self, opcode: u8) -> bool {
305 self.inner.driver().io_uring_probe(opcode)
306 }
307
308 #[cfg(target_os = "linux")]
309 unsafe fn io_uring_submit(
310 &self,
311 sqe: pal_async::io_uring::Entry,
312 ) -> Pin<Box<dyn Future<Output = std::io::Result<i32>> + Send + '_>> {
313 unsafe { self.inner.driver().io_uring_submit(sqe) }
315 }
316
317 #[cfg(target_os = "macos")]
318 fn new_dyn_process_wait(
319 &self,
320 pid: i32,
321 ) -> std::io::Result<pal_async::driver::PollImpl<dyn pal_async::process::macos::PollProcessWait>>
322 {
323 self.inner.driver().new_dyn_process_wait(pid)
324 }
325}
326
327impl Spawn for VmTaskDriver {
328 fn scheduler(&self, metadata: &TaskMetadata) -> Arc<dyn pal_async::task::Schedule> {
329 self.inner.spawner().scheduler(metadata)
330 }
331}
332
333#[derive(Debug)]
335pub struct SingleDriverBackend<T>(T);
336
337impl<T: Driver + Spawn + Clone> SingleDriverBackend<T> {
338 pub fn new(driver: T) -> Self {
341 Self(driver)
342 }
343}
344
345#[derive(Debug)]
347pub struct SingleDriver<T>(T);
348
349impl<T> Inspect for SingleDriver<T> {
350 fn inspect(&self, req: inspect::Request<'_>) {
351 req.ignore();
352 }
353}
354
355impl<T: Driver + Spawn + Clone> BuildVmTaskDriver for SingleDriverBackend<T> {
356 type Driver = SingleDriver<T>;
357 type CurrentDriver = SingleDriver<T>;
358
359 fn build(&self, _name: String, _target_vp: Option<u32>, _run_on_target: bool) -> Self::Driver {
360 SingleDriver(self.0.clone())
361 }
362
363 fn build_current(&self) -> Self::CurrentDriver {
364 SingleDriver(self.0.clone())
365 }
366}
367
368impl<T: Driver + Spawn> TargetedDriver for SingleDriver<T> {
369 fn spawner(&self) -> &dyn Spawn {
370 &self.0
371 }
372
373 fn driver(&self) -> &dyn Driver {
374 &self.0
375 }
376
377 fn retarget_vp(&self, _target_vp: u32) {}
378}
379
380pub mod thread {
381 use super::BuildVmTaskDriver;
385 use super::TargetedDriver;
386 use inspect::Inspect;
387 use loan_cell::LoanCell;
388 use pal_async::DefaultDriver;
389 use pal_async::DefaultPool;
390 use pal_async::WeakDefaultDriver;
391 use pal_async::driver::Driver;
392 use pal_async::task::Spawn;
393 use pal_async::task::TaskMetadata;
394 use std::sync::Arc;
395
396 thread_local! {
397 static CURRENT_DRIVER: LoanCell<WeakDefaultDriver> = const { LoanCell::new() };
402 }
403
404 #[derive(Debug)]
411 pub struct ThreadDriverBackend {
412 default_driver: DefaultDriver,
413 }
414
415 impl ThreadDriverBackend {
416 pub fn new(default_driver: DefaultDriver) -> Self {
419 Self { default_driver }
420 }
421 }
422
423 impl BuildVmTaskDriver for ThreadDriverBackend {
424 type Driver = ThreadDriver;
425 type CurrentDriver = CurrentThreadDriver;
426
427 fn build(
428 &self,
429 name: String,
430 target_vp: Option<u32>,
431 _run_on_target: bool,
432 ) -> Self::Driver {
433 if target_vp.is_some() {
435 let pool = DefaultPool::new();
436 let driver = pool.driver();
437 let tls_driver = driver.downgrade();
447 std::thread::Builder::new()
448 .name(name)
449 .spawn(move || {
450 CURRENT_DRIVER.with(|cell| cell.lend(&tls_driver, || pool.run()));
451 })
452 .unwrap();
453 ThreadDriver {
454 inner: driver,
455 has_dedicated_thread: true,
456 }
457 } else {
458 ThreadDriver {
459 inner: self.default_driver.clone(),
460 has_dedicated_thread: false,
461 }
462 }
463 }
464
465 fn build_current(&self) -> Self::CurrentDriver {
466 CurrentThreadDriver {
467 default: self.default_driver.clone(),
468 }
469 }
470 }
471
472 #[derive(Debug, Inspect)]
474 pub struct ThreadDriver {
475 #[inspect(skip)]
476 inner: DefaultDriver,
477 has_dedicated_thread: bool,
478 }
479
480 impl TargetedDriver for ThreadDriver {
481 fn spawner(&self) -> &dyn Spawn {
482 &self.inner
483 }
484
485 fn driver(&self) -> &dyn Driver {
486 &self.inner
487 }
488
489 fn retarget_vp(&self, _target_vp: u32) {}
490 }
491
492 #[derive(Inspect)]
498 pub struct CurrentThreadDriver {
499 #[inspect(skip)]
500 default: DefaultDriver,
501 }
502
503 impl CurrentThreadDriver {
504 fn with_driver<R>(&self, f: impl FnOnce(&DefaultDriver) -> R) -> R {
505 CURRENT_DRIVER.with(|cell| {
506 cell.borrow(|weak| {
507 match weak.and_then(|w| w.upgrade()) {
515 Some(driver) => f(&driver),
516 None => f(&self.default),
517 }
518 })
519 })
520 }
521 }
522
523 impl Driver for CurrentThreadDriver {
524 fn new_dyn_timer(&self) -> pal_async::driver::PollImpl<dyn pal_async::timer::PollTimer> {
525 self.with_driver(|d| d.new_dyn_timer())
526 }
527
528 #[cfg(unix)]
529 fn new_dyn_fd_ready(
530 &self,
531 fd: std::os::fd::RawFd,
532 ) -> std::io::Result<pal_async::driver::PollImpl<dyn pal_async::fd::PollFdReady>> {
533 self.with_driver(|d| d.new_dyn_fd_ready(fd))
534 }
535
536 #[cfg(unix)]
537 fn new_dyn_socket_ready(
538 &self,
539 socket: std::os::fd::RawFd,
540 ) -> std::io::Result<pal_async::driver::PollImpl<dyn pal_async::socket::PollSocketReady>>
541 {
542 self.with_driver(|d| d.new_dyn_socket_ready(socket))
543 }
544
545 #[cfg(windows)]
546 fn new_dyn_socket_ready(
547 &self,
548 socket: std::os::windows::io::RawSocket,
549 ) -> std::io::Result<pal_async::driver::PollImpl<dyn pal_async::socket::PollSocketReady>>
550 {
551 self.with_driver(|d| d.new_dyn_socket_ready(socket))
552 }
553
554 #[cfg(unix)]
555 fn new_dyn_wait(
556 &self,
557 fd: std::os::fd::RawFd,
558 read_size: usize,
559 ) -> std::io::Result<pal_async::driver::PollImpl<dyn pal_async::wait::PollWait>> {
560 self.with_driver(|d| d.new_dyn_wait(fd, read_size))
561 }
562
563 #[cfg(windows)]
564 fn new_dyn_wait(
565 &self,
566 handle: std::os::windows::io::RawHandle,
567 ) -> std::io::Result<pal_async::driver::PollImpl<dyn pal_async::wait::PollWait>> {
568 self.with_driver(|d| d.new_dyn_wait(handle))
569 }
570
571 #[cfg(windows)]
572 unsafe fn new_dyn_overlapped_file(
573 &self,
574 handle: std::os::windows::io::RawHandle,
575 ) -> std::io::Result<
576 pal_async::driver::PollImpl<dyn pal_async::windows::overlapped::IoOverlapped>,
577 > {
578 self.with_driver(|d| {
579 unsafe { d.new_dyn_overlapped_file(handle) }
581 })
582 }
583
584 #[cfg(target_os = "linux")]
585 fn io_uring_probe(&self, opcode: u8) -> bool {
586 self.with_driver(|d| d.io_uring_probe(opcode))
587 }
588
589 #[cfg(target_os = "linux")]
590 unsafe fn io_uring_submit(
591 &self,
592 sqe: pal_async::io_uring::Entry,
593 ) -> std::pin::Pin<Box<dyn Future<Output = std::io::Result<i32>> + Send + '_>> {
594 use pal_async::io_uring::{IoUringDriver, IoUringSubmit};
595 Box::pin(async move {
596 let driver =
604 CURRENT_DRIVER.with(|cell| cell.borrow(|weak| weak.and_then(|w| w.upgrade())));
605 let driver = driver.as_ref().unwrap_or(&self.default);
606 unsafe {
608 driver
609 .io_uring_submitter()
610 .ok_or(std::io::ErrorKind::Unsupported)?
611 .submit(sqe)
612 .await
613 }
614 })
615 }
616
617 #[cfg(target_os = "macos")]
618 fn new_dyn_process_wait(
619 &self,
620 pid: i32,
621 ) -> std::io::Result<
622 pal_async::driver::PollImpl<dyn pal_async::process::macos::PollProcessWait>,
623 > {
624 self.with_driver(|d| d.new_dyn_process_wait(pid))
625 }
626 }
627
628 impl Spawn for CurrentThreadDriver {
629 fn scheduler(&self, metadata: &TaskMetadata) -> Arc<dyn pal_async::task::Schedule> {
630 self.with_driver(|d| d.scheduler(metadata))
631 }
632 }
633
634 impl TargetedDriver for CurrentThreadDriver {
635 fn spawner(&self) -> &dyn Spawn {
636 self
637 }
638
639 fn driver(&self) -> &dyn Driver {
640 self
641 }
642
643 fn retarget_vp(&self, _target_vp: u32) {}
644 }
645
646 #[cfg(test)]
647 mod tests {
648 use super::BuildVmTaskDriver;
649 use super::CURRENT_DRIVER;
650 use super::ThreadDriver;
651 use super::ThreadDriverBackend;
652 use futures::task::ArcWake;
653 use pal_async::DefaultPool;
654 use pal_async::task::Spawn;
655 use std::cell::RefCell;
656 use std::future::Future;
657 use std::pin::Pin;
658 use std::sync::Arc;
659 use std::sync::mpsc;
660 use std::task::Context;
661 use std::task::Poll;
662
663 const POOL_NAME: &str = "test-vp-pool";
664
665 thread_local! {
666 static EXIT_SIGNAL: RefCell<Option<SignalOnDrop>> = const { RefCell::new(None) };
667 }
668
669 struct SignalOnDrop(mpsc::Sender<()>);
671
672 impl Drop for SignalOnDrop {
673 fn drop(&mut self) {
674 let _ = self.0.send(());
675 }
676 }
677
678 fn dedicated_driver() -> (ThreadDriver, mpsc::Receiver<()>) {
681 let default_pool = DefaultPool::new();
682 let backend = ThreadDriverBackend::new(default_pool.driver());
683 let driver = backend.build(POOL_NAME.into(), Some(0), false);
684 assert!(
685 driver.has_dedicated_thread,
686 "a target VP must get its own pool thread, else this test asserts nothing"
687 );
688 let (send, exited) = mpsc::channel();
692 driver
693 .inner
694 .spawn("exit-signal", async move {
695 EXIT_SIGNAL.with(|slot| *slot.borrow_mut() = Some(SignalOnDrop(send)));
696 })
697 .detach();
698 (driver, exited)
699 }
700
701 fn assert_thread_exits(exited: &mpsc::Receiver<()>) {
704 exited
705 .recv()
706 .expect("the exit signal was dropped without being sent");
707 }
708
709 #[derive(Debug, PartialEq)]
711 struct DriverView {
712 thread: Option<String>,
713 lent: bool,
714 upgraded: bool,
715 }
716
717 fn driver_view() -> DriverView {
721 CURRENT_DRIVER.with(|cell| {
722 cell.borrow(|weak| DriverView {
723 thread: std::thread::current().name().map(String::from),
724 lent: weak.is_some(),
725 upgraded: weak.and_then(|w| w.upgrade()).is_some(),
726 })
727 })
728 }
729
730 struct ViewOnDrop(mpsc::Sender<DriverView>);
732
733 impl Drop for ViewOnDrop {
734 fn drop(&mut self) {
735 let _ = self.0.send(driver_view());
736 }
737 }
738
739 struct ViewOnWake(mpsc::Sender<DriverView>);
741
742 impl ArcWake for ViewOnWake {
743 fn wake_by_ref(arc_self: &Arc<Self>) {
744 let _ = arc_self.0.send(driver_view());
745 }
746 }
747
748 struct CompleteOnSignal {
751 go: mpsc::Receiver<()>,
752 _view: ViewOnDrop,
753 }
754
755 impl Future for CompleteOnSignal {
756 type Output = ();
757
758 fn poll(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll<()> {
759 self.go.recv().unwrap();
761 Poll::Ready(())
762 }
763 }
764
765 fn assert_dedicated_driver(views: &mpsc::Receiver<DriverView>, path: &str) {
768 let view = views
769 .recv()
770 .unwrap_or_else(|_| panic!("{path}: the probe was dropped without reporting"));
771 assert_eq!(
772 view,
773 DriverView {
774 thread: Some(POOL_NAME.into()),
775 lent: true,
776 upgraded: true,
777 },
778 "{path}: the pool's driver must stay reachable while the task is \
779 torn down on its thread, else work started there falls back to \
780 the default driver"
781 );
782 }
783
784 #[test]
791 fn dedicated_pool_thread_exits_when_driver_dropped() {
792 let (driver, exited) = dedicated_driver();
793
794 drop(driver);
797
798 assert_thread_exits(&exited);
799 }
800
801 #[test]
805 fn driver_reachable_from_completed_future_destructor() {
806 let (driver, exited) = dedicated_driver();
807 let (view_send, views) = mpsc::channel();
808 let (go, go_recv) = mpsc::channel();
809 driver
810 .inner
811 .spawn(
812 "probe",
813 CompleteOnSignal {
814 go: go_recv,
815 _view: ViewOnDrop(view_send),
816 },
817 )
818 .detach();
819 drop(driver);
820 go.send(()).unwrap();
821
822 assert_dedicated_driver(&views, "completed future destructor");
823 assert_thread_exits(&exited);
824 }
825
826 #[test]
829 fn driver_reachable_from_cancelled_future_destructor() {
830 let (driver, exited) = dedicated_driver();
831 let (view_send, views) = mpsc::channel();
832 let view = ViewOnDrop(view_send);
833 let task = driver.inner.spawn("probe", async move {
834 let _view = view;
835 std::future::pending::<()>().await
836 });
837 drop(driver);
838 drop(task);
839
840 assert_dedicated_driver(&views, "cancelled future destructor");
841 assert_thread_exits(&exited);
842 }
843
844 #[test]
847 fn driver_reachable_from_detached_output_destructor() {
848 let (driver, exited) = dedicated_driver();
849 let (view_send, views) = mpsc::channel();
850 let (go, go_recv) = mpsc::channel::<()>();
851 driver
852 .inner
853 .spawn("probe", async move {
854 go_recv.recv().unwrap();
855 ViewOnDrop(view_send)
856 })
857 .detach();
858 drop(driver);
859 go.send(()).unwrap();
860
861 assert_dedicated_driver(&views, "detached output destructor");
862 assert_thread_exits(&exited);
863 }
864
865 #[test]
868 fn driver_reachable_from_completion_wake() {
869 let (driver, exited) = dedicated_driver();
870 let (view_send, views) = mpsc::channel();
871 let (go, go_recv) = mpsc::channel::<()>();
872 let mut task = driver.inner.spawn("probe", async move {
873 go_recv.recv().unwrap();
874 });
875 let waker = futures::task::waker(Arc::new(ViewOnWake(view_send)));
876 assert!(
877 Pin::new(&mut task)
878 .poll(&mut Context::from_waker(&waker))
879 .is_pending(),
880 "the task cannot complete before `go` fires"
881 );
882 drop(driver);
883 go.send(()).unwrap();
884
885 assert_dedicated_driver(&views, "completion wake");
886 drop(task);
887 assert_thread_exits(&exited);
888 }
889 }
890}