petri/
test.rs

1// Copyright (c) Microsoft Corporation.
2// Licensed under the MIT License.
3
4//! Infrastructure for defining tests.
5
6#[doc(hidden)]
7pub mod test_macro_support {
8    // UNSAFETY: Needed for linkme.
9    #![expect(unsafe_code)]
10
11    use super::TestCase;
12    pub use linkme;
13
14    #[linkme::distributed_slice]
15    pub static TESTS: [Option<fn() -> (&'static str, Vec<TestCase>)>];
16
17    // Always have at least one entry to work around linker bugs.
18    //
19    // See <https://github.com/llvm/llvm-project/issues/65855>.
20    #[linkme::distributed_slice(TESTS)]
21    static WORKAROUND: Option<fn() -> (&'static str, Vec<TestCase>)> = None;
22}
23
24use crate::PetriLogSource;
25use crate::TestArtifactRequirements;
26use crate::TestArtifacts;
27use crate::requirements::HostContext;
28use crate::requirements::TestCaseRequirements;
29use crate::requirements::can_run_test_with_context;
30use crate::tracing::try_init_tracing;
31use anyhow::Context as _;
32use futures::FutureExt as _;
33use pal_async::DefaultDriver;
34use pal_async::DefaultPool;
35use petri_artifacts_core::ArtifactResolver;
36use petri_artifacts_core::RemoteAccess;
37use std::panic::AssertUnwindSafe;
38use std::panic::catch_unwind;
39use test_macro_support::TESTS;
40
41/// Defines a single test named after the async run function `$f`, using `$req`
42/// to resolve its artifacts. See [`SimpleTest::new_async`].
43#[macro_export]
44macro_rules! test {
45    ($f:ident, $req:expr) => {
46        $crate::multitest!(vec![
47            $crate::SimpleTest::new_async(stringify!($f), $req, $f).into()
48        ]);
49    };
50}
51
52/// Defines a single unstable test named after the async run function `$f`,
53/// using `$req` to resolve its artifacts.
54///
55/// `$reason` documents why the test is unstable and is logged when an unstable
56/// failure is ignored.
57#[macro_export]
58macro_rules! unstable_test {
59    ($f:ident, $req:expr, $reason:expr) => {
60        $crate::multitest!(vec![
61            $crate::SimpleTest::new_async(stringify!($f), $req, $f)
62                .unstable($reason)
63                .into()
64        ]);
65    };
66}
67
68/// Defines a single test named after the synchronous run function `$f`, using
69/// `$req` to resolve its artifacts. See [`SimpleTest::new_sync`].
70#[macro_export]
71macro_rules! test_sync {
72    ($f:ident, $req:expr) => {
73        $crate::multitest!(vec![
74            $crate::SimpleTest::new_sync(stringify!($f), $req, $f).into()
75        ]);
76    };
77}
78
79/// Defines a single unstable test named after the synchronous run function
80/// `$f`, using `$req` to resolve its artifacts.
81///
82/// `$reason` documents why the test is unstable and is logged when an unstable
83/// failure is ignored.
84#[macro_export]
85macro_rules! unstable_test_sync {
86    ($f:ident, $req:expr, $reason:expr) => {
87        $crate::multitest!(vec![
88            $crate::SimpleTest::new_sync(stringify!($f), $req, $f)
89                .unstable($reason)
90                .into()
91        ]);
92    };
93}
94
95/// Defines a set of tests from an expression evaluating to a
96/// [`Vec<TestCase>`](TestCase).
97#[macro_export]
98macro_rules! multitest {
99    ($tests:expr) => {
100        const _: () = {
101            use $crate::test_macro_support::linkme;
102            #[linkme::distributed_slice($crate::test_macro_support::TESTS)]
103            #[linkme(crate = linkme)]
104            static TEST: Option<fn() -> (&'static str, Vec<$crate::TestCase>)> =
105                Some(|| (module_path!(), $tests));
106        };
107    };
108}
109
110/// A single test case.
111pub struct TestCase(Box<dyn DynRunTest>);
112
113impl TestCase {
114    /// Creates a new test case from a value that implements [`RunTest`].
115    pub fn new(test: impl 'static + RunTest) -> Self {
116        Self(Box::new(test))
117    }
118}
119
120impl<T: 'static + RunTest> From<T> for TestCase {
121    fn from(test: T) -> Self {
122        Self::new(test)
123    }
124}
125
126/// A single test, with module name.
127struct Test {
128    module: &'static str,
129    test: TestCase,
130    artifact_requirements: TestArtifactRequirements,
131}
132
133impl Test {
134    /// Returns all the tests defined in this crate.
135    fn all() -> impl Iterator<Item = Self> {
136        TESTS.iter().flatten().flat_map(|f| {
137            let (module, tests) = f();
138            tests.into_iter().filter_map(move |test| {
139                let mut artifact_requirements = test.0.artifact_requirements()?;
140                // All tests require the log directory.
141                artifact_requirements.require(
142                    petri_artifacts_common::artifacts::TEST_LOG_DIRECTORY,
143                    RemoteAccess::LocalOnly,
144                    false,
145                );
146                Some(Self {
147                    module,
148                    artifact_requirements,
149                    test,
150                })
151            })
152        })
153    }
154
155    /// Returns the name of the test.
156    fn name(&self) -> String {
157        // Strip the crate name from the module path, for consistency with libtest.
158        match self.module.split_once("::") {
159            Some((_crate_name, rest)) => format!("{}::{}", rest, self.test.0.leaf_name()),
160            None => self.test.0.leaf_name().to_owned(),
161        }
162    }
163
164    fn run(
165        &self,
166        resolve: fn(&str, TestArtifactRequirements) -> anyhow::Result<TestArtifacts>,
167    ) -> anyhow::Result<()> {
168        let name = self.name();
169        let artifacts = resolve(&name, self.artifact_requirements.clone())
170            .context("failed to resolve artifacts")?;
171        let output_dir = artifacts.get(petri_artifacts_common::artifacts::TEST_LOG_DIRECTORY);
172        let logger = try_init_tracing(output_dir, tracing::level_filters::LevelFilter::DEBUG)
173            .context("failed to initialize tracing")?;
174        // Record the test's identity up front, so that a test which is killed
175        // or crashes before reporting a result is still identifiable.
176        logger.log_test_start(&name);
177        let mut post_test_hooks = Vec::new();
178
179        // A process that is faulted or killed writes nothing to its own logs,
180        // so the host's error reporting events are the only record that it
181        // crashed and the only source of the Watson report ID needed to find
182        // the dump.
183        #[cfg(windows)]
184        post_test_hooks.push(collect_watson_events_hook(logger.clone()));
185
186        // Catch test panics in order to cleanly log the panic result. Without
187        // this, `libtest_mimic` will report the panic to stdout and fail the
188        // test, but the details won't end up in our per-test JSON log.
189        let r = catch_unwind(AssertUnwindSafe(|| {
190            self.test.0.run(
191                PetriTestParams {
192                    test_name: &name,
193                    logger: &logger,
194                    post_test_hooks: &mut post_test_hooks,
195                },
196                &artifacts,
197            )
198        }));
199        let r = r.unwrap_or_else(|err| {
200            // The error from `catch_unwind` is almost always either a
201            // `&str` or a `String`, since that's what `panic!` produces.
202            let msg = err
203                .downcast_ref::<&str>()
204                .copied()
205                .or_else(|| err.downcast_ref::<String>().map(|x| x.as_str()));
206
207            let err = if let Some(msg) = msg {
208                anyhow::anyhow!("test panicked: {msg}")
209            } else {
210                anyhow::anyhow!("test panicked (unknown payload type)")
211            };
212            Err(err)
213        });
214        logger.log_test_result(&r, self.test.0.unstable().is_some());
215
216        for hook in post_test_hooks {
217            tracing::info!(name = hook.name(), "Running post-test hook");
218            if let Err(e) = hook.run(r.is_ok()) {
219                tracing::error!(
220                    error = e.as_ref() as &dyn std::error::Error,
221                    "Post-test hook failed"
222                );
223            } else {
224                tracing::info!("Post-test hook completed successfully");
225            }
226        }
227
228        r
229    }
230
231    /// Returns a libtest-mimic trial to run the test.
232    fn trial(
233        self,
234        resolve: fn(&str, TestArtifactRequirements) -> anyhow::Result<TestArtifacts>,
235    ) -> libtest_mimic::Trial {
236        libtest_mimic::Trial::test(self.name(), move || {
237            let unstable = self.test.0.unstable();
238            match self.run(resolve) {
239                Ok(()) => Ok(()),
240                Err(err) => {
241                    let Some(reason) = unstable else {
242                        return Err(format!("{err:#}").into());
243                    };
244                    if std::env::var("PETRI_IGNORE_UNSTABLE_FAILURES")
245                        .ok()
246                        .is_some_and(|v| !v.is_empty() && v != "0")
247                    {
248                        tracing::warn!(reason, "ignoring unstable test failure: {err:#}");
249                        return Ok(());
250                    }
251                    Err(format!("unstable test failed (reason: {reason}): {err:#}").into())
252                }
253            }
254        })
255    }
256}
257
258/// A test that can be run.
259///
260/// Register it to be run with [`test!`] or [`multitest!`].
261pub trait RunTest: Send {
262    /// The type of artifacts required by the test.
263    type Artifacts;
264
265    /// The leaf name of the test.
266    ///
267    /// To produce the full test name, this will be prefixed with the module
268    /// name where the test is defined.
269    fn leaf_name(&self) -> &str;
270    /// Returns the artifacts required by the test.
271    ///
272    /// Returns `None` if this test makes no sense for this host environment
273    /// (e.g., an x86_64 test on an aarch64 host) and should be left out of the
274    /// test list.
275    fn resolve(&self, resolver: ArtifactResolver<'_>) -> Option<Self::Artifacts>;
276    /// Runs the test, which has been assigned `name`, with the given
277    /// `artifacts`.
278    fn run(&self, params: PetriTestParams<'_>, artifacts: Self::Artifacts) -> anyhow::Result<()>;
279    /// Returns the host requirements of the current test, if any.
280    fn host_requirements(&self) -> Option<&TestCaseRequirements>;
281    /// If this test is unstable, the reason why; `None` if stable.
282    fn unstable(&self) -> Option<&str>;
283    /// Whether this test is ignored (skipped by default, like a libtest
284    /// `#[ignore]` test).
285    fn ignored(&self) -> bool;
286}
287
288trait DynRunTest: Send {
289    fn leaf_name(&self) -> &str;
290    fn artifact_requirements(&self) -> Option<TestArtifactRequirements>;
291    fn run(&self, params: PetriTestParams<'_>, artifacts: &TestArtifacts) -> anyhow::Result<()>;
292    fn host_requirements(&self) -> Option<&TestCaseRequirements>;
293    fn unstable(&self) -> Option<&str>;
294    fn ignored(&self) -> bool;
295}
296
297impl<T: RunTest> DynRunTest for T {
298    fn leaf_name(&self) -> &str {
299        self.leaf_name()
300    }
301
302    fn artifact_requirements(&self) -> Option<TestArtifactRequirements> {
303        let mut requirements = TestArtifactRequirements::new();
304        self.resolve(ArtifactResolver::collector(&mut requirements))?;
305        Some(requirements)
306    }
307
308    fn run(&self, params: PetriTestParams<'_>, artifacts: &TestArtifacts) -> anyhow::Result<()> {
309        let artifacts = self
310            .resolve(ArtifactResolver::resolver(artifacts))
311            .context("test should have been skipped")?;
312        self.run(params, artifacts)
313    }
314
315    fn host_requirements(&self) -> Option<&TestCaseRequirements> {
316        self.host_requirements()
317    }
318
319    fn unstable(&self) -> Option<&str> {
320        self.unstable()
321    }
322
323    fn ignored(&self) -> bool {
324        self.ignored()
325    }
326}
327
328/// Parameters passed to a [`RunTest`] when it is run.
329pub struct PetriTestParams<'a> {
330    /// The name of the running test.
331    pub test_name: &'a str,
332    /// The logger for the test.
333    pub logger: &'a PetriLogSource,
334    /// Any hooks that want to run after the test completes.
335    pub post_test_hooks: &'a mut Vec<PetriPostTestHook>,
336}
337
338/// A post-test hook to be run after the test completes, regardless of if it
339/// succeeds or fails.
340pub struct PetriPostTestHook {
341    /// The name of the hook.
342    name: String,
343    /// The hook function.
344    hook: Box<dyn FnOnce(bool) -> anyhow::Result<()>>,
345}
346
347impl PetriPostTestHook {
348    pub fn new(name: String, hook: impl FnOnce(bool) -> anyhow::Result<()> + 'static) -> Self {
349        Self {
350            name,
351            hook: Box::new(hook),
352        }
353    }
354
355    pub fn name(&self) -> &str {
356        &self.name
357    }
358
359    pub fn run(self, test_passed: bool) -> anyhow::Result<()> {
360        (self.hook)(test_passed)
361    }
362}
363
364/// Returns a hook that, if the test failed, writes the Windows Error Reporting
365/// and Azure Watson events from the test's execution window to
366/// `watson_events.log`.
367#[cfg(windows)]
368fn collect_watson_events_hook(logger: PetriLogSource) -> PetriPostTestHook {
369    let start_time = jiff::Timestamp::now();
370    PetriPostTestHook::new("collect watson events".into(), move |test_passed| {
371        if test_passed {
372            return Ok(());
373        }
374        let events =
375            futures::executor::block_on(crate::vm::hyperv::powershell::watson_events(&start_time));
376        if events.is_empty() {
377            return Ok(());
378        }
379        let log_file = logger.log_file("watson_events")?;
380        for event in events {
381            event.write_to(&log_file);
382        }
383        Ok(())
384    })
385}
386
387/// A test defined by an artifact resolver function and a run function.
388pub struct SimpleTest<A, F> {
389    leaf_name: &'static str,
390    resolve: A,
391    run: F,
392    /// Optional test requirements
393    pub host_requirements: Option<TestCaseRequirements>,
394    unstable: Option<&'static str>,
395    ignored: bool,
396    remote_policy: RemoteAccess,
397}
398
399impl<A, AR, F, E> SimpleTest<A, F>
400where
401    A: 'static + Send + Fn(&ArtifactResolver<'_>) -> Option<AR>,
402    F: 'static + Send + AsyncFn(PetriTestParams<'_>, DefaultDriver, AR) -> Result<(), E>,
403    E: Into<anyhow::Error>,
404{
405    /// Returns a new test with the given `leaf_name`, `resolve`, and `run`
406    /// functions, whose `run` function is async, running it on a task
407    /// pool owned by petri.
408    ///
409    /// The test defaults to stable, not ignored, with no host requirements and
410    /// a [`RemoteAccess::LocalOnly`] policy. Use the builder methods
411    /// ([`requirements`](Self::requirements), [`unstable`](Self::unstable),
412    /// [`ignore`](Self::ignore), [`remote_access`](Self::remote_access)) to
413    /// override these.
414    pub fn new_async(
415        leaf_name: &'static str,
416        resolve: A,
417        run: F,
418    ) -> SimpleTest<A, impl 'static + Send + Fn(PetriTestParams<'_>, AR) -> Result<(), E>> {
419        SimpleTest::new_sync(leaf_name, resolve, move |params, artifacts| {
420            let mut pool = DefaultPool::named(std::thread::current().name().unwrap_or(leaf_name));
421            let driver = pool.driver();
422            // The inner catch keeps a panicking test body from unwinding through
423            // the pool; the outer one catches panics raised by spawned tasks,
424            // such as the VM's timeout watchdog.
425            let r = catch_unwind(AssertUnwindSafe(|| {
426                pool.run_until(
427                    AssertUnwindSafe(run(params, driver.clone(), artifacts)).catch_unwind(),
428                )
429            }));
430            // Let the diagnostic tasks the VM spawns as it is dropped finish
431            // before the failure is reported.
432            drop(driver);
433            pool.run();
434            match r.and_then(|r| r) {
435                Ok(r) => r,
436                Err(panic) => std::panic::resume_unwind(panic),
437            }
438        })
439    }
440}
441
442impl<A, AR, F, E> SimpleTest<A, F>
443where
444    A: 'static + Send + Fn(&ArtifactResolver<'_>) -> Option<AR>,
445    F: 'static + Send + Fn(PetriTestParams<'_>, AR) -> Result<(), E>,
446    E: Into<anyhow::Error>,
447{
448    /// Returns a new test with the given `leaf_name`, `resolve`, and `run`
449    /// functions.
450    ///
451    /// Use [`new_async`](Self::new_async) instead if the test constructs a
452    /// [`PetriVm`](crate::PetriVm); its diagnostics on failure rely on petri
453    /// owning the task pool.
454    ///
455    /// The test defaults to stable, not ignored, with no host requirements and
456    /// a [`RemoteAccess::LocalOnly`] policy. Use the builder methods
457    /// ([`requirements`](Self::requirements), [`unstable`](Self::unstable),
458    /// [`ignore`](Self::ignore), [`remote_access`](Self::remote_access)) to
459    /// override these.
460    pub fn new_sync(leaf_name: &'static str, resolve: A, run: F) -> Self {
461        SimpleTest {
462            leaf_name,
463            resolve,
464            run,
465            host_requirements: None,
466            unstable: None,
467            ignored: false,
468            remote_policy: RemoteAccess::LocalOnly,
469        }
470    }
471}
472
473impl<A, F> SimpleTest<A, F> {
474    /// Sets the host requirements that must be satisfied for this test to run.
475    pub fn requirements(mut self, requirements: TestCaseRequirements) -> Self {
476        self.host_requirements = Some(requirements);
477        self
478    }
479
480    /// Marks this test as unstable. When `PETRI_IGNORE_UNSTABLE_FAILURES` is
481    /// set (as it is in CI), a failure of this test is logged and ignored
482    /// rather than failing the run; otherwise it fails like any other test.
483    ///
484    /// `reason` documents why the test is unstable and is logged when an
485    /// unstable failure is ignored.
486    pub fn unstable(mut self, reason: &'static str) -> Self {
487        self.unstable = Some(reason);
488        self
489    }
490
491    /// Marks this test as ignored: it is skipped by default and only runs when
492    /// explicitly requested (like a libtest `#[ignore]` test).
493    pub fn ignore(mut self) -> Self {
494        self.ignored = true;
495        self
496    }
497
498    /// Sets the remote-access policy used when resolving artifacts.
499    pub fn remote_access(mut self, policy: RemoteAccess) -> Self {
500        self.remote_policy = policy;
501        self
502    }
503}
504
505impl<A, AR, F, E> RunTest for SimpleTest<A, F>
506where
507    A: 'static + Send + Fn(&ArtifactResolver<'_>) -> Option<AR>,
508    F: 'static + Send + Fn(PetriTestParams<'_>, AR) -> Result<(), E>,
509    E: Into<anyhow::Error>,
510{
511    type Artifacts = AR;
512
513    fn leaf_name(&self) -> &str {
514        self.leaf_name
515    }
516
517    fn resolve(&self, mut resolver: ArtifactResolver<'_>) -> Option<Self::Artifacts> {
518        resolver.set_remote_policy(self.remote_policy);
519        (self.resolve)(&resolver)
520    }
521
522    fn run(&self, params: PetriTestParams<'_>, artifacts: Self::Artifacts) -> anyhow::Result<()> {
523        (self.run)(params, artifacts).map_err(Into::into)
524    }
525
526    fn host_requirements(&self) -> Option<&TestCaseRequirements> {
527        self.host_requirements.as_ref()
528    }
529
530    fn unstable(&self) -> Option<&str> {
531        self.unstable
532    }
533
534    fn ignored(&self) -> bool {
535        self.ignored
536    }
537}
538
539#[derive(clap::Parser)]
540struct Options {
541    /// Lists the required artifacts for all tests in JSON format.
542    /// Use --tests-from-stdin to query artifacts for specific tests.
543    #[clap(long)]
544    list_required_artifacts: bool,
545    /// When used with --list-required-artifacts, read exact test names from
546    /// stdin (one per line) to query artifacts for specific tests only.
547    ///
548    /// Even though users can use nextest's filter logic to run a subset of
549    /// tests, due to nextest's architecture of running one test per binary we
550    /// cannot accept a nextest filter here directly. Instead, vmm-tests-run
551    /// must first call nextest with the desired filter to determine the exact
552    /// test names to pass via stdin, then ask petri what artifacts are required
553    /// for those tests.
554    #[clap(long, requires = "list_required_artifacts")]
555    tests_from_stdin: bool,
556    #[clap(flatten)]
557    inner: libtest_mimic::Arguments,
558}
559
560/// Entry point for test binaries.
561pub fn test_main(
562    resolve: fn(&str, TestArtifactRequirements) -> anyhow::Result<TestArtifacts>,
563) -> ! {
564    let mut args = <Options as clap::Parser>::parse();
565    if args.list_required_artifacts {
566        use std::collections::BTreeSet;
567
568        // Collect all artifacts from tests (all tests, or those specified via stdin)
569        let mut required_set = BTreeSet::new();
570        let mut optional_set = BTreeSet::new();
571
572        // If reading test names from stdin, collect them into a set for exact matching
573        let stdin_tests: Option<BTreeSet<String>> = if args.tests_from_stdin {
574            use std::io::BufRead;
575            let stdin = std::io::stdin();
576            let tests: BTreeSet<String> = stdin
577                .lock()
578                .lines()
579                .map_while(Result::ok)
580                .filter(|line| !line.is_empty())
581                .collect();
582            if tests.is_empty() {
583                eprintln!("warning: no test names provided on stdin");
584            }
585            Some(tests)
586        } else {
587            None
588        };
589
590        for test in Test::all() {
591            let name = test.name();
592
593            // If reading from stdin, do exact matching; otherwise include all tests
594            let matches = match stdin_tests {
595                Some(ref stdin_tests) => stdin_tests.contains(&name),
596                None => true,
597            };
598
599            if matches {
600                for artifact in test.artifact_requirements.required_artifacts() {
601                    required_set.insert(artifact.global_unique_id());
602                }
603                for artifact in test.artifact_requirements.optional_artifacts() {
604                    optional_set.insert(artifact.global_unique_id());
605                }
606            }
607        }
608
609        // Remove from optional any artifacts that are required
610        let optional_set: BTreeSet<_> = optional_set.difference(&required_set).cloned().collect();
611
612        let output = petri_artifacts_core::ArtifactListOutput {
613            required: required_set.into_iter().collect(),
614            optional: optional_set.into_iter().collect(),
615        };
616
617        println!(
618            "{}",
619            serde_json::to_string(&output).expect("JSON serialization failed")
620        );
621        std::process::exit(0);
622    }
623
624    // Always just use one thread to avoid interleaving logs and to avoid using
625    // too many resources. These tests are usually run under nextest, which will
626    // run them in parallel in separate processes with appropriate concurrency
627    // limits.
628    if !matches!(args.inner.test_threads, None | Some(1)) {
629        eprintln!("warning: ignoring value passed to --test-threads, using 1");
630    }
631    args.inner.test_threads = Some(1);
632
633    // Create the host context once to avoid repeated expensive queries
634    let host_context = futures::executor::block_on(HostContext::new());
635
636    let trials = Test::all()
637        .map(|test| {
638            let can_run = can_run_test_with_context(test.test.0.host_requirements(), &host_context);
639            let ignored = test.test.0.ignored();
640            test.trial(resolve).with_ignored_flag(!can_run || ignored)
641        })
642        .collect();
643
644    libtest_mimic::run(&args.inner, trials).exit();
645}
petri/test.rs

petri/
test.rs
