petri/
test.rs

1// Copyright (c) Microsoft Corporation.
2// Licensed under the MIT License.
3
4//! Infrastructure for defining tests.
5
6#[doc(hidden)]
7pub mod test_macro_support {
8    // UNSAFETY: Needed for linkme.
9    #![expect(unsafe_code)]
10
11    use super::TestCase;
12    pub use linkme;
13
14    #[linkme::distributed_slice]
15    pub static TESTS: [Option<fn() -> (&'static str, Vec<TestCase>)>];
16
17    // Always have at least one entry to work around linker bugs.
18    //
19    // See <https://github.com/llvm/llvm-project/issues/65855>.
20    #[linkme::distributed_slice(TESTS)]
21    static WORKAROUND: Option<fn() -> (&'static str, Vec<TestCase>)> = None;
22}
23
24use crate::PetriLogSource;
25use crate::TestArtifactRequirements;
26use crate::TestArtifacts;
27use crate::requirements::HostContext;
28use crate::requirements::TestCaseRequirements;
29use crate::requirements::can_run_test_with_context;
30use crate::tracing::try_init_tracing;
31use anyhow::Context as _;
32use petri_artifacts_core::ArtifactResolver;
33use petri_artifacts_core::RemoteAccess;
34use std::panic::AssertUnwindSafe;
35use std::panic::catch_unwind;
36use test_macro_support::TESTS;
37
38/// Defines a single test from a value that implements [`RunTest`].
39#[macro_export]
40macro_rules! test {
41    ($f:ident, $req:expr) => {
42        $crate::multitest!(vec![
43            $crate::SimpleTest::new(stringify!($f), $req, $f).into()
44        ]);
45    };
46}
47
48/// Defines a single unstable test from a value that implements [`RunTest`].
49///
50/// `$reason` documents why the test is unstable and is logged when an unstable
51/// failure is ignored.
52#[macro_export]
53macro_rules! unstable_test {
54    ($f:ident, $req:expr, $reason:expr) => {
55        $crate::multitest!(vec![
56            $crate::SimpleTest::new(stringify!($f), $req, $f)
57                .unstable($reason)
58                .into()
59        ]);
60    };
61}
62
63/// Defines a set of tests from a [`TestCase`].
64#[macro_export]
65macro_rules! multitest {
66    ($tests:expr) => {
67        const _: () = {
68            use $crate::test_macro_support::linkme;
69            #[linkme::distributed_slice($crate::test_macro_support::TESTS)]
70            #[linkme(crate = linkme)]
71            static TEST: Option<fn() -> (&'static str, Vec<$crate::TestCase>)> =
72                Some(|| (module_path!(), $tests));
73        };
74    };
75}
76
77/// A single test case.
78pub struct TestCase(Box<dyn DynRunTest>);
79
80impl TestCase {
81    /// Creates a new test case from a value that implements [`RunTest`].
82    pub fn new(test: impl 'static + RunTest) -> Self {
83        Self(Box::new(test))
84    }
85}
86
87impl<T: 'static + RunTest> From<T> for TestCase {
88    fn from(test: T) -> Self {
89        Self::new(test)
90    }
91}
92
93/// A single test, with module name.
94struct Test {
95    module: &'static str,
96    test: TestCase,
97    artifact_requirements: TestArtifactRequirements,
98}
99
100impl Test {
101    /// Returns all the tests defined in this crate.
102    fn all() -> impl Iterator<Item = Self> {
103        TESTS.iter().flatten().flat_map(|f| {
104            let (module, tests) = f();
105            tests.into_iter().filter_map(move |test| {
106                let mut artifact_requirements = test.0.artifact_requirements()?;
107                // All tests require the log directory.
108                artifact_requirements.require(
109                    petri_artifacts_common::artifacts::TEST_LOG_DIRECTORY,
110                    RemoteAccess::LocalOnly,
111                    false,
112                );
113                Some(Self {
114                    module,
115                    artifact_requirements,
116                    test,
117                })
118            })
119        })
120    }
121
122    /// Returns the name of the test.
123    fn name(&self) -> String {
124        // Strip the crate name from the module path, for consistency with libtest.
125        match self.module.split_once("::") {
126            Some((_crate_name, rest)) => format!("{}::{}", rest, self.test.0.leaf_name()),
127            None => self.test.0.leaf_name().to_owned(),
128        }
129    }
130
131    fn run(
132        &self,
133        resolve: fn(&str, TestArtifactRequirements) -> anyhow::Result<TestArtifacts>,
134    ) -> anyhow::Result<()> {
135        let name = self.name();
136        let artifacts = resolve(&name, self.artifact_requirements.clone())
137            .context("failed to resolve artifacts")?;
138        let output_dir = artifacts.get(petri_artifacts_common::artifacts::TEST_LOG_DIRECTORY);
139        let logger = try_init_tracing(output_dir, tracing::level_filters::LevelFilter::DEBUG)
140            .context("failed to initialize tracing")?;
141        // Record the test's identity up front, so that a test which is killed
142        // or crashes before reporting a result is still identifiable.
143        logger.log_test_start(&name);
144        let mut post_test_hooks = Vec::new();
145
146        // A process that is faulted or killed writes nothing to its own logs,
147        // so the host's error reporting events are the only record that it
148        // crashed and the only source of the Watson report ID needed to find
149        // the dump.
150        #[cfg(windows)]
151        post_test_hooks.push(collect_watson_events_hook(logger.clone()));
152
153        // Catch test panics in order to cleanly log the panic result. Without
154        // this, `libtest_mimic` will report the panic to stdout and fail the
155        // test, but the details won't end up in our per-test JSON log.
156        let r = catch_unwind(AssertUnwindSafe(|| {
157            self.test.0.run(
158                PetriTestParams {
159                    test_name: &name,
160                    logger: &logger,
161                    post_test_hooks: &mut post_test_hooks,
162                },
163                &artifacts,
164            )
165        }));
166        let r = r.unwrap_or_else(|err| {
167            // The error from `catch_unwind` is almost always either a
168            // `&str` or a `String`, since that's what `panic!` produces.
169            let msg = err
170                .downcast_ref::<&str>()
171                .copied()
172                .or_else(|| err.downcast_ref::<String>().map(|x| x.as_str()));
173
174            let err = if let Some(msg) = msg {
175                anyhow::anyhow!("test panicked: {msg}")
176            } else {
177                anyhow::anyhow!("test panicked (unknown payload type)")
178            };
179            Err(err)
180        });
181        logger.log_test_result(&r, self.test.0.unstable().is_some());
182
183        for hook in post_test_hooks {
184            tracing::info!(name = hook.name(), "Running post-test hook");
185            if let Err(e) = hook.run(r.is_ok()) {
186                tracing::error!(
187                    error = e.as_ref() as &dyn std::error::Error,
188                    "Post-test hook failed"
189                );
190            } else {
191                tracing::info!("Post-test hook completed successfully");
192            }
193        }
194
195        r
196    }
197
198    /// Returns a libtest-mimic trial to run the test.
199    fn trial(
200        self,
201        resolve: fn(&str, TestArtifactRequirements) -> anyhow::Result<TestArtifacts>,
202    ) -> libtest_mimic::Trial {
203        libtest_mimic::Trial::test(self.name(), move || {
204            let unstable = self.test.0.unstable();
205            match self.run(resolve) {
206                Ok(()) => Ok(()),
207                Err(err) => {
208                    let Some(reason) = unstable else {
209                        return Err(format!("{err:#}").into());
210                    };
211                    if std::env::var("PETRI_IGNORE_UNSTABLE_FAILURES")
212                        .ok()
213                        .is_some_and(|v| !v.is_empty() && v != "0")
214                    {
215                        tracing::warn!(reason, "ignoring unstable test failure: {err:#}");
216                        return Ok(());
217                    }
218                    Err(format!("unstable test failed (reason: {reason}): {err:#}").into())
219                }
220            }
221        })
222    }
223}
224
225/// A test that can be run.
226///
227/// Register it to be run with [`test!`] or [`multitest!`].
228pub trait RunTest: Send {
229    /// The type of artifacts required by the test.
230    type Artifacts;
231
232    /// The leaf name of the test.
233    ///
234    /// To produce the full test name, this will be prefixed with the module
235    /// name where the test is defined.
236    fn leaf_name(&self) -> &str;
237    /// Returns the artifacts required by the test.
238    ///
239    /// Returns `None` if this test makes no sense for this host environment
240    /// (e.g., an x86_64 test on an aarch64 host) and should be left out of the
241    /// test list.
242    fn resolve(&self, resolver: ArtifactResolver<'_>) -> Option<Self::Artifacts>;
243    /// Runs the test, which has been assigned `name`, with the given
244    /// `artifacts`.
245    fn run(&self, params: PetriTestParams<'_>, artifacts: Self::Artifacts) -> anyhow::Result<()>;
246    /// Returns the host requirements of the current test, if any.
247    fn host_requirements(&self) -> Option<&TestCaseRequirements>;
248    /// If this test is unstable, the reason why; `None` if stable.
249    fn unstable(&self) -> Option<&str>;
250    /// Whether this test is ignored (skipped by default, like a libtest
251    /// `#[ignore]` test).
252    fn ignored(&self) -> bool;
253}
254
255trait DynRunTest: Send {
256    fn leaf_name(&self) -> &str;
257    fn artifact_requirements(&self) -> Option<TestArtifactRequirements>;
258    fn run(&self, params: PetriTestParams<'_>, artifacts: &TestArtifacts) -> anyhow::Result<()>;
259    fn host_requirements(&self) -> Option<&TestCaseRequirements>;
260    fn unstable(&self) -> Option<&str>;
261    fn ignored(&self) -> bool;
262}
263
264impl<T: RunTest> DynRunTest for T {
265    fn leaf_name(&self) -> &str {
266        self.leaf_name()
267    }
268
269    fn artifact_requirements(&self) -> Option<TestArtifactRequirements> {
270        let mut requirements = TestArtifactRequirements::new();
271        self.resolve(ArtifactResolver::collector(&mut requirements))?;
272        Some(requirements)
273    }
274
275    fn run(&self, params: PetriTestParams<'_>, artifacts: &TestArtifacts) -> anyhow::Result<()> {
276        let artifacts = self
277            .resolve(ArtifactResolver::resolver(artifacts))
278            .context("test should have been skipped")?;
279        self.run(params, artifacts)
280    }
281
282    fn host_requirements(&self) -> Option<&TestCaseRequirements> {
283        self.host_requirements()
284    }
285
286    fn unstable(&self) -> Option<&str> {
287        self.unstable()
288    }
289
290    fn ignored(&self) -> bool {
291        self.ignored()
292    }
293}
294
295/// Parameters passed to a [`RunTest`] when it is run.
296pub struct PetriTestParams<'a> {
297    /// The name of the running test.
298    pub test_name: &'a str,
299    /// The logger for the test.
300    pub logger: &'a PetriLogSource,
301    /// Any hooks that want to run after the test completes.
302    pub post_test_hooks: &'a mut Vec<PetriPostTestHook>,
303}
304
305/// A post-test hook to be run after the test completes, regardless of if it
306/// succeeds or fails.
307pub struct PetriPostTestHook {
308    /// The name of the hook.
309    name: String,
310    /// The hook function.
311    hook: Box<dyn FnOnce(bool) -> anyhow::Result<()>>,
312}
313
314impl PetriPostTestHook {
315    pub fn new(name: String, hook: impl FnOnce(bool) -> anyhow::Result<()> + 'static) -> Self {
316        Self {
317            name,
318            hook: Box::new(hook),
319        }
320    }
321
322    pub fn name(&self) -> &str {
323        &self.name
324    }
325
326    pub fn run(self, test_passed: bool) -> anyhow::Result<()> {
327        (self.hook)(test_passed)
328    }
329}
330
331/// Returns a hook that, if the test failed, writes the Windows Error Reporting
332/// and Azure Watson events from the test's execution window to
333/// `watson_events.log`.
334#[cfg(windows)]
335fn collect_watson_events_hook(logger: PetriLogSource) -> PetriPostTestHook {
336    let start_time = jiff::Timestamp::now();
337    PetriPostTestHook::new("collect watson events".into(), move |test_passed| {
338        if test_passed {
339            return Ok(());
340        }
341        let events =
342            futures::executor::block_on(crate::vm::hyperv::powershell::watson_events(&start_time));
343        if events.is_empty() {
344            return Ok(());
345        }
346        let log_file = logger.log_file("watson_events")?;
347        for event in events {
348            event.write_to(&log_file);
349        }
350        Ok(())
351    })
352}
353
354/// A test defined by an artifact resolver function and a run function.
355pub struct SimpleTest<A, F> {
356    leaf_name: &'static str,
357    resolve: A,
358    run: F,
359    /// Optional test requirements
360    pub host_requirements: Option<TestCaseRequirements>,
361    unstable: Option<&'static str>,
362    ignored: bool,
363    remote_policy: RemoteAccess,
364}
365
366impl<A, AR, F, E> SimpleTest<A, F>
367where
368    A: 'static + Send + Fn(&ArtifactResolver<'_>) -> Option<AR>,
369    F: 'static + Send + Fn(PetriTestParams<'_>, AR) -> Result<(), E>,
370    E: Into<anyhow::Error>,
371{
372    /// Returns a new test with the given `leaf_name`, `resolve`, and `run`
373    /// functions.
374    ///
375    /// The test defaults to stable, not ignored, with no host requirements and
376    /// a [`RemoteAccess::LocalOnly`] policy. Use the builder methods
377    /// ([`requirements`](Self::requirements), [`unstable`](Self::unstable),
378    /// [`ignore`](Self::ignore), [`remote_access`](Self::remote_access)) to
379    /// override these.
380    pub fn new(leaf_name: &'static str, resolve: A, run: F) -> Self {
381        SimpleTest {
382            leaf_name,
383            resolve,
384            run,
385            host_requirements: None,
386            unstable: None,
387            ignored: false,
388            remote_policy: RemoteAccess::LocalOnly,
389        }
390    }
391
392    /// Sets the host requirements that must be satisfied for this test to run.
393    pub fn requirements(mut self, requirements: TestCaseRequirements) -> Self {
394        self.host_requirements = Some(requirements);
395        self
396    }
397
398    /// Marks this test as unstable. When `PETRI_IGNORE_UNSTABLE_FAILURES` is
399    /// set (as it is in CI), a failure of this test is logged and ignored
400    /// rather than failing the run; otherwise it fails like any other test.
401    ///
402    /// `reason` documents why the test is unstable and is logged when an
403    /// unstable failure is ignored.
404    pub fn unstable(mut self, reason: &'static str) -> Self {
405        self.unstable = Some(reason);
406        self
407    }
408
409    /// Marks this test as ignored: it is skipped by default and only runs when
410    /// explicitly requested (like a libtest `#[ignore]` test).
411    pub fn ignore(mut self) -> Self {
412        self.ignored = true;
413        self
414    }
415
416    /// Sets the remote-access policy used when resolving artifacts.
417    pub fn remote_access(mut self, policy: RemoteAccess) -> Self {
418        self.remote_policy = policy;
419        self
420    }
421}
422
423impl<A, AR, F, E> RunTest for SimpleTest<A, F>
424where
425    A: 'static + Send + Fn(&ArtifactResolver<'_>) -> Option<AR>,
426    F: 'static + Send + Fn(PetriTestParams<'_>, AR) -> Result<(), E>,
427    E: Into<anyhow::Error>,
428{
429    type Artifacts = AR;
430
431    fn leaf_name(&self) -> &str {
432        self.leaf_name
433    }
434
435    fn resolve(&self, mut resolver: ArtifactResolver<'_>) -> Option<Self::Artifacts> {
436        resolver.set_remote_policy(self.remote_policy);
437        (self.resolve)(&resolver)
438    }
439
440    fn run(&self, params: PetriTestParams<'_>, artifacts: Self::Artifacts) -> anyhow::Result<()> {
441        (self.run)(params, artifacts).map_err(Into::into)
442    }
443
444    fn host_requirements(&self) -> Option<&TestCaseRequirements> {
445        self.host_requirements.as_ref()
446    }
447
448    fn unstable(&self) -> Option<&str> {
449        self.unstable
450    }
451
452    fn ignored(&self) -> bool {
453        self.ignored
454    }
455}
456
457#[derive(clap::Parser)]
458struct Options {
459    /// Lists the required artifacts for all tests in JSON format.
460    /// Use --tests-from-stdin to query artifacts for specific tests.
461    #[clap(long)]
462    list_required_artifacts: bool,
463    /// When used with --list-required-artifacts, read exact test names from
464    /// stdin (one per line) to query artifacts for specific tests only.
465    ///
466    /// Even though users can use nextest's filter logic to run a subset of
467    /// tests, due to nextest's architecture of running one test per binary we
468    /// cannot accept a nextest filter here directly. Instead, vmm-tests-run
469    /// must first call nextest with the desired filter to determine the exact
470    /// test names to pass via stdin, then ask petri what artifacts are required
471    /// for those tests.
472    #[clap(long, requires = "list_required_artifacts")]
473    tests_from_stdin: bool,
474    #[clap(flatten)]
475    inner: libtest_mimic::Arguments,
476}
477
478/// Entry point for test binaries.
479pub fn test_main(
480    resolve: fn(&str, TestArtifactRequirements) -> anyhow::Result<TestArtifacts>,
481) -> ! {
482    let mut args = <Options as clap::Parser>::parse();
483    if args.list_required_artifacts {
484        use std::collections::BTreeSet;
485
486        // Collect all artifacts from tests (all tests, or those specified via stdin)
487        let mut required_set = BTreeSet::new();
488        let mut optional_set = BTreeSet::new();
489
490        // If reading test names from stdin, collect them into a set for exact matching
491        let stdin_tests: Option<BTreeSet<String>> = if args.tests_from_stdin {
492            use std::io::BufRead;
493            let stdin = std::io::stdin();
494            let tests: BTreeSet<String> = stdin
495                .lock()
496                .lines()
497                .map_while(Result::ok)
498                .filter(|line| !line.is_empty())
499                .collect();
500            if tests.is_empty() {
501                eprintln!("warning: no test names provided on stdin");
502            }
503            Some(tests)
504        } else {
505            None
506        };
507
508        for test in Test::all() {
509            let name = test.name();
510
511            // If reading from stdin, do exact matching; otherwise include all tests
512            let matches = match stdin_tests {
513                Some(ref stdin_tests) => stdin_tests.contains(&name),
514                None => true,
515            };
516
517            if matches {
518                for artifact in test.artifact_requirements.required_artifacts() {
519                    required_set.insert(artifact.global_unique_id());
520                }
521                for artifact in test.artifact_requirements.optional_artifacts() {
522                    optional_set.insert(artifact.global_unique_id());
523                }
524            }
525        }
526
527        // Remove from optional any artifacts that are required
528        let optional_set: BTreeSet<_> = optional_set.difference(&required_set).cloned().collect();
529
530        let output = petri_artifacts_core::ArtifactListOutput {
531            required: required_set.into_iter().collect(),
532            optional: optional_set.into_iter().collect(),
533        };
534
535        println!(
536            "{}",
537            serde_json::to_string(&output).expect("JSON serialization failed")
538        );
539        std::process::exit(0);
540    }
541
542    // Always just use one thread to avoid interleaving logs and to avoid using
543    // too many resources. These tests are usually run under nextest, which will
544    // run them in parallel in separate processes with appropriate concurrency
545    // limits.
546    if !matches!(args.inner.test_threads, None | Some(1)) {
547        eprintln!("warning: ignoring value passed to --test-threads, using 1");
548    }
549    args.inner.test_threads = Some(1);
550
551    // Create the host context once to avoid repeated expensive queries
552    let host_context = futures::executor::block_on(HostContext::new());
553
554    let trials = Test::all()
555        .map(|test| {
556            let can_run = can_run_test_with_context(test.test.0.host_requirements(), &host_context);
557            let ignored = test.test.0.ignored();
558            test.trial(resolve).with_ignored_flag(!can_run || ignored)
559        })
560        .collect();
561
562    libtest_mimic::run(&args.inner, trials).exit();
563}
petri/test.rs

petri/
test.rs
