petri/
test.rs

1// Copyright (c) Microsoft Corporation.
2// Licensed under the MIT License.
3
4//! Infrastructure for defining tests.
5
6#[doc(hidden)]
7pub mod test_macro_support {
8    use super::TestCase;
9    pub use linkme;
10
11    #[linkme::distributed_slice]
12    pub static TESTS: [fn() -> (&'static str, Vec<TestCase>)];
13}
14
15use crate::PetriLogSource;
16use crate::TestArtifactRequirements;
17use crate::TestArtifacts;
18use crate::tracing::try_init_tracing;
19use anyhow::Context as _;
20use petri_artifacts_core::ArtifactResolver;
21use std::panic::AssertUnwindSafe;
22use std::panic::catch_unwind;
23use std::path::Path;
24use test_macro_support::TESTS;
25
26/// Defines a single test from a value that implements [`RunTest`].
27#[macro_export]
28macro_rules! test {
29    ($f:ident, $req:expr) => {
30        $crate::multitest!(vec![
31            $crate::SimpleTest::new(stringify!($f), $req, $f).into()
32        ]);
33    };
34}
35
36/// Defines a set of tests from a [`TestCase`].
37#[macro_export]
38macro_rules! multitest {
39    ($tests:expr) => {
40        const _: () = {
41            use $crate::test_macro_support::linkme;
42            #[linkme::distributed_slice($crate::test_macro_support::TESTS)]
43            #[linkme(crate = linkme)]
44            static TEST: fn() -> (&'static str, Vec<$crate::TestCase>) =
45                || (module_path!(), $tests);
46        };
47    };
48}
49
50/// A single test case.
51pub struct TestCase(Box<dyn DynRunTest>);
52
53impl TestCase {
54    /// Creates a new test case from a value that implements [`RunTest`].
55    pub fn new(test: impl 'static + RunTest) -> Self {
56        Self(Box::new(test))
57    }
58}
59
60impl<T: 'static + RunTest> From<T> for TestCase {
61    fn from(test: T) -> Self {
62        Self::new(test)
63    }
64}
65
66/// A single test, with module name.
67struct Test {
68    module: &'static str,
69    test: TestCase,
70    requirements: TestArtifactRequirements,
71}
72
73impl Test {
74    /// Returns all the tests defined in this crate.
75    fn all() -> impl Iterator<Item = Self> {
76        TESTS.iter().flat_map(|f| {
77            let (module, tests) = f();
78            tests.into_iter().filter_map(move |test| {
79                let mut requirements = test.0.requirements()?;
80                // All tests require the log directory.
81                requirements.require(petri_artifacts_common::artifacts::TEST_LOG_DIRECTORY);
82                Some(Self {
83                    module,
84                    requirements,
85                    test,
86                })
87            })
88        })
89    }
90
91    /// Returns the name of the test.
92    fn name(&self) -> String {
93        // Strip the crate name from the module path, for consistency with libtest.
94        match self.module.split_once("::") {
95            Some((_crate_name, rest)) => format!("{}::{}", rest, self.test.0.leaf_name()),
96            None => self.test.0.leaf_name().to_owned(),
97        }
98    }
99
100    fn run(
101        &self,
102        resolve: fn(&str, TestArtifactRequirements) -> anyhow::Result<TestArtifacts>,
103    ) -> anyhow::Result<()> {
104        let name = self.name();
105        let artifacts =
106            resolve(&name, self.requirements.clone()).context("failed to resolve artifacts")?;
107        let output_dir = artifacts.get(petri_artifacts_common::artifacts::TEST_LOG_DIRECTORY);
108        let logger = try_init_tracing(output_dir).context("failed to initialize tracing")?;
109
110        // Catch test panics in order to cleanly log the panic result. Without
111        // this, `libtest_mimic` will report the panic to stdout and fail the
112        // test, but the details won't end up in our per-test JSON log.
113        let r = catch_unwind(AssertUnwindSafe(|| {
114            self.test.0.run(
115                PetriTestParams {
116                    test_name: &name,
117                    logger: &logger,
118                    output_dir,
119                },
120                &artifacts,
121            )
122        }));
123        let r = r.unwrap_or_else(|err| {
124            // The error from `catch_unwind` is almost always either a
125            // `&str` or a `String`, since that's what `panic!` produces.
126            let msg = err
127                .downcast_ref::<&str>()
128                .copied()
129                .or_else(|| err.downcast_ref::<String>().map(|x| x.as_str()));
130
131            let err = if let Some(msg) = msg {
132                anyhow::anyhow!("test panicked: {msg}")
133            } else {
134                anyhow::anyhow!("test panicked (unknown payload type)")
135            };
136            Err(err)
137        });
138        let result_path = match &r {
139            Ok(()) => {
140                tracing::info!("test passed");
141                "petri.passed"
142            }
143            Err(err) => {
144                tracing::error!(
145                    error = err.as_ref() as &dyn std::error::Error,
146                    "test failed"
147                );
148                "petri.failed"
149            }
150        };
151        // Write a file to the output directory to indicate whether the test
152        // passed, for easy scanning via tools.
153        fs_err::write(output_dir.join(result_path), &name).unwrap();
154        r
155    }
156
157    /// Returns a libtest-mimic trial to run the test.
158    fn trial(
159        self,
160        resolve: fn(&str, TestArtifactRequirements) -> anyhow::Result<TestArtifacts>,
161    ) -> libtest_mimic::Trial {
162        libtest_mimic::Trial::test(self.name(), move || {
163            self.run(resolve).map_err(|err| format!("{err:#}").into())
164        })
165    }
166}
167
168/// A test that can be run.
169///
170/// Register it to be run with [`test!`] or [`multitest!`].
171pub trait RunTest: Send {
172    /// The type of artifacts required by the test.
173    type Artifacts;
174
175    /// The leaf name of the test.
176    ///
177    /// To produce the full test name, this will be prefixed with the module
178    /// name where the test is defined.
179    fn leaf_name(&self) -> &str;
180    /// Returns the artifacts required by the test.
181    ///
182    /// Returns `None` if this test makes no sense for this host environment
183    /// (e.g., an x86_64 test on an aarch64 host) and should be left out of the
184    /// test list.
185    fn resolve(&self, resolver: &ArtifactResolver<'_>) -> Option<Self::Artifacts>;
186    /// Runs the test, which has been assigned `name`, with the given
187    /// `artifacts`.
188    fn run(&self, params: PetriTestParams<'_>, artifacts: Self::Artifacts) -> anyhow::Result<()>;
189}
190
191trait DynRunTest: Send {
192    fn leaf_name(&self) -> &str;
193    fn requirements(&self) -> Option<TestArtifactRequirements>;
194    fn run(&self, params: PetriTestParams<'_>, artifacts: &TestArtifacts) -> anyhow::Result<()>;
195}
196
197impl<T: RunTest> DynRunTest for T {
198    fn leaf_name(&self) -> &str {
199        self.leaf_name()
200    }
201
202    fn requirements(&self) -> Option<TestArtifactRequirements> {
203        let mut requirements = TestArtifactRequirements::new();
204        self.resolve(&ArtifactResolver::collector(&mut requirements))?;
205        Some(requirements)
206    }
207
208    fn run(&self, params: PetriTestParams<'_>, artifacts: &TestArtifacts) -> anyhow::Result<()> {
209        let artifacts = self
210            .resolve(&ArtifactResolver::resolver(artifacts))
211            .context("test should have been skipped")?;
212        self.run(params, artifacts)
213    }
214}
215
216/// Parameters passed to a [`RunTest`] when it is run.
217#[non_exhaustive]
218pub struct PetriTestParams<'a> {
219    /// The name of the running test.
220    pub test_name: &'a str,
221    /// The logger for the test.
222    pub logger: &'a PetriLogSource,
223    /// The test output directory.
224    pub output_dir: &'a Path,
225}
226
227/// A test defined by an artifact resolver function and a run function.
228pub struct SimpleTest<A, F> {
229    leaf_name: &'static str,
230    resolve: A,
231    run: F,
232}
233
234impl<A, AR, F, E> SimpleTest<A, F>
235where
236    A: 'static + Send + Fn(&ArtifactResolver<'_>) -> Option<AR>,
237    F: 'static + Send + Fn(PetriTestParams<'_>, AR) -> Result<(), E>,
238    E: Into<anyhow::Error>,
239{
240    /// Returns a new test with the given `leaf_name`, `resolve`, and `run`
241    /// functions.
242    pub fn new(leaf_name: &'static str, resolve: A, run: F) -> Self {
243        SimpleTest {
244            leaf_name,
245            resolve,
246            run,
247        }
248    }
249}
250
251impl<A, AR, F, E> RunTest for SimpleTest<A, F>
252where
253    A: 'static + Send + Fn(&ArtifactResolver<'_>) -> Option<AR>,
254    F: 'static + Send + Fn(PetriTestParams<'_>, AR) -> Result<(), E>,
255    E: Into<anyhow::Error>,
256{
257    type Artifacts = AR;
258
259    fn leaf_name(&self) -> &str {
260        self.leaf_name
261    }
262
263    fn resolve(&self, resolver: &ArtifactResolver<'_>) -> Option<Self::Artifacts> {
264        (self.resolve)(resolver)
265    }
266
267    fn run(&self, params: PetriTestParams<'_>, artifacts: Self::Artifacts) -> anyhow::Result<()> {
268        (self.run)(params, artifacts).map_err(Into::into)
269    }
270}
271
272#[derive(clap::Parser)]
273struct Options {
274    /// Lists the required artifacts for all tests.
275    #[clap(long)]
276    list_required_artifacts: bool,
277    #[clap(flatten)]
278    inner: libtest_mimic::Arguments,
279}
280
281/// Entry point for test binaries.
282pub fn test_main(
283    resolve: fn(&str, TestArtifactRequirements) -> anyhow::Result<TestArtifacts>,
284) -> ! {
285    let mut args = <Options as clap::Parser>::parse();
286    if args.list_required_artifacts {
287        // FUTURE: write this in a machine readable format.
288        for test in Test::all() {
289            println!("{}:", test.name());
290            for artifact in test.requirements.required_artifacts() {
291                println!("required: {artifact:?}");
292            }
293            for artifact in test.requirements.optional_artifacts() {
294                println!("optional: {artifact:?}");
295            }
296            println!();
297        }
298        std::process::exit(0);
299    }
300
301    // Always just use one thread to avoid interleaving logs and to avoid using
302    // too many resources. These tests are usually run under nextest, which will
303    // run them in parallel in separate processes with appropriate concurrency
304    // limits.
305    if !matches!(args.inner.test_threads, None | Some(1)) {
306        eprintln!("warning: ignoring value passed to --test-threads, using 1");
307    }
308    args.inner.test_threads = Some(1);
309
310    let trials = Test::all().map(|test| test.trial(resolve)).collect();
311    libtest_mimic::run(&args.inner, trials).exit()
312}
petri/test.rs

petri/
test.rs
