flowey_lib_common/
run_cargo_nextest_run.rs

1// Copyright (c) Microsoft Corporation.
2// Licensed under the MIT License.
3
4//! Run cargo-nextest tests.
5
6use crate::gen_cargo_nextest_run_cmd::RunKindDeps;
7use flowey::node::prelude::*;
8use std::collections::BTreeMap;
9#[derive(Serialize, Deserialize)]
10pub struct TestResults {
11    pub all_tests_passed: bool,
12    /// Path to JUnit XML output (if enabled by the nextest profile)
13    pub junit_xml: Option<PathBuf>,
14}
15
16/// Parameters related to building nextest tests
17pub mod build_params {
18    use crate::run_cargo_build::CargoBuildProfile;
19    use flowey::node::prelude::*;
20    use std::collections::BTreeMap;
21
22    #[derive(Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord, Debug)]
23    pub enum PanicAbortTests {
24        /// Assume the current rust toolchain is nightly
25        // FUTURE: current flowey infrastructure doesn't actually have a path for
26        // multi-toolchain drifting
27        UsingNightly,
28        /// Build with `RUSTC_BOOTSTRAP=1` set
29        UsingRustcBootstrap,
30    }
31
32    #[derive(Serialize, Deserialize)]
33    pub enum FeatureSet {
34        All,
35        Specific(Vec<String>),
36    }
37
38    /// Types of things that can be documented
39    #[derive(Serialize, Deserialize)]
40    pub enum TestPackages {
41        /// Document an entire workspace workspace (with exclusions)
42        Workspace {
43            /// Exclude certain crates
44            exclude: Vec<String>,
45        },
46        /// Document a specific set of crates.
47        Crates {
48            /// Crates to document
49            crates: Vec<String>,
50        },
51    }
52
53    #[derive(Serialize, Deserialize)]
54    pub struct NextestBuildParams<C = VarNotClaimed> {
55        /// Packages to test for
56        pub packages: ReadVar<TestPackages, C>,
57        /// Cargo features to enable when building
58        pub features: FeatureSet,
59        /// Whether to disable default features
60        pub no_default_features: bool,
61        /// Whether to build tests with unstable `-Zpanic-abort-tests` flag
62        pub unstable_panic_abort_tests: Option<PanicAbortTests>,
63        /// Build tests for the specified target
64        pub target: target_lexicon::Triple,
65        /// Build tests with the specified cargo profile
66        pub profile: CargoBuildProfile,
67        /// Additional env vars set when building the tests
68        pub extra_env: ReadVar<BTreeMap<String, String>, C>,
69    }
70}
71
72/// Nextest run mode to use
73#[derive(Serialize, Deserialize)]
74pub enum NextestRunKind {
75    /// Build and run tests in a single step.
76    BuildAndRun(build_params::NextestBuildParams),
77    /// Run tests from pre-built nextest archive file.
78    RunFromArchive {
79        archive_file: ReadVar<PathBuf>,
80        target: Option<ReadVar<target_lexicon::Triple>>,
81        nextest_bin: Option<ReadVar<PathBuf>>,
82    },
83}
84
85#[derive(Serialize, Deserialize)]
86pub struct Run {
87    /// Friendly name for this test group that will be displayed in logs.
88    pub friendly_name: String,
89    /// What kind of test run this is (inline build vs. from nextest archive).
90    pub run_kind: NextestRunKind,
91    /// Working directory the test archive was created from.
92    pub working_dir: ReadVar<PathBuf>,
93    /// Path to `.config/nextest.toml`
94    pub config_file: ReadVar<PathBuf>,
95    /// Path to any tool-specific config files
96    pub tool_config_files: Vec<(String, ReadVar<PathBuf>)>,
97    /// Nextest profile to use when running the source code (as defined in the
98    /// `.config.nextest.toml`).
99    pub nextest_profile: String,
100    /// Nextest test filter expression
101    pub nextest_filter_expr: Option<String>,
102    /// Whether to run ignored tests
103    pub run_ignored: bool,
104    /// Set rlimits to allow unlimited sized coredump file (if supported)
105    pub with_rlimit_unlimited_core_size: bool,
106    /// Additional env vars set when executing the tests.
107    pub extra_env: Option<ReadVar<BTreeMap<String, String>>>,
108    /// Wait for specified side-effects to resolve before building / running any
109    /// tests. (e.g: to allow for some ambient packages / dependencies to
110    /// get installed).
111    pub pre_run_deps: Vec<ReadVar<SideEffect>>,
112    /// Results of running the tests
113    pub results: WriteVar<TestResults>,
114}
115
116flowey_request! {
117    pub enum Request {
118        /// Set the default nextest fast fail behavior. Defaults to not
119        /// fast-failing when a single test fails.
120        DefaultNextestFailFast(bool),
121        /// Set the default behavior when a test failure is encountered.
122        /// Defaults to not terminating the job when a single test fails.
123        DefaultTerminateJobOnFail(bool),
124        Run(Run),
125    }
126}
127
128new_flow_node!(struct Node);
129
130impl FlowNode for Node {
131    type Request = Request;
132
133    fn imports(ctx: &mut ImportCtx<'_>) {
134        ctx.import::<crate::cfg_cargo_common_flags::Node>();
135        ctx.import::<crate::download_cargo_nextest::Node>();
136        ctx.import::<crate::install_cargo_nextest::Node>();
137        ctx.import::<crate::install_rust::Node>();
138        ctx.import::<crate::gen_cargo_nextest_run_cmd::Node>();
139    }
140
141    fn emit(requests: Vec<Self::Request>, ctx: &mut NodeCtx<'_>) -> anyhow::Result<()> {
142        let mut run = Vec::new();
143        let mut fail_fast = None;
144        let mut terminate_job_on_fail = None;
145
146        for req in requests {
147            match req {
148                Request::DefaultNextestFailFast(v) => {
149                    same_across_all_reqs("OverrideFailFast", &mut fail_fast, v)?
150                }
151                Request::DefaultTerminateJobOnFail(v) => {
152                    same_across_all_reqs("TerminateJobOnFail", &mut terminate_job_on_fail, v)?
153                }
154                Request::Run(v) => run.push(v),
155            }
156        }
157
158        let terminate_job_on_fail = terminate_job_on_fail.unwrap_or(false);
159
160        for Run {
161            friendly_name,
162            run_kind,
163            working_dir,
164            config_file,
165            tool_config_files,
166            nextest_profile,
167            extra_env,
168            with_rlimit_unlimited_core_size,
169            nextest_filter_expr,
170            run_ignored,
171            pre_run_deps,
172            results,
173        } in run
174        {
175            let run_kind_deps = match run_kind {
176                NextestRunKind::BuildAndRun(params) => {
177                    let cargo_flags = ctx.reqv(crate::cfg_cargo_common_flags::Request::GetFlags);
178
179                    let nextest_installed = ctx.reqv(crate::install_cargo_nextest::Request);
180
181                    let rust_toolchain = ctx.reqv(crate::install_rust::Request::GetRustupToolchain);
182
183                    ctx.req(crate::install_rust::Request::InstallTargetTriple(
184                        params.target.clone(),
185                    ));
186
187                    RunKindDeps::BuildAndRun {
188                        params,
189                        nextest_installed,
190                        rust_toolchain,
191                        cargo_flags,
192                    }
193                }
194                NextestRunKind::RunFromArchive {
195                    archive_file,
196                    target,
197                    nextest_bin,
198                } => {
199                    let target =
200                        target.unwrap_or(ReadVar::from_static(target_lexicon::Triple::host()));
201
202                    let nextest_bin = nextest_bin.unwrap_or_else(|| {
203                        ctx.reqv(|v| crate::download_cargo_nextest::Request::Get(target.clone(), v))
204                    });
205
206                    RunKindDeps::RunFromArchive {
207                        archive_file,
208                        nextest_bin,
209                        target,
210                    }
211                }
212            };
213
214            let cmd = ctx.reqv(|v| crate::gen_cargo_nextest_run_cmd::Request {
215                run_kind_deps,
216                working_dir: working_dir.clone(),
217                config_file: config_file.clone(),
218                tool_config_files,
219                nextest_profile: nextest_profile.clone(),
220                nextest_filter_expr,
221                run_ignored,
222                fail_fast,
223                extra_env,
224                portable: false,
225                command: v,
226            });
227
228            let (all_tests_passed_read, all_tests_passed_write) = ctx.new_var();
229            let (junit_xml_read, junit_xml_write) = ctx.new_var();
230
231            ctx.emit_rust_step(format!("run '{friendly_name}' nextest tests"), |ctx| {
232                pre_run_deps.claim(ctx);
233
234                let working_dir = working_dir.claim(ctx);
235                let config_file = config_file.claim(ctx);
236                let all_tests_passed_var = all_tests_passed_write.claim(ctx);
237                let junit_xml_write = junit_xml_write.claim(ctx);
238                let cmd = cmd.claim(ctx);
239
240                move |rt| {
241                    let working_dir = rt.read(working_dir);
242                    let config_file = rt.read(config_file);
243                    let cmd = rt.read(cmd);
244
245                    // first things first - determine if junit is supported by
246                    // the profile, and if so, where the output if going to be.
247                    let junit_path = {
248                        let nextest_toml = fs_err::read_to_string(&config_file)?
249                            .parse::<toml_edit::DocumentMut>()
250                            .context("failed to parse nextest.toml")?;
251
252                        let path = Some(&nextest_toml)
253                            .and_then(|i| i.get("profile"))
254                            .and_then(|i| i.get(&nextest_profile))
255                            .and_then(|i| i.get("junit"))
256                            .and_then(|i| i.get("path"));
257
258                        if let Some(path) = path {
259                            let path: PathBuf =
260                                path.as_str().context("malformed nextest.toml")?.into();
261                            Some(path)
262                        } else {
263                            None
264                        }
265                    };
266
267                    // allow unlimited coredump sizes
268                    //
269                    // FUTURE: would be cool if `flowey` had the ability to pass
270                    // around "callbacks" as part of a, which would subsume the
271                    // need to support things like `with_env` and
272                    // `with_rlimit_unlimited_core_size`.
273                    //
274                    // This _should_ be doable using the same sort of mechanism
275                    // that regular flowey Rust-based steps get registered +
276                    // invoked. i.e: the serializable "callback" object is just
277                    // a unique identifier for a set of
278                    // (NodeHandle,callback_idx,requests), which flowey can use
279                    // to "play-through" the specified node in order to get the
280                    // caller a handle to a concrete `Box<dyn Fn...>`.
281                    //
282                    // I suspect there'll need to be some `Any` involved to get
283                    // things to line up... but honestly, this seems doable?
284                    // Will need to find time to experiment with this...
285                    #[cfg(unix)]
286                    let old_core_rlimits = if with_rlimit_unlimited_core_size
287                        && matches!(rt.platform(), FlowPlatform::Linux(_))
288                    {
289                        let limits = rlimit::getrlimit(rlimit::Resource::CORE)?;
290                        rlimit::setrlimit(
291                            rlimit::Resource::CORE,
292                            rlimit::INFINITY,
293                            rlimit::INFINITY,
294                        )?;
295                        Some(limits)
296                    } else {
297                        None
298                    };
299
300                    #[cfg(not(unix))]
301                    let _ = with_rlimit_unlimited_core_size;
302
303                    log::info!("$ {cmd}");
304
305                    // nextest has meaningful exit codes that we want to parse.
306                    // <https://github.com/nextest-rs/nextest/blob/main/nextest-metadata/src/exit_codes.rs#L12>
307                    //
308                    // unfortunately, xshell doesn't have a mode where it can
309                    // both emit to stdout/stderr, _and_ report the specific
310                    // exit code of the process.
311                    //
312                    // So we have to use the raw process API instead.
313                    let mut command = std::process::Command::new(&cmd.argv0);
314                    command
315                        .args(&cmd.args)
316                        .envs(&cmd.env)
317                        .current_dir(&working_dir);
318
319                    let mut child = command.spawn().with_context(|| {
320                        format!("failed to spawn '{}'", cmd.argv0.to_string_lossy())
321                    })?;
322
323                    let status = child.wait()?;
324
325                    #[cfg(unix)]
326                    if let Some((soft, hard)) = old_core_rlimits {
327                        rlimit::setrlimit(rlimit::Resource::CORE, soft, hard)?;
328                    }
329
330                    let all_tests_passed = match (status.success(), status.code()) {
331                        (true, _) => true,
332                        // documented nextest exit code for when a test has failed
333                        (false, Some(100)) => false,
334                        // any other exit code means something has gone disastrously wrong
335                        (false, _) => anyhow::bail!("failed to run nextest"),
336                    };
337
338                    rt.write(all_tests_passed_var, &all_tests_passed);
339
340                    if !all_tests_passed {
341                        log::warn!("encountered at least one test failure!");
342
343                        if terminate_job_on_fail {
344                            anyhow::bail!("terminating job (TerminateJobOnFail = true)")
345                        } else {
346                            // special string on ADO that causes step to show orange (!)
347                            // FUTURE: flowey should prob have a built-in API for this
348                            if matches!(rt.backend(), FlowBackend::Ado) {
349                                eprintln!("##vso[task.complete result=SucceededWithIssues;]")
350                            } else {
351                                log::warn!("encountered at least one test failure");
352                            }
353                        }
354                    }
355
356                    let junit_xml = if let Some(junit_path) = junit_path {
357                        let emitted_xml = working_dir
358                            .join("target")
359                            .join("nextest")
360                            .join(&nextest_profile)
361                            .join(junit_path);
362                        let final_xml = std::env::current_dir()?.join("junit.xml");
363                        // copy locally to avoid trashing the output between test runs
364                        fs_err::rename(emitted_xml, &final_xml)?;
365                        Some(final_xml.absolute()?)
366                    } else {
367                        None
368                    };
369
370                    rt.write(junit_xml_write, &junit_xml);
371
372                    Ok(())
373                }
374            });
375
376            ctx.emit_minor_rust_step("write results", |ctx| {
377                let all_tests_passed = all_tests_passed_read.claim(ctx);
378                let junit_xml = junit_xml_read.claim(ctx);
379                let results = results.claim(ctx);
380
381                move |rt| {
382                    let all_tests_passed = rt.read(all_tests_passed);
383                    let junit_xml = rt.read(junit_xml);
384
385                    rt.write(
386                        results,
387                        &TestResults {
388                            all_tests_passed,
389                            junit_xml,
390                        },
391                    );
392                }
393            });
394        }
395
396        Ok(())
397    }
398}
399
400// FUTURE: this seems like something a proc-macro can help with...
401impl build_params::NextestBuildParams {
402    pub fn claim(self, ctx: &mut StepCtx<'_>) -> build_params::NextestBuildParams<VarClaimed> {
403        let build_params::NextestBuildParams {
404            packages,
405            features,
406            no_default_features,
407            unstable_panic_abort_tests,
408            target,
409            profile,
410            extra_env,
411        } = self;
412
413        build_params::NextestBuildParams {
414            packages: packages.claim(ctx),
415            features,
416            no_default_features,
417            unstable_panic_abort_tests,
418            target,
419            profile,
420            extra_env: extra_env.claim(ctx),
421        }
422    }
423}