flowey_core/pipeline.rs
1// Copyright (c) Microsoft Corporation.
2// Licensed under the MIT License.
3
4//! Core types and traits used to create and work with flowey pipelines.
5
6mod artifact;
7
8pub use artifact::Artifact;
9pub use artifact::ArtifactType;
10pub use artifact::resolve;
11
12use self::internal::*;
13use crate::node::FlowArch;
14use crate::node::FlowNodeBase;
15use crate::node::FlowPlatform;
16use crate::node::FlowPlatformLinuxDistro;
17use crate::node::GhUserSecretVar;
18use crate::node::IntoConfig;
19use crate::node::IntoRequest;
20use crate::node::NodeHandle;
21use crate::node::ReadVar;
22use crate::node::WriteVar;
23use crate::node::steps::ado::AdoResourcesRepositoryId;
24use crate::node::user_facing::AdoRuntimeVar;
25use crate::node::user_facing::GhPermission;
26use crate::node::user_facing::GhPermissionValue;
27use crate::patch::PatchResolver;
28use crate::patch::ResolvedPatches;
29use serde::Serialize;
30use serde::de::DeserializeOwned;
31use std::collections::BTreeMap;
32use std::collections::BTreeSet;
33use std::path::PathBuf;
34
35/// Pipeline types which are considered "user facing", and included in the
36/// `flowey` prelude.
37pub mod user_facing {
38 pub use super::AdoCiTriggers;
39 pub use super::AdoPool;
40 pub use super::AdoPrTriggers;
41 pub use super::AdoResourcesRepository;
42 pub use super::AdoResourcesRepositoryRef;
43 pub use super::AdoResourcesRepositoryType;
44 pub use super::AdoScheduleTriggers;
45 pub use super::GhCiTriggers;
46 pub use super::GhConcurrencyGroup;
47 pub use super::GhPrTriggers;
48 pub use super::GhRunner;
49 pub use super::GhRunnerOsLabel;
50 pub use super::GhScheduleTriggers;
51 pub use super::HostExt;
52 pub use super::IntoPipeline;
53 pub use super::ParameterKind;
54 pub use super::Pipeline;
55 pub use super::PipelineBackendHint;
56 pub use super::PipelineJob;
57 pub use super::PipelineJobCtx;
58 pub use super::PipelineJobHandle;
59 pub use super::PublishArtifact;
60 pub use super::PublishTypedArtifact;
61 pub use super::UseArtifact;
62 pub use super::UseParameter;
63 pub use super::UseTypedArtifact;
64 pub use crate::node::FlowArch;
65 pub use crate::node::FlowPlatform;
66}
67
68fn linux_distro() -> FlowPlatformLinuxDistro {
69 // Check for nix environment first - takes precedence over distro detection
70 if std::env::var("IN_NIX_SHELL").is_ok() {
71 return FlowPlatformLinuxDistro::Nix;
72 }
73
74 // A `nix develop` shell doesn't set `IN_NIX_SHELL`, but the PATH should include a nix store path
75 if std::env::var("PATH").is_ok_and(|path| path.contains("/nix/store")) {
76 return FlowPlatformLinuxDistro::Nix;
77 }
78
79 if let Ok(etc_os_release) = fs_err::read_to_string("/etc/os-release") {
80 if etc_os_release.contains("ID=ubuntu") {
81 FlowPlatformLinuxDistro::Ubuntu
82 } else if etc_os_release.contains("ID=fedora") {
83 FlowPlatformLinuxDistro::Fedora
84 } else if etc_os_release.contains("ID=azurelinux") || etc_os_release.contains("ID=mariner")
85 {
86 FlowPlatformLinuxDistro::AzureLinux
87 } else if etc_os_release.contains("ID=arch") {
88 FlowPlatformLinuxDistro::Arch
89 } else if etc_os_release
90 .lines()
91 .find_map(|line| line.strip_prefix("ID="))
92 .is_some_and(|id| id.trim_matches(['"', '\'']).starts_with("opensuse"))
93 {
94 FlowPlatformLinuxDistro::OpenSuse
95 } else {
96 FlowPlatformLinuxDistro::Unknown
97 }
98 } else {
99 FlowPlatformLinuxDistro::Unknown
100 }
101}
102
103pub trait HostExt: Sized {
104 /// Return the value for the current host machine.
105 ///
106 /// Will panic on non-local backends.
107 fn host(backend_hint: PipelineBackendHint) -> Self;
108}
109
110impl HostExt for FlowPlatform {
111 /// Return the platform of the current host machine.
112 ///
113 /// Will panic on non-local backends.
114 fn host(backend_hint: PipelineBackendHint) -> Self {
115 if !matches!(backend_hint, PipelineBackendHint::Local) {
116 panic!("can only use `FlowPlatform::host` when defining a local-only pipeline");
117 }
118
119 if cfg!(target_os = "windows") {
120 Self::Windows
121 } else if cfg!(target_os = "linux") {
122 Self::Linux(linux_distro())
123 } else if cfg!(target_os = "macos") {
124 Self::MacOs
125 } else {
126 panic!("no valid host-os")
127 }
128 }
129}
130
131impl HostExt for FlowArch {
132 /// Return the arch of the current host machine.
133 ///
134 /// Will panic on non-local backends.
135 fn host(backend_hint: PipelineBackendHint) -> Self {
136 if !matches!(backend_hint, PipelineBackendHint::Local) {
137 panic!("can only use `FlowArch::host` when defining a local-only pipeline");
138 }
139
140 cfg_select! {
141 // xtask-fmt allow-target-arch oneoff-flowey
142 target_arch = "x86_64" => Self::X86_64,
143 // xtask-fmt allow-target-arch oneoff-flowey
144 target_arch = "aarch64" => Self::Aarch64,
145 _ => compile_error!("unsupported target architecture"),
146 }
147 }
148}
149
150/// Trigger ADO pipelines via Continuous Integration
151#[derive(Default, Debug)]
152pub struct AdoScheduleTriggers {
153 /// Friendly name for the scheduled run
154 pub display_name: String,
155 /// Run the pipeline whenever there is a commit on these specified branches
156 /// (supports glob syntax)
157 pub branches: Vec<String>,
158 /// Specify any branches which should be filtered out from the list of
159 /// `branches` (supports glob syntax)
160 pub exclude_branches: Vec<String>,
161 /// Run the pipeline in a schedule, as specified by a cron string
162 pub cron: String,
163}
164
165/// Trigger ADO pipelines per PR
166#[derive(Debug)]
167pub struct AdoPrTriggers {
168 /// Run the pipeline whenever there is a PR to these specified branches
169 /// (supports glob syntax)
170 pub branches: Vec<String>,
171 /// Specify any branches which should be filtered out from the list of
172 /// `branches` (supports glob syntax)
173 pub exclude_branches: Vec<String>,
174 /// Run the pipeline even if the PR is a draft PR. Defaults to `false`.
175 pub run_on_draft: bool,
176 /// Automatically cancel the pipeline run if a new commit lands in the
177 /// branch. Defaults to `true`.
178 pub auto_cancel: bool,
179 /// Only run the pipeline when files matching these paths are changed
180 /// (supports glob syntax)
181 pub paths: Vec<String>,
182 /// Specify any paths which should be filtered out from the list of
183 /// `paths` (supports glob syntax)
184 pub exclude_paths: Vec<String>,
185}
186
187/// Trigger ADO pipelines per CI
188#[derive(Debug, Default)]
189pub struct AdoCiTriggers {
190 /// Run the pipeline whenever there is a change to these specified branches
191 /// (supports glob syntax)
192 pub branches: Vec<String>,
193 /// Specify any branches which should be filtered out from the list of
194 /// `branches` (supports glob syntax)
195 pub exclude_branches: Vec<String>,
196 /// Run the pipeline whenever a matching tag is created (supports glob
197 /// syntax)
198 pub tags: Vec<String>,
199 /// Specify any tags which should be filtered out from the list of `tags`
200 /// (supports glob syntax)
201 pub exclude_tags: Vec<String>,
202 /// Whether to batch changes per branch.
203 pub batch: bool,
204 /// Only run the pipeline when files matching these paths are changed
205 /// (supports glob syntax)
206 pub paths: Vec<String>,
207 /// Specify any paths which should be filtered out from the list of
208 /// `paths` (supports glob syntax)
209 pub exclude_paths: Vec<String>,
210}
211
212impl Default for AdoPrTriggers {
213 fn default() -> Self {
214 Self {
215 branches: Vec::new(),
216 exclude_branches: Vec::new(),
217 run_on_draft: false,
218 auto_cancel: true,
219 paths: Vec::new(),
220 exclude_paths: Vec::new(),
221 }
222 }
223}
224
225/// ADO repository resource.
226#[derive(Debug)]
227pub struct AdoResourcesRepository {
228 /// Type of repo that is being connected to.
229 pub repo_type: AdoResourcesRepositoryType,
230 /// Repository name. Format depends on `repo_type`.
231 pub name: String,
232 /// git ref to checkout.
233 pub git_ref: AdoResourcesRepositoryRef,
234 /// (optional) ID of the service endpoint connecting to this repository.
235 pub endpoint: Option<String>,
236}
237
238/// ADO repository resource type
239#[derive(Debug)]
240pub enum AdoResourcesRepositoryType {
241 /// Azure Repos Git repository
242 AzureReposGit,
243 /// Github repository
244 GitHub,
245}
246
247/// ADO repository ref
248#[derive(Debug)]
249pub enum AdoResourcesRepositoryRef<P = UseParameter<String>> {
250 /// Hard-coded ref (e.g: refs/heads/main)
251 Fixed(String),
252 /// Connected to pipeline-level parameter
253 Parameter(P),
254}
255
256/// Trigger Github Actions pipelines via Continuous Integration
257///
258/// NOTE: Github Actions doesn't support specifying the branch when triggered by `schedule`.
259/// To run on a specific branch, modify the branch checked out in the pipeline.
260#[derive(Default, Debug)]
261pub struct GhScheduleTriggers {
262 /// Run the pipeline in a schedule, as specified by a cron string
263 pub cron: String,
264 /// IANA timezone string
265 pub timezone: Option<String>,
266}
267
268/// Trigger Github Actions pipelines per PR
269#[derive(Debug)]
270pub struct GhPrTriggers {
271 /// Run the pipeline whenever there is a PR to these specified branches
272 /// (supports glob syntax)
273 pub branches: Vec<String>,
274 /// Specify any branches which should be filtered out from the list of
275 /// `branches` (supports glob syntax)
276 pub exclude_branches: Vec<String>,
277 /// Automatically cancel the pipeline run if a new commit lands in the
278 /// branch. Defaults to `true`.
279 pub auto_cancel: bool,
280 /// Run the pipeline whenever the PR trigger matches the specified types
281 pub types: Vec<String>,
282 /// Only run the pipeline when files matching these paths are changed
283 /// (supports glob syntax)
284 pub paths: Vec<String>,
285 /// Specify any paths which should be filtered out from the list of
286 /// `paths` (supports glob syntax)
287 pub paths_ignore: Vec<String>,
288}
289
290/// Trigger Github Actions pipelines per PR
291#[derive(Debug, Default)]
292pub struct GhCiTriggers {
293 /// Run the pipeline whenever there is a change to these specified branches
294 /// (supports glob syntax)
295 pub branches: Vec<String>,
296 /// Specify any branches which should be filtered out from the list of
297 /// `branches` (supports glob syntax)
298 pub exclude_branches: Vec<String>,
299 /// Run the pipeline whenever a matching tag is created (supports glob
300 /// syntax)
301 pub tags: Vec<String>,
302 /// Specify any tags which should be filtered out from the list of `tags`
303 /// (supports glob syntax)
304 pub exclude_tags: Vec<String>,
305 /// Only run the pipeline when files matching these paths are changed
306 /// (supports glob syntax)
307 pub paths: Vec<String>,
308 /// Specify any paths which should be filtered out from the list of
309 /// `paths` (supports glob syntax)
310 pub paths_ignore: Vec<String>,
311 /// If set, only one pipeline run in this concurrency group runs at a time.
312 pub concurrency_group: Option<GhConcurrencyGroup>,
313}
314
315/// Settings for a Github concurrency group.
316#[derive(Debug, Default, Clone)]
317pub struct GhConcurrencyGroup {
318 /// The name of the concurrency group.
319 pub name: String,
320 /// Cancel the active run when a new run joins the concurrency group.
321 /// Defaults to `false`.
322 pub cancel_in_progress: bool,
323}
324
325impl GhPrTriggers {
326 /// Triggers the pipeline on the default PR events plus when a draft is marked as ready for review.
327 pub fn new_draftable() -> Self {
328 Self {
329 branches: Vec::new(),
330 exclude_branches: Vec::new(),
331 types: vec![
332 "opened".into(),
333 "synchronize".into(),
334 "reopened".into(),
335 "ready_for_review".into(),
336 ],
337 auto_cancel: true,
338 paths: Vec::new(),
339 paths_ignore: Vec::new(),
340 }
341 }
342}
343
344#[derive(Debug, Clone, PartialEq)]
345pub enum GhRunnerOsLabel {
346 UbuntuLatest,
347 Ubuntu2404,
348 Ubuntu2204,
349 WindowsLatest,
350 Windows2025,
351 Windows2022,
352 Ubuntu2404Arm,
353 Ubuntu2204Arm,
354 Windows11Arm,
355 Custom(String),
356}
357
358/// GitHub runner type
359#[derive(Debug, Clone, PartialEq)]
360pub enum GhRunner {
361 // See <https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#choosing-github-hosted-runners>
362 // for more details.
363 GhHosted(GhRunnerOsLabel),
364 // Self hosted runners are selected by matching runner labels to <labels>.
365 // 'self-hosted' is a common label for self hosted runners, but is not required.
366 // Labels are case-insensitive and can take the form of arbitrary strings.
367 // See <https://docs.github.com/en/actions/hosting-your-own-runners> for more details.
368 SelfHosted(Vec<String>),
369 // This uses a runner belonging to <group> that matches all labels in <labels>.
370 // See <https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#choosing-github-hosted-runners>
371 // for more details.
372 RunnerGroup { group: String, labels: Vec<String> },
373}
374
375impl GhRunner {
376 /// Whether this is a self-hosted runner with the provided label
377 pub fn is_self_hosted_with_label(&self, label: &str) -> bool {
378 matches!(self, GhRunner::SelfHosted(labels) if labels.iter().any(|s| s.as_str() == label))
379 }
380}
381
382// TODO: support a more structured format for demands
383// See https://learn.microsoft.com/en-us/azure/devops/pipelines/yaml-schema/pool-demands
384#[derive(Debug, Clone)]
385pub struct AdoPool {
386 pub name: String,
387 pub demands: Vec<String>,
388}
389
390/// Parameter type (unstable / stable).
391#[derive(Debug, Clone)]
392pub enum ParameterKind {
393 // The parameter is considered an unstable API, and should not be
394 // taken as a dependency.
395 Unstable,
396 // The parameter is considered a stable API, and can be used by
397 // external pipelines to control behavior of the pipeline.
398 Stable,
399}
400
401#[derive(Clone, Debug)]
402#[must_use]
403pub struct UseParameter<T> {
404 idx: usize,
405 _kind: std::marker::PhantomData<T>,
406}
407
408/// Opaque handle to an artifact which must be published by a single job.
409#[must_use]
410pub struct PublishArtifact {
411 idx: usize,
412}
413
414/// Opaque handle to an artifact which can be used by one or more jobs.
415#[derive(Clone)]
416#[must_use]
417pub struct UseArtifact {
418 idx: usize,
419}
420
421/// Opaque handle to an artifact of type `T` which must be published by a single job.
422#[must_use]
423pub struct PublishTypedArtifact<T>(PublishArtifact, std::marker::PhantomData<fn() -> T>);
424
425/// Opaque handle to an artifact of type `T` which can be used by one or more
426/// jobs.
427#[must_use]
428pub struct UseTypedArtifact<T>(UseArtifact, std::marker::PhantomData<fn(T)>);
429
430impl<T> Clone for UseTypedArtifact<T> {
431 fn clone(&self) -> Self {
432 UseTypedArtifact(self.0.clone(), std::marker::PhantomData)
433 }
434}
435
436#[derive(Default)]
437pub struct Pipeline {
438 jobs: Vec<PipelineJobMetadata>,
439 artifacts: Vec<ArtifactMeta>,
440 parameters: Vec<ParameterMeta>,
441 extra_deps: BTreeSet<(usize, usize)>,
442 // builder internal
443 artifact_names: BTreeSet<String>,
444 dummy_done_idx: usize,
445 artifact_map_idx: usize,
446 global_patchfns: Vec<crate::patch::PatchFn>,
447 inject_all_jobs_with: Option<Box<dyn for<'a> Fn(PipelineJob<'a>) -> PipelineJob<'a>>>,
448 // backend specific
449 ado_name: Option<String>,
450 ado_job_id_overrides: BTreeMap<usize, String>,
451 ado_schedule_triggers: Vec<AdoScheduleTriggers>,
452 ado_ci_triggers: Option<AdoCiTriggers>,
453 ado_pr_triggers: Option<AdoPrTriggers>,
454 ado_resources_repository: Vec<InternalAdoResourcesRepository>,
455 ado_bootstrap_template: Option<Box<AdoBootstrapTemplate>>,
456 ado_variables: BTreeMap<String, String>,
457 ado_post_process_yaml_cb: Option<Box<dyn FnOnce(serde_yaml::Value) -> serde_yaml::Value>>,
458 gh_name: Option<String>,
459 gh_schedule_triggers: Vec<GhScheduleTriggers>,
460 gh_ci_triggers: Option<GhCiTriggers>,
461 gh_pr_triggers: Option<GhPrTriggers>,
462 gh_bootstrap_template: Option<Box<GhBootstrapTemplate>>,
463}
464
465impl Pipeline {
466 pub fn new() -> Pipeline {
467 Pipeline::default()
468 }
469
470 /// Inject all pipeline jobs with some common logic. (e.g: to resolve common
471 /// configuration requirements shared by all jobs).
472 ///
473 /// Can only be invoked once per pipeline.
474 #[track_caller]
475 pub fn inject_all_jobs_with(
476 &mut self,
477 cb: impl for<'a> Fn(PipelineJob<'a>) -> PipelineJob<'a> + 'static,
478 ) -> &mut Self {
479 if self.inject_all_jobs_with.is_some() {
480 panic!("can only call inject_all_jobs_with once!")
481 }
482 self.inject_all_jobs_with = Some(Box::new(cb));
483 self
484 }
485
486 /// (ADO only) Provide a callback to provide a job-specific YAML template used to bootstrap flowey at the start
487 /// of an ADO pipeline.
488 ///
489 /// The callback receives the job's platform and architecture and is invoked
490 /// for each job that builds flowey, but not jobs that consume a prebuilt
491 /// binary.
492 ///
493 /// The template has access to the following vars, which will be statically
494 /// interpolated into the template's text:
495 ///
496 /// - `{{FLOWEY_OUTDIR}}`
497 /// - Directory to copy artifacts into.
498 /// - NOTE: this var will include `\` on Windows, and `/` on linux!
499 /// - `{{FLOWEY_BIN_EXTENSION}}`
500 /// - Extension of the expected flowey bin (either "", or ".exe")
501 /// - `{{FLOWEY_CRATE}}`
502 /// - Name of the project-specific flowey crate to be built
503 /// - `{{FLOWEY_TARGET}}`
504 /// - The target-triple flowey is being built for
505 /// - `{{FLOWEY_PIPELINE_PATH}}`
506 /// - Repo-root relative path to the pipeline (as provided when
507 /// generating the pipeline via the flowey CLI)
508 ///
509 /// The template's sole responsibility is to copy 3 files into the
510 /// `{{FLOWEY_OUTDIR}}`:
511 ///
512 /// 1. The bootstrapped flowey binary, with the file name
513 /// `flowey{{FLOWEY_BIN_EXTENSION}}`
514 /// 2. Two files called `pipeline.yaml` and `pipeline.json`, which are
515 /// copied of the pipeline YAML and pipeline JSON currently being run.
516 /// `{{FLOWEY_PIPELINE_PATH}}` is provided as a way to disambiguate in
517 /// cases where the same template is being for multiple pipelines (e.g: a
518 /// debug vs. release pipeline).
519 pub fn ado_set_flowey_bootstrap_template_fn(
520 &mut self,
521 template: impl Fn(FlowPlatform, FlowArch) -> anyhow::Result<String> + 'static,
522 ) -> &mut Self {
523 self.ado_bootstrap_template = Some(Box::new(template));
524 self
525 }
526
527 /// (ADO only) Provide a callback function which will be used to
528 /// post-process any YAML flowey generates for the pipeline.
529 ///
530 /// Until flowey defines a stable API for maintaining out-of-tree backends,
531 /// this method can be used to integrate the output from the generic ADO
532 /// backend with any organization-specific templates that one may be
533 /// required to use (e.g: for compliance reasons).
534 pub fn ado_post_process_yaml(
535 &mut self,
536 cb: impl FnOnce(serde_yaml::Value) -> serde_yaml::Value + 'static,
537 ) -> &mut Self {
538 self.ado_post_process_yaml_cb = Some(Box::new(cb));
539 self
540 }
541
542 /// (ADO only) Add a new scheduled CI trigger. Can be called multiple times
543 /// to set up multiple schedules runs.
544 pub fn ado_add_schedule_trigger(&mut self, triggers: AdoScheduleTriggers) -> &mut Self {
545 self.ado_schedule_triggers.push(triggers);
546 self
547 }
548
549 /// (ADO only) Set a PR trigger. Calling this method multiple times will
550 /// overwrite any previously set triggers.
551 pub fn ado_set_pr_triggers(&mut self, triggers: AdoPrTriggers) -> &mut Self {
552 self.ado_pr_triggers = Some(triggers);
553 self
554 }
555
556 /// (ADO only) Set a CI trigger. Calling this method multiple times will
557 /// overwrite any previously set triggers.
558 pub fn ado_set_ci_triggers(&mut self, triggers: AdoCiTriggers) -> &mut Self {
559 self.ado_ci_triggers = Some(triggers);
560 self
561 }
562
563 /// (ADO only) Declare a new repository resource, returning a type-safe
564 /// handle which downstream ADO steps are able to consume via
565 /// [`AdoStepServices::resolve_repository_id`](crate::node::user_facing::AdoStepServices::resolve_repository_id).
566 pub fn ado_add_resources_repository(
567 &mut self,
568 repo: AdoResourcesRepository,
569 ) -> AdoResourcesRepositoryId {
570 let AdoResourcesRepository {
571 repo_type,
572 name,
573 git_ref,
574 endpoint,
575 } = repo;
576
577 let repo_id = format!("repo{}", self.ado_resources_repository.len());
578
579 self.ado_resources_repository
580 .push(InternalAdoResourcesRepository {
581 repo_id: repo_id.clone(),
582 repo_type,
583 name,
584 git_ref: match git_ref {
585 AdoResourcesRepositoryRef::Fixed(s) => AdoResourcesRepositoryRef::Fixed(s),
586 AdoResourcesRepositoryRef::Parameter(p) => {
587 AdoResourcesRepositoryRef::Parameter(p.idx)
588 }
589 },
590 endpoint,
591 });
592 AdoResourcesRepositoryId { repo_id }
593 }
594
595 /// (GitHub Actions only) Set the pipeline-level name.
596 ///
597 /// <https://docs.github.com/en/actions/writing-workflows/workflow-syntax-for-github-actions#name>
598 pub fn gh_set_name(&mut self, name: impl AsRef<str>) -> &mut Self {
599 self.gh_name = Some(name.as_ref().into());
600 self
601 }
602
603 /// Provide a callback that provides a job-specific YAML template used to
604 /// bootstrap flowey at the start of a GitHub Actions pipeline.
605 ///
606 /// The callback receives the job's platform and architecture and is invoked
607 /// for each job that builds flowey, but not jobs that consume a prebuilt
608 /// binary.
609 ///
610 /// The template has access to the following vars, which will be statically
611 /// interpolated into the template's text:
612 ///
613 /// - `{{FLOWEY_OUTDIR}}`
614 /// - Directory to copy artifacts into.
615 /// - NOTE: this var will include `\` on Windows, and `/` on linux!
616 /// - `{{FLOWEY_BIN_EXTENSION}}`
617 /// - Extension of the expected flowey bin (either "", or ".exe")
618 /// - `{{FLOWEY_CRATE}}`
619 /// - Name of the project-specific flowey crate to be built
620 /// - `{{FLOWEY_TARGET}}`
621 /// - The target-triple flowey is being built for
622 /// - `{{FLOWEY_PIPELINE_PATH}}`
623 /// - Repo-root relative path to the pipeline (as provided when
624 /// generating the pipeline via the flowey CLI)
625 ///
626 /// The template's sole responsibility is to copy 3 files into the
627 /// `{{FLOWEY_OUTDIR}}`:
628 ///
629 /// 1. The bootstrapped flowey binary, with the file name
630 /// `flowey{{FLOWEY_BIN_EXTENSION}}`
631 /// 2. Two files called `pipeline.yaml` and `pipeline.json`, which are
632 /// copied of the pipeline YAML and pipeline JSON currently being run.
633 /// `{{FLOWEY_PIPELINE_PATH}}` is provided as a way to disambiguate in
634 /// cases where the same template is being for multiple pipelines (e.g: a
635 /// debug vs. release pipeline).
636 pub fn gh_set_flowey_bootstrap_template_fn(
637 &mut self,
638 template: impl Fn(FlowPlatform, FlowArch) -> anyhow::Result<String> + 'static,
639 ) -> &mut Self {
640 self.gh_bootstrap_template = Some(Box::new(template));
641 self
642 }
643
644 /// (GitHub Actions only) Add a new scheduled CI trigger. Can be called multiple times
645 /// to set up multiple schedules runs.
646 pub fn gh_add_schedule_trigger(&mut self, triggers: GhScheduleTriggers) -> &mut Self {
647 self.gh_schedule_triggers.push(triggers);
648 self
649 }
650
651 /// (GitHub Actions only) Set a PR trigger. Calling this method multiple times will
652 /// overwrite any previously set triggers.
653 pub fn gh_set_pr_triggers(&mut self, triggers: GhPrTriggers) -> &mut Self {
654 self.gh_pr_triggers = Some(triggers);
655 self
656 }
657
658 /// (GitHub Actions only) Set a CI trigger. Calling this method multiple times will
659 /// overwrite any previously set triggers.
660 pub fn gh_set_ci_triggers(&mut self, triggers: GhCiTriggers) -> &mut Self {
661 self.gh_ci_triggers = Some(triggers);
662 self
663 }
664
665 /// (GitHub Actions only) Use a pre-defined GitHub Actions secret variable.
666 ///
667 /// For more information on defining secrets for use in GitHub Actions, see
668 /// <https://docs.github.com/en/actions/security-guides/using-secrets-in-github-actions>
669 pub fn gh_use_secret(&mut self, secret_name: impl AsRef<str>) -> GhUserSecretVar {
670 GhUserSecretVar(secret_name.as_ref().to_string())
671 }
672
673 pub fn new_job(
674 &mut self,
675 platform: FlowPlatform,
676 arch: FlowArch,
677 label: impl AsRef<str>,
678 ) -> PipelineJob<'_> {
679 let idx = self.jobs.len();
680 self.jobs.push(PipelineJobMetadata {
681 root_nodes: BTreeMap::new(),
682 root_configs: BTreeMap::new(),
683 patches: ResolvedPatches::build(),
684 label: label.as_ref().into(),
685 platform,
686 arch,
687 cond_param_idx: None,
688 timeout_minutes: None,
689 command_wrapper: None,
690 ado_pool: None,
691 ado_variables: BTreeMap::new(),
692 gh_override_if: None,
693 gh_global_env: BTreeMap::new(),
694 gh_pool: None,
695 gh_concurrency_group: None,
696 gh_permissions: BTreeMap::new(),
697 });
698
699 PipelineJob {
700 pipeline: self,
701 job_idx: idx,
702 }
703 }
704
705 /// Declare a dependency between two jobs that does is not a result of an
706 /// artifact.
707 pub fn non_artifact_dep(
708 &mut self,
709 job: &PipelineJobHandle,
710 depends_on_job: &PipelineJobHandle,
711 ) -> &mut Self {
712 self.extra_deps
713 .insert((depends_on_job.job_idx, job.job_idx));
714 self
715 }
716
717 #[track_caller]
718 pub fn new_artifact(&mut self, name: impl AsRef<str>) -> (PublishArtifact, UseArtifact) {
719 let name = name.as_ref();
720 let owned_name = name.to_string();
721
722 let not_exists = self.artifact_names.insert(owned_name.clone());
723 if !not_exists {
724 panic!("duplicate artifact name: {}", name)
725 }
726
727 let idx = self.artifacts.len();
728 self.artifacts.push(ArtifactMeta {
729 name: owned_name,
730 published_by_job: None,
731 used_by_jobs: BTreeSet::new(),
732 });
733
734 (PublishArtifact { idx }, UseArtifact { idx })
735 }
736
737 /// Returns a pair of opaque handles to a new artifact for use across jobs
738 /// in the pipeline.
739 #[track_caller]
740 pub fn new_typed_artifact<T: Artifact>(
741 &mut self,
742 name: impl AsRef<str>,
743 ) -> (PublishTypedArtifact<T>, UseTypedArtifact<T>) {
744 let (publish, use_artifact) = self.new_artifact(name);
745 (
746 PublishTypedArtifact(publish, std::marker::PhantomData),
747 UseTypedArtifact(use_artifact, std::marker::PhantomData),
748 )
749 }
750
751 /// Returns a pair of sets of opaque handles to a new artifact for use
752 /// across jobs in the pipeline. The artifact names are derived by the impl
753 /// of [`ArtifactType::name`] using common prefixes and suffixes if
754 /// specified (although the implementor can choose to use those values
755 /// differently).
756 #[track_caller]
757 pub fn new_typed_artifact_collection<T: Artifact, U: ArtifactType>(
758 &mut self,
759 artifact_types: impl IntoIterator<Item = U>,
760 prefix: Option<&str>,
761 suffix: Option<&str>,
762 ) -> (
763 BTreeMap<U, PublishTypedArtifact<T>>,
764 BTreeMap<U, UseTypedArtifact<T>>,
765 ) {
766 artifact_types
767 .into_iter()
768 .map(|artifact_type| {
769 let (pub_artifact, use_artifact) =
770 self.new_typed_artifact(artifact_type.name(prefix, suffix));
771 (
772 (artifact_type.clone(), pub_artifact),
773 (artifact_type, use_artifact),
774 )
775 })
776 .unzip()
777 }
778
779 /// (ADO only) Set the pipeline-level name.
780 ///
781 /// <https://learn.microsoft.com/en-us/azure/devops/pipelines/process/run-number?view=azure-devops&tabs=yaml>
782 pub fn ado_add_name(&mut self, name: String) -> &mut Self {
783 self.ado_name = Some(name);
784 self
785 }
786
787 /// (ADO only) Declare a pipeline-level, named, read-only ADO variable.
788 ///
789 /// `name` and `value` are both arbitrary strings.
790 ///
791 /// Returns an instance of [`AdoRuntimeVar`], which, if need be, can be
792 /// converted into a [`ReadVar<String>`] using
793 /// [`NodeCtx::get_ado_variable`].
794 ///
795 /// NOTE: Unless required by some particular third-party task, it's strongly
796 /// recommended to _avoid_ using this method, and to simply use
797 /// [`ReadVar::from_static`] to get a obtain a static variable.
798 ///
799 /// [`NodeCtx::get_ado_variable`]: crate::node::NodeCtx::get_ado_variable
800 pub fn ado_new_named_variable(
801 &mut self,
802 name: impl AsRef<str>,
803 value: impl AsRef<str>,
804 ) -> AdoRuntimeVar {
805 let name = name.as_ref();
806 let value = value.as_ref();
807
808 self.ado_variables.insert(name.into(), value.into());
809
810 // safe, since we'll ensure that the global exists in the ADO backend
811 AdoRuntimeVar::dangerous_from_global(name, false)
812 }
813
814 /// (ADO only) Declare multiple pipeline-level, named, read-only ADO
815 /// variables at once.
816 ///
817 /// This is a convenience method to streamline invoking
818 /// [`Self::ado_new_named_variable`] multiple times.
819 ///
820 /// NOTE: Unless required by some particular third-party task, it's strongly
821 /// recommended to _avoid_ using this method, and to simply use
822 /// [`ReadVar::from_static`] to get a obtain a static variable.
823 ///
824 /// DEVNOTE: In the future, this API may be updated to return a handle that
825 /// will allow resolving the resulting `AdoRuntimeVar`, but for
826 /// implementation expediency, this API does not currently do this. If you
827 /// need to read the value of this variable at runtime, you may need to
828 /// invoke [`AdoRuntimeVar::dangerous_from_global`] manually.
829 ///
830 /// [`NodeCtx::get_ado_variable`]: crate::node::NodeCtx::get_ado_variable
831 pub fn ado_new_named_variables<K, V>(
832 &mut self,
833 vars: impl IntoIterator<Item = (K, V)>,
834 ) -> &mut Self
835 where
836 K: AsRef<str>,
837 V: AsRef<str>,
838 {
839 self.ado_variables.extend(
840 vars.into_iter()
841 .map(|(k, v)| (k.as_ref().into(), v.as_ref().into())),
842 );
843 self
844 }
845
846 /// Declare a pipeline-level runtime parameter with type `bool`.
847 ///
848 /// To obtain a [`ReadVar<bool>`] that can be used within a node, use the
849 /// [`PipelineJobCtx::use_parameter`] method.
850 ///
851 /// `name` is the name of the parameter.
852 ///
853 /// `description` is an arbitrary string, which will be be shown to users.
854 ///
855 /// `kind` is the type of parameter and if it should be treated as a stable
856 /// external API to callers of the pipeline.
857 ///
858 /// `default` is the default value for the parameter. If none is provided,
859 /// the parameter _must_ be specified in order for the pipeline to run.
860 ///
861 /// `possible_values` can be used to limit the set of valid values the
862 /// parameter accepts.
863 pub fn new_parameter_bool(
864 &mut self,
865 name: impl AsRef<str>,
866 description: impl AsRef<str>,
867 kind: ParameterKind,
868 default: Option<bool>,
869 ) -> UseParameter<bool> {
870 let idx = self.parameters.len();
871 let name = new_parameter_name(name, kind.clone());
872 self.parameters.push(ParameterMeta {
873 parameter: Parameter::Bool {
874 name,
875 description: description.as_ref().into(),
876 kind,
877 default,
878 },
879 used_by_jobs: BTreeSet::new(),
880 });
881
882 UseParameter {
883 idx,
884 _kind: std::marker::PhantomData,
885 }
886 }
887
888 /// Declare a pipeline-level runtime parameter with type `i64`.
889 ///
890 /// To obtain a [`ReadVar<i64>`] that can be used within a node, use the
891 /// [`PipelineJobCtx::use_parameter`] method.
892 ///
893 /// `name` is the name of the parameter.
894 ///
895 /// `description` is an arbitrary string, which will be be shown to users.
896 ///
897 /// `kind` is the type of parameter and if it should be treated as a stable
898 /// external API to callers of the pipeline.
899 ///
900 /// `default` is the default value for the parameter. If none is provided,
901 /// the parameter _must_ be specified in order for the pipeline to run.
902 ///
903 /// `possible_values` can be used to limit the set of valid values the
904 /// parameter accepts.
905 pub fn new_parameter_num(
906 &mut self,
907 name: impl AsRef<str>,
908 description: impl AsRef<str>,
909 kind: ParameterKind,
910 default: Option<i64>,
911 possible_values: Option<Vec<i64>>,
912 ) -> UseParameter<i64> {
913 let idx = self.parameters.len();
914 let name = new_parameter_name(name, kind.clone());
915 self.parameters.push(ParameterMeta {
916 parameter: Parameter::Num {
917 name,
918 description: description.as_ref().into(),
919 kind,
920 default,
921 possible_values,
922 },
923 used_by_jobs: BTreeSet::new(),
924 });
925
926 UseParameter {
927 idx,
928 _kind: std::marker::PhantomData,
929 }
930 }
931
932 /// Declare a pipeline-level runtime parameter with type `String`.
933 ///
934 /// To obtain a [`ReadVar<String>`] that can be used within a node, use the
935 /// [`PipelineJobCtx::use_parameter`] method.
936 ///
937 /// `name` is the name of the parameter.
938 ///
939 /// `description` is an arbitrary string, which will be be shown to users.
940 ///
941 /// `kind` is the type of parameter and if it should be treated as a stable
942 /// external API to callers of the pipeline.
943 ///
944 /// `default` is the default value for the parameter. If none is provided,
945 /// the parameter _must_ be specified in order for the pipeline to run.
946 ///
947 /// `possible_values` allows restricting inputs to a set of possible values.
948 /// Depending on the backend, these options may be presented as a set of
949 /// radio buttons, a dropdown menu, or something in that vein. If `None`,
950 /// then any string is allowed.
951 pub fn new_parameter_string(
952 &mut self,
953 name: impl AsRef<str>,
954 description: impl AsRef<str>,
955 kind: ParameterKind,
956 default: Option<impl AsRef<str>>,
957 possible_values: Option<Vec<String>>,
958 ) -> UseParameter<String> {
959 let idx = self.parameters.len();
960 let name = new_parameter_name(name, kind.clone());
961 self.parameters.push(ParameterMeta {
962 parameter: Parameter::String {
963 name,
964 description: description.as_ref().into(),
965 kind,
966 default: default.map(|x| x.as_ref().into()),
967 possible_values,
968 },
969 used_by_jobs: BTreeSet::new(),
970 });
971
972 UseParameter {
973 idx,
974 _kind: std::marker::PhantomData,
975 }
976 }
977}
978
979pub struct PipelineJobCtx<'a> {
980 pipeline: &'a mut Pipeline,
981 job_idx: usize,
982}
983
984impl PipelineJobCtx<'_> {
985 /// Create a new `WriteVar<SideEffect>` anchored to the pipeline job.
986 pub fn new_done_handle(&mut self) -> WriteVar<crate::node::SideEffect> {
987 self.pipeline.dummy_done_idx += 1;
988 crate::node::thin_air_write_runtime_var(format!("start{}", self.pipeline.dummy_done_idx))
989 }
990
991 /// Claim that this job will use this artifact, obtaining a path to a folder
992 /// with the artifact's contents.
993 pub fn use_artifact(&mut self, artifact: &UseArtifact) -> ReadVar<PathBuf> {
994 self.pipeline.artifacts[artifact.idx]
995 .used_by_jobs
996 .insert(self.job_idx);
997
998 crate::node::thin_air_read_runtime_var(consistent_artifact_runtime_var_name(
999 &self.pipeline.artifacts[artifact.idx].name,
1000 true,
1001 ))
1002 }
1003
1004 /// Claim that this job will publish this artifact, obtaining a path to a
1005 /// fresh, empty folder which will be published as the specific artifact at
1006 /// the end of the job.
1007 pub fn publish_artifact(&mut self, artifact: PublishArtifact) -> ReadVar<PathBuf> {
1008 let existing = self.pipeline.artifacts[artifact.idx]
1009 .published_by_job
1010 .replace(self.job_idx);
1011 assert!(existing.is_none()); // PublishArtifact isn't cloneable
1012
1013 crate::node::thin_air_read_runtime_var(consistent_artifact_runtime_var_name(
1014 &self.pipeline.artifacts[artifact.idx].name,
1015 false,
1016 ))
1017 }
1018
1019 fn helper_request<R: IntoRequest>(&mut self, req: R)
1020 where
1021 R::Node: 'static,
1022 {
1023 self.pipeline.jobs[self.job_idx]
1024 .root_nodes
1025 .entry(NodeHandle::from_type::<R::Node>())
1026 .or_default()
1027 .push(serde_json::to_vec(&req.into_request()).unwrap().into());
1028 }
1029
1030 fn new_artifact_map_vars<T: Artifact>(&mut self) -> (ReadVar<T>, WriteVar<T>) {
1031 let artifact_map_idx = self.pipeline.artifact_map_idx;
1032 self.pipeline.artifact_map_idx += 1;
1033
1034 let backing_var = format!("artifact_map{}", artifact_map_idx);
1035 let read_var = crate::node::thin_air_read_runtime_var(backing_var.clone());
1036 let write_var = crate::node::thin_air_write_runtime_var(backing_var);
1037 (read_var, write_var)
1038 }
1039
1040 /// Claim that this job will use this artifact, obtaining the resolved
1041 /// contents of the artifact.
1042 pub fn use_typed_artifact<T: Artifact>(
1043 &mut self,
1044 artifact: &UseTypedArtifact<T>,
1045 ) -> ReadVar<T> {
1046 let artifact_path = self.use_artifact(&artifact.0);
1047 let (read, write) = self.new_artifact_map_vars::<T>();
1048 self.helper_request(resolve::Request::new(artifact_path, write));
1049 read
1050 }
1051
1052 /// Claim that this job will publish this artifact, obtaining a variable to
1053 /// write the artifact's contents to. The artifact will be published at
1054 /// the end of the job.
1055 pub fn publish_typed_artifact<T: Artifact>(
1056 &mut self,
1057 artifact: PublishTypedArtifact<T>,
1058 ) -> WriteVar<T> {
1059 let artifact_path = self.publish_artifact(artifact.0);
1060 let (read, write) = self.new_artifact_map_vars::<T>();
1061 let done = self.new_done_handle();
1062 self.helper_request(artifact::publish::Request::new(read, artifact_path, done));
1063 write
1064 }
1065
1066 /// Obtain a `ReadVar<T>` corresponding to a pipeline parameter which is
1067 /// specified at runtime.
1068 pub fn use_parameter<T>(&mut self, param: UseParameter<T>) -> ReadVar<T>
1069 where
1070 T: Serialize + DeserializeOwned,
1071 {
1072 self.pipeline.parameters[param.idx]
1073 .used_by_jobs
1074 .insert(self.job_idx);
1075
1076 crate::node::thin_air_read_runtime_var(
1077 self.pipeline.parameters[param.idx]
1078 .parameter
1079 .name()
1080 .to_string(),
1081 )
1082 }
1083
1084 /// Shortcut which allows defining a bool pipeline parameter within a Job.
1085 ///
1086 /// To share a single parameter between multiple jobs, don't use this method
1087 /// - use [`Pipeline::new_parameter_bool`] + [`Self::use_parameter`] instead.
1088 pub fn new_parameter_bool(
1089 &mut self,
1090 name: impl AsRef<str>,
1091 description: impl AsRef<str>,
1092 kind: ParameterKind,
1093 default: Option<bool>,
1094 ) -> ReadVar<bool> {
1095 let param = self
1096 .pipeline
1097 .new_parameter_bool(name, description, kind, default);
1098 self.use_parameter(param)
1099 }
1100
1101 /// Shortcut which allows defining a number pipeline parameter within a Job.
1102 ///
1103 /// To share a single parameter between multiple jobs, don't use this method
1104 /// - use [`Pipeline::new_parameter_num`] + [`Self::use_parameter`] instead.
1105 pub fn new_parameter_num(
1106 &mut self,
1107 name: impl AsRef<str>,
1108 description: impl AsRef<str>,
1109 kind: ParameterKind,
1110 default: Option<i64>,
1111 possible_values: Option<Vec<i64>>,
1112 ) -> ReadVar<i64> {
1113 let param =
1114 self.pipeline
1115 .new_parameter_num(name, description, kind, default, possible_values);
1116 self.use_parameter(param)
1117 }
1118
1119 /// Shortcut which allows defining a string pipeline parameter within a Job.
1120 ///
1121 /// To share a single parameter between multiple jobs, don't use this method
1122 /// - use [`Pipeline::new_parameter_string`] + [`Self::use_parameter`] instead.
1123 pub fn new_parameter_string(
1124 &mut self,
1125 name: impl AsRef<str>,
1126 description: impl AsRef<str>,
1127 kind: ParameterKind,
1128 default: Option<String>,
1129 possible_values: Option<Vec<String>>,
1130 ) -> ReadVar<String> {
1131 let param =
1132 self.pipeline
1133 .new_parameter_string(name, description, kind, default, possible_values);
1134 self.use_parameter(param)
1135 }
1136}
1137
1138#[must_use]
1139pub struct PipelineJob<'a> {
1140 pipeline: &'a mut Pipeline,
1141 job_idx: usize,
1142}
1143
1144impl PipelineJob<'_> {
1145 /// (ADO only) specify which agent pool this job will be run on.
1146 pub fn ado_set_pool(self, pool: AdoPool) -> Self {
1147 self.pipeline.jobs[self.job_idx].ado_pool = Some(pool);
1148 self
1149 }
1150
1151 /// (ADO only) specify which agent pool this job will be run on, with
1152 /// additional special runner demands.
1153 pub fn ado_set_pool_with_demands(self, pool: impl AsRef<str>, demands: Vec<String>) -> Self {
1154 self.pipeline.jobs[self.job_idx].ado_pool = Some(AdoPool {
1155 name: pool.as_ref().into(),
1156 demands,
1157 });
1158 self
1159 }
1160
1161 /// (ADO only) Declare a job-level, named, read-only ADO variable.
1162 ///
1163 /// `name` and `value` are both arbitrary strings, which may include ADO
1164 /// template expressions.
1165 ///
1166 /// NOTE: Unless required by some particular third-party task, it's strongly
1167 /// recommended to _avoid_ using this method, and to simply use
1168 /// [`ReadVar::from_static`] to get a obtain a static variable.
1169 ///
1170 /// DEVNOTE: In the future, this API may be updated to return a handle that
1171 /// will allow resolving the resulting `AdoRuntimeVar`, but for
1172 /// implementation expediency, this API does not currently do this. If you
1173 /// need to read the value of this variable at runtime, you may need to
1174 /// invoke [`AdoRuntimeVar::dangerous_from_global`] manually.
1175 ///
1176 /// [`NodeCtx::get_ado_variable`]: crate::node::NodeCtx::get_ado_variable
1177 pub fn ado_new_named_variable(self, name: impl AsRef<str>, value: impl AsRef<str>) -> Self {
1178 let name = name.as_ref();
1179 let value = value.as_ref();
1180 self.pipeline.jobs[self.job_idx]
1181 .ado_variables
1182 .insert(name.into(), value.into());
1183 self
1184 }
1185
1186 /// (ADO only) Declare multiple job-level, named, read-only ADO variables at
1187 /// once.
1188 ///
1189 /// This is a convenience method to streamline invoking
1190 /// [`Self::ado_new_named_variable`] multiple times.
1191 ///
1192 /// NOTE: Unless required by some particular third-party task, it's strongly
1193 /// recommended to _avoid_ using this method, and to simply use
1194 /// [`ReadVar::from_static`] to get a obtain a static variable.
1195 ///
1196 /// DEVNOTE: In the future, this API may be updated to return a handle that
1197 /// will allow resolving the resulting `AdoRuntimeVar`, but for
1198 /// implementation expediency, this API does not currently do this. If you
1199 /// need to read the value of this variable at runtime, you may need to
1200 /// invoke [`AdoRuntimeVar::dangerous_from_global`] manually.
1201 ///
1202 /// [`NodeCtx::get_ado_variable`]: crate::node::NodeCtx::get_ado_variable
1203 pub fn ado_new_named_variables<K, V>(self, vars: impl IntoIterator<Item = (K, V)>) -> Self
1204 where
1205 K: AsRef<str>,
1206 V: AsRef<str>,
1207 {
1208 self.pipeline.jobs[self.job_idx].ado_variables.extend(
1209 vars.into_iter()
1210 .map(|(k, v)| (k.as_ref().into(), v.as_ref().into())),
1211 );
1212 self
1213 }
1214
1215 /// Overrides the id of the job.
1216 ///
1217 /// Flowey typically generates a reasonable job ID but some use cases that depend
1218 /// on the ID may find it useful to override it to something custom.
1219 pub fn ado_override_job_id(self, name: impl AsRef<str>) -> Self {
1220 self.pipeline
1221 .ado_job_id_overrides
1222 .insert(self.job_idx, name.as_ref().into());
1223 self
1224 }
1225
1226 /// (GitHub Actions only) specify which Github runner this job will be run on.
1227 pub fn gh_set_pool(self, pool: GhRunner) -> Self {
1228 self.pipeline.jobs[self.job_idx].gh_pool = Some(pool);
1229 self
1230 }
1231
1232 /// (GitHub Actions only) Set the concurrency group for this job.
1233 pub fn gh_set_concurrency_group(self, group: GhConcurrencyGroup) -> Self {
1234 self.pipeline.jobs[self.job_idx].gh_concurrency_group = Some(group);
1235 self
1236 }
1237
1238 /// (GitHub Actions only) Manually override the `if:` condition for this
1239 /// particular job.
1240 ///
1241 /// **This is dangerous**, as an improperly set `if` condition may break
1242 /// downstream flowey jobs which assume flowey is in control of the job's
1243 /// scheduling logic.
1244 ///
1245 /// See
1246 /// <https://docs.github.com/en/actions/writing-workflows/workflow-syntax-for-github-actions#jobsjob_idif>
1247 /// for more info.
1248 pub fn gh_dangerous_override_if(self, condition: impl AsRef<str>) -> Self {
1249 self.pipeline.jobs[self.job_idx].gh_override_if = Some(condition.as_ref().into());
1250 self
1251 }
1252
1253 /// (GitHub Actions only) Declare a global job-level environment variable,
1254 /// visible to all downstream steps.
1255 ///
1256 /// `name` and `value` are both arbitrary strings, which may include GitHub
1257 /// Actions template expressions.
1258 ///
1259 /// **This is dangerous**, as it is easy to misuse this API in order to
1260 /// write a node which takes an implicit dependency on there being a global
1261 /// variable set on its behalf by the top-level pipeline code, making it
1262 /// difficult to "locally reason" about the behavior of a node simply by
1263 /// reading its code.
1264 ///
1265 /// Whenever possible, nodes should "late bind" environment variables:
1266 /// accepting a compile-time / runtime flowey parameter, and then setting it
1267 /// prior to executing a child command that requires it.
1268 ///
1269 /// Only use this API in exceptional cases, such as obtaining an environment
1270 /// variable whose value is determined by a job-level GitHub Actions
1271 /// expression evaluation.
1272 pub fn gh_dangerous_global_env_var(
1273 self,
1274 name: impl AsRef<str>,
1275 value: impl AsRef<str>,
1276 ) -> Self {
1277 let name = name.as_ref();
1278 let value = value.as_ref();
1279 self.pipeline.jobs[self.job_idx]
1280 .gh_global_env
1281 .insert(name.into(), value.into());
1282 self
1283 }
1284
1285 /// (GitHub Actions only) Grant permissions required by nodes in the job.
1286 ///
1287 /// For a given node handle, grant the specified permissions.
1288 /// The list provided must match the permissions specified within the node
1289 /// using `requires_permission`.
1290 ///
1291 /// NOTE: While this method is called at a node-level for auditability, the emitted
1292 /// yaml grants permissions at the job-level.
1293 ///
1294 /// This can lead to weird situations where node 1 might not specify a permission
1295 /// required according to Github Actions, but due to job-level granting of the permission
1296 /// by another node 2, the pipeline executes even though it wouldn't if node 2 was removed.
1297 ///
1298 /// For available permission scopes and their descriptions, see
1299 /// <https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#permissions>.
1300 pub fn gh_grant_permissions<N: FlowNodeBase + 'static>(
1301 self,
1302 permissions: impl IntoIterator<Item = (GhPermission, GhPermissionValue)>,
1303 ) -> Self {
1304 let node_handle = NodeHandle::from_type::<N>();
1305 for (permission, value) in permissions {
1306 self.pipeline.jobs[self.job_idx]
1307 .gh_permissions
1308 .entry(node_handle)
1309 .or_default()
1310 .insert(permission, value);
1311 }
1312 self
1313 }
1314
1315 pub fn apply_patchfn(self, patchfn: crate::patch::PatchFn) -> Self {
1316 self.pipeline.jobs[self.job_idx]
1317 .patches
1318 .apply_patchfn(patchfn);
1319 self
1320 }
1321
1322 /// Set a timeout for the job, in minutes.
1323 ///
1324 /// Not calling this will result in the platform's default timeout being used,
1325 /// which is typically 60 minutes, but may vary.
1326 pub fn with_timeout_in_minutes(self, timeout: u32) -> Self {
1327 self.pipeline.jobs[self.job_idx].timeout_minutes = Some(timeout);
1328 self
1329 }
1330
1331 /// (ADO+Local Only) Only run the job if the specified condition is true.
1332 pub fn with_condition(self, cond: UseParameter<bool>) -> Self {
1333 self.pipeline.jobs[self.job_idx].cond_param_idx = Some(cond.idx);
1334 self.pipeline.parameters[cond.idx]
1335 .used_by_jobs
1336 .insert(self.job_idx);
1337 self
1338 }
1339
1340 /// Set a [`CommandWrapperKind`] that will be applied to all shell
1341 /// commands executed in this job's steps.
1342 ///
1343 /// The wrapper is applied both when running locally (via direct run)
1344 /// and when running in CI (the kind is serialized into
1345 /// `pipeline.json` and reconstructed at runtime).
1346 ///
1347 /// [`CommandWrapperKind`]: crate::shell::CommandWrapperKind
1348 pub fn set_command_wrapper(self, wrapper: crate::shell::CommandWrapperKind) -> Self {
1349 self.pipeline.jobs[self.job_idx].command_wrapper = Some(wrapper);
1350 self
1351 }
1352
1353 /// Add a flow node which will be run as part of the job.
1354 pub fn dep_on<R: IntoRequest + 'static>(
1355 self,
1356 f: impl FnOnce(&mut PipelineJobCtx<'_>) -> R,
1357 ) -> Self {
1358 // JobToNodeCtx will ensure artifact deps are taken care of
1359 let req = f(&mut PipelineJobCtx {
1360 pipeline: self.pipeline,
1361 job_idx: self.job_idx,
1362 });
1363
1364 self.pipeline.jobs[self.job_idx]
1365 .root_nodes
1366 .entry(NodeHandle::from_type::<R::Node>())
1367 .or_default()
1368 .push(serde_json::to_vec(&req.into_request()).unwrap().into());
1369
1370 self
1371 }
1372
1373 /// Add a flow node whose request publishes a typed artifact.
1374 ///
1375 /// This is a shortcut for the common pattern of calling
1376 /// [`PipelineJobCtx::publish_typed_artifact`] inside a [`Self::dep_on`]
1377 /// closure and passing the resulting [`WriteVar`] into a request.
1378 pub fn publish<T: Artifact, R: IntoRequest + 'static>(
1379 self,
1380 artifact: PublishTypedArtifact<T>,
1381 f: impl FnOnce(WriteVar<T>) -> R,
1382 ) -> Self {
1383 self.dep_on(|ctx| f(ctx.publish_typed_artifact(artifact)))
1384 }
1385
1386 /// Add a flow node whose request is run purely for its side effect.
1387 ///
1388 /// This is a shortcut for the common pattern of calling
1389 /// [`PipelineJobCtx::new_done_handle`] inside a [`Self::dep_on`]
1390 /// closure and passing the resulting [`WriteVar`] into a request.
1391 pub fn side_effect<R: IntoRequest + 'static>(
1392 self,
1393 f: impl FnOnce(WriteVar<crate::node::SideEffect>) -> R,
1394 ) -> Self {
1395 self.dep_on(|ctx| f(ctx.new_done_handle()))
1396 }
1397
1398 /// Set config on a node for this job.
1399 ///
1400 /// This is the pipeline-level equivalent of [`NodeCtx::config`]. Config
1401 /// set here is merged with any config set by nodes within the job.
1402 ///
1403 /// [`NodeCtx::config`]: crate::node::NodeCtx::config
1404 pub fn config<C: IntoConfig + 'static>(self, config: C) -> Self {
1405 self.pipeline.jobs[self.job_idx]
1406 .root_configs
1407 .entry(NodeHandle::from_type::<C::Node>())
1408 .or_default()
1409 .push(serde_json::to_vec(&config).unwrap().into());
1410
1411 self
1412 }
1413
1414 /// Finish describing the pipeline job.
1415 pub fn finish(self) -> PipelineJobHandle {
1416 PipelineJobHandle {
1417 job_idx: self.job_idx,
1418 }
1419 }
1420
1421 /// Return the job's platform.
1422 pub fn get_platform(&self) -> FlowPlatform {
1423 self.pipeline.jobs[self.job_idx].platform
1424 }
1425
1426 /// Return the job's architecture.
1427 pub fn get_arch(&self) -> FlowArch {
1428 self.pipeline.jobs[self.job_idx].arch
1429 }
1430}
1431
1432#[derive(Clone)]
1433pub struct PipelineJobHandle {
1434 job_idx: usize,
1435}
1436
1437impl PipelineJobHandle {
1438 pub fn is_handle_for(&self, job: &PipelineJob<'_>) -> bool {
1439 self.job_idx == job.job_idx
1440 }
1441}
1442
1443#[derive(Clone, Copy)]
1444pub enum PipelineBackendHint {
1445 /// Pipeline is being run on the user's dev machine (via bash / direct run)
1446 Local,
1447 /// Pipeline is run on ADO
1448 Ado,
1449 /// Pipeline is run on GitHub Actions
1450 Github,
1451}
1452
1453/// Trait for types that can be converted into a [`Pipeline`].
1454///
1455/// This is the primary entry point for defining flowey pipelines. Implement this trait
1456/// to create a pipeline definition that can be executed locally or converted to CI YAML.
1457///
1458/// # Example
1459///
1460/// ```rust,no_run
1461/// use flowey_core::pipeline::{IntoPipeline, Pipeline, PipelineBackendHint};
1462/// use flowey_core::node::{FlowPlatform, FlowPlatformLinuxDistro, FlowArch};
1463///
1464/// struct MyPipeline;
1465///
1466/// impl IntoPipeline for MyPipeline {
1467/// fn into_pipeline(self, backend_hint: PipelineBackendHint) -> anyhow::Result<Pipeline> {
1468/// let mut pipeline = Pipeline::new();
1469///
1470/// // Define a job that runs on Linux x86_64
1471/// let _job = pipeline
1472/// .new_job(
1473/// FlowPlatform::Linux(FlowPlatformLinuxDistro::Ubuntu),
1474/// FlowArch::X86_64,
1475/// "build"
1476/// )
1477/// .finish();
1478///
1479/// Ok(pipeline)
1480/// }
1481/// }
1482/// ```
1483///
1484/// # Complex Example with Parameters and Artifacts
1485///
1486/// ```rust,ignore
1487/// use flowey_core::pipeline::{IntoPipeline, Pipeline, PipelineBackendHint, ParameterKind};
1488/// use flowey_core::node::{FlowPlatform, FlowPlatformLinuxDistro, FlowArch};
1489///
1490/// struct BuildPipeline;
1491///
1492/// impl IntoPipeline for BuildPipeline {
1493/// fn into_pipeline(self, backend_hint: PipelineBackendHint) -> anyhow::Result<Pipeline> {
1494/// let mut pipeline = Pipeline::new();
1495///
1496/// // Define a runtime parameter
1497/// let enable_tests = pipeline.new_parameter_bool(
1498/// "enable_tests",
1499/// "Whether to run tests",
1500/// ParameterKind::Stable,
1501/// Some(true) // default value
1502/// );
1503///
1504/// // Create an artifact for passing data between jobs
1505/// let (publish_build, use_build) = pipeline.new_artifact("build-output");
1506///
1507/// // Job 1: Build
1508/// let build_job = pipeline
1509/// .new_job(
1510/// FlowPlatform::Linux(FlowPlatformLinuxDistro::Ubuntu),
1511/// FlowArch::X86_64,
1512/// "build"
1513/// )
1514/// .with_timeout_in_minutes(30)
1515/// .dep_on(|ctx| flowey_lib_hvlite::_jobs::example_node::Request {
1516/// output_dir: ctx.publish_artifact(publish_build),
1517/// })
1518/// .finish();
1519///
1520/// // Job 2: Test (conditionally run based on parameter)
1521/// let _test_job = pipeline
1522/// .new_job(
1523/// FlowPlatform::Linux(FlowPlatformLinuxDistro::Ubuntu),
1524/// FlowArch::X86_64,
1525/// "test"
1526/// )
1527/// .with_condition(enable_tests)
1528/// .dep_on(|ctx| flowey_lib_hvlite::_jobs::example_node2::Request {
1529/// input_dir: ctx.use_artifact(&use_build),
1530/// })
1531/// .finish();
1532///
1533/// Ok(pipeline)
1534/// }
1535/// }
1536/// ```
1537pub trait IntoPipeline {
1538 fn into_pipeline(self, backend_hint: PipelineBackendHint) -> anyhow::Result<Pipeline>;
1539}
1540
1541fn new_parameter_name(name: impl AsRef<str>, kind: ParameterKind) -> String {
1542 match kind {
1543 ParameterKind::Unstable => format!("__unstable_{}", name.as_ref()),
1544 ParameterKind::Stable => name.as_ref().into(),
1545 }
1546}
1547
1548/// Structs which should only be used by top-level flowey emitters. If you're a
1549/// pipeline author, these are not types you need to care about!
1550pub mod internal {
1551 use super::*;
1552 use std::collections::BTreeMap;
1553
1554 /// Generate an ADO bootstrap template for a job's platform and architecture.
1555 pub type AdoBootstrapTemplate = dyn Fn(FlowPlatform, FlowArch) -> anyhow::Result<String>;
1556
1557 /// Generate a GitHub bootstrap template for a job's platform and architecture.
1558 pub type GhBootstrapTemplate = dyn Fn(FlowPlatform, FlowArch) -> anyhow::Result<String>;
1559
1560 pub fn consistent_artifact_runtime_var_name(artifact: impl AsRef<str>, is_use: bool) -> String {
1561 format!(
1562 "artifact_{}_{}",
1563 if is_use { "use_from" } else { "publish_from" },
1564 artifact.as_ref()
1565 )
1566 }
1567
1568 #[derive(Debug)]
1569 pub struct InternalAdoResourcesRepository {
1570 /// flowey-generated unique repo identifier
1571 pub repo_id: String,
1572 /// Type of repo that is being connected to.
1573 pub repo_type: AdoResourcesRepositoryType,
1574 /// Repository name. Format depends on `repo_type`.
1575 pub name: String,
1576 /// git ref to checkout.
1577 pub git_ref: AdoResourcesRepositoryRef<usize>,
1578 /// (optional) ID of the service endpoint connecting to this repository.
1579 pub endpoint: Option<String>,
1580 }
1581
1582 pub struct PipelineJobMetadata {
1583 pub root_nodes: BTreeMap<NodeHandle, Vec<Box<[u8]>>>,
1584 pub root_configs: BTreeMap<NodeHandle, Vec<Box<[u8]>>>,
1585 pub patches: PatchResolver,
1586 pub label: String,
1587 pub platform: FlowPlatform,
1588 pub arch: FlowArch,
1589 pub cond_param_idx: Option<usize>,
1590 pub timeout_minutes: Option<u32>,
1591 pub command_wrapper: Option<crate::shell::CommandWrapperKind>,
1592 // backend specific
1593 pub ado_pool: Option<AdoPool>,
1594 pub ado_variables: BTreeMap<String, String>,
1595 pub gh_override_if: Option<String>,
1596 pub gh_pool: Option<GhRunner>,
1597 pub gh_concurrency_group: Option<GhConcurrencyGroup>,
1598 pub gh_global_env: BTreeMap<String, String>,
1599 pub gh_permissions: BTreeMap<NodeHandle, BTreeMap<GhPermission, GhPermissionValue>>,
1600 }
1601
1602 #[derive(Debug)]
1603 pub struct ArtifactMeta {
1604 pub name: String,
1605 pub published_by_job: Option<usize>,
1606 pub used_by_jobs: BTreeSet<usize>,
1607 }
1608
1609 #[derive(Debug)]
1610 pub struct ParameterMeta {
1611 pub parameter: Parameter,
1612 pub used_by_jobs: BTreeSet<usize>,
1613 }
1614
1615 /// Mirror of [`Pipeline`], except with all field marked as `pub`.
1616 pub struct PipelineFinalized {
1617 pub jobs: Vec<PipelineJobMetadata>,
1618 pub artifacts: Vec<ArtifactMeta>,
1619 pub parameters: Vec<ParameterMeta>,
1620 pub extra_deps: BTreeSet<(usize, usize)>,
1621 // backend specific
1622 pub ado_name: Option<String>,
1623 pub ado_schedule_triggers: Vec<AdoScheduleTriggers>,
1624 pub ado_ci_triggers: Option<AdoCiTriggers>,
1625 pub ado_pr_triggers: Option<AdoPrTriggers>,
1626 pub ado_bootstrap_template: Option<Box<AdoBootstrapTemplate>>,
1627 pub ado_resources_repository: Vec<InternalAdoResourcesRepository>,
1628 pub ado_post_process_yaml_cb:
1629 Option<Box<dyn FnOnce(serde_yaml::Value) -> serde_yaml::Value>>,
1630 pub ado_variables: BTreeMap<String, String>,
1631 pub ado_job_id_overrides: BTreeMap<usize, String>,
1632 pub gh_name: Option<String>,
1633 pub gh_schedule_triggers: Vec<GhScheduleTriggers>,
1634 pub gh_ci_triggers: Option<GhCiTriggers>,
1635 pub gh_pr_triggers: Option<GhPrTriggers>,
1636 pub gh_bootstrap_template: Option<Box<GhBootstrapTemplate>>,
1637 }
1638
1639 impl PipelineFinalized {
1640 pub fn from_pipeline(mut pipeline: Pipeline) -> Self {
1641 if let Some(cb) = pipeline.inject_all_jobs_with.take() {
1642 for job_idx in 0..pipeline.jobs.len() {
1643 let _ = cb(PipelineJob {
1644 pipeline: &mut pipeline,
1645 job_idx,
1646 });
1647 }
1648 }
1649
1650 let Pipeline {
1651 mut jobs,
1652 artifacts,
1653 parameters,
1654 extra_deps,
1655 ado_name,
1656 ado_bootstrap_template,
1657 ado_schedule_triggers,
1658 ado_ci_triggers,
1659 ado_pr_triggers,
1660 ado_resources_repository,
1661 ado_post_process_yaml_cb,
1662 ado_variables,
1663 ado_job_id_overrides,
1664 gh_name,
1665 gh_schedule_triggers,
1666 gh_ci_triggers,
1667 gh_pr_triggers,
1668 gh_bootstrap_template,
1669 // not relevant to consumer code
1670 dummy_done_idx: _,
1671 artifact_map_idx: _,
1672 artifact_names: _,
1673 global_patchfns,
1674 inject_all_jobs_with: _, // processed above
1675 } = pipeline;
1676
1677 for patchfn in global_patchfns {
1678 for job in &mut jobs {
1679 job.patches.apply_patchfn(patchfn)
1680 }
1681 }
1682
1683 Self {
1684 jobs,
1685 artifacts,
1686 parameters,
1687 extra_deps,
1688 ado_name,
1689 ado_schedule_triggers,
1690 ado_ci_triggers,
1691 ado_pr_triggers,
1692 ado_bootstrap_template,
1693 ado_resources_repository,
1694 ado_post_process_yaml_cb,
1695 ado_variables,
1696 ado_job_id_overrides,
1697 gh_name,
1698 gh_schedule_triggers,
1699 gh_ci_triggers,
1700 gh_pr_triggers,
1701 gh_bootstrap_template,
1702 }
1703 }
1704 }
1705
1706 #[derive(Debug, Clone)]
1707 pub enum Parameter {
1708 Bool {
1709 name: String,
1710 description: String,
1711 kind: ParameterKind,
1712 default: Option<bool>,
1713 },
1714 String {
1715 name: String,
1716 description: String,
1717 default: Option<String>,
1718 kind: ParameterKind,
1719 possible_values: Option<Vec<String>>,
1720 },
1721 Num {
1722 name: String,
1723 description: String,
1724 default: Option<i64>,
1725 kind: ParameterKind,
1726 possible_values: Option<Vec<i64>>,
1727 },
1728 }
1729
1730 impl Parameter {
1731 pub fn name(&self) -> &str {
1732 match self {
1733 Parameter::Bool { name, .. } => name,
1734 Parameter::String { name, .. } => name,
1735 Parameter::Num { name, .. } => name,
1736 }
1737 }
1738 }
1739}