flowey_core/pipeline.rs
1// Copyright (c) Microsoft Corporation.
2// Licensed under the MIT License.
3
4//! Core types and traits used to create and work with flowey pipelines.
5
6mod artifact;
7
8pub use artifact::Artifact;
9pub use artifact::ArtifactType;
10pub use artifact::resolve;
11
12use self::internal::*;
13use crate::node::FlowArch;
14use crate::node::FlowNodeBase;
15use crate::node::FlowPlatform;
16use crate::node::FlowPlatformLinuxDistro;
17use crate::node::GhUserSecretVar;
18use crate::node::IntoConfig;
19use crate::node::IntoRequest;
20use crate::node::NodeHandle;
21use crate::node::ReadVar;
22use crate::node::WriteVar;
23use crate::node::steps::ado::AdoResourcesRepositoryId;
24use crate::node::user_facing::AdoRuntimeVar;
25use crate::node::user_facing::GhPermission;
26use crate::node::user_facing::GhPermissionValue;
27use crate::patch::PatchResolver;
28use crate::patch::ResolvedPatches;
29use serde::Serialize;
30use serde::de::DeserializeOwned;
31use std::collections::BTreeMap;
32use std::collections::BTreeSet;
33use std::path::PathBuf;
34
35/// Pipeline types which are considered "user facing", and included in the
36/// `flowey` prelude.
37pub mod user_facing {
38 pub use super::AdoCiTriggers;
39 pub use super::AdoPool;
40 pub use super::AdoPrTriggers;
41 pub use super::AdoResourcesRepository;
42 pub use super::AdoResourcesRepositoryRef;
43 pub use super::AdoResourcesRepositoryType;
44 pub use super::AdoScheduleTriggers;
45 pub use super::GhCiTriggers;
46 pub use super::GhConcurrencyGroup;
47 pub use super::GhPrTriggers;
48 pub use super::GhRunner;
49 pub use super::GhRunnerOsLabel;
50 pub use super::GhScheduleTriggers;
51 pub use super::HostExt;
52 pub use super::IntoPipeline;
53 pub use super::ParameterKind;
54 pub use super::Pipeline;
55 pub use super::PipelineBackendHint;
56 pub use super::PipelineJob;
57 pub use super::PipelineJobCtx;
58 pub use super::PipelineJobHandle;
59 pub use super::PublishArtifact;
60 pub use super::PublishTypedArtifact;
61 pub use super::UseArtifact;
62 pub use super::UseParameter;
63 pub use super::UseTypedArtifact;
64 pub use crate::node::FlowArch;
65 pub use crate::node::FlowPlatform;
66}
67
68fn linux_distro() -> FlowPlatformLinuxDistro {
69 // Check for nix environment first - takes precedence over distro detection
70 if std::env::var("IN_NIX_SHELL").is_ok() {
71 return FlowPlatformLinuxDistro::Nix;
72 }
73
74 // A `nix develop` shell doesn't set `IN_NIX_SHELL`, but the PATH should include a nix store path
75 if std::env::var("PATH").is_ok_and(|path| path.contains("/nix/store")) {
76 return FlowPlatformLinuxDistro::Nix;
77 }
78
79 if let Ok(etc_os_release) = fs_err::read_to_string("/etc/os-release") {
80 if etc_os_release.contains("ID=ubuntu") {
81 FlowPlatformLinuxDistro::Ubuntu
82 } else if etc_os_release.contains("ID=fedora") {
83 FlowPlatformLinuxDistro::Fedora
84 } else if etc_os_release.contains("ID=azurelinux") || etc_os_release.contains("ID=mariner")
85 {
86 FlowPlatformLinuxDistro::AzureLinux
87 } else if etc_os_release.contains("ID=arch") {
88 FlowPlatformLinuxDistro::Arch
89 } else {
90 FlowPlatformLinuxDistro::Unknown
91 }
92 } else {
93 FlowPlatformLinuxDistro::Unknown
94 }
95}
96
97pub trait HostExt: Sized {
98 /// Return the value for the current host machine.
99 ///
100 /// Will panic on non-local backends.
101 fn host(backend_hint: PipelineBackendHint) -> Self;
102}
103
104impl HostExt for FlowPlatform {
105 /// Return the platform of the current host machine.
106 ///
107 /// Will panic on non-local backends.
108 fn host(backend_hint: PipelineBackendHint) -> Self {
109 if !matches!(backend_hint, PipelineBackendHint::Local) {
110 panic!("can only use `FlowPlatform::host` when defining a local-only pipeline");
111 }
112
113 if cfg!(target_os = "windows") {
114 Self::Windows
115 } else if cfg!(target_os = "linux") {
116 Self::Linux(linux_distro())
117 } else if cfg!(target_os = "macos") {
118 Self::MacOs
119 } else {
120 panic!("no valid host-os")
121 }
122 }
123}
124
125impl HostExt for FlowArch {
126 /// Return the arch of the current host machine.
127 ///
128 /// Will panic on non-local backends.
129 fn host(backend_hint: PipelineBackendHint) -> Self {
130 if !matches!(backend_hint, PipelineBackendHint::Local) {
131 panic!("can only use `FlowArch::host` when defining a local-only pipeline");
132 }
133
134 cfg_select! {
135 // xtask-fmt allow-target-arch oneoff-flowey
136 target_arch = "x86_64" => Self::X86_64,
137 // xtask-fmt allow-target-arch oneoff-flowey
138 target_arch = "aarch64" => Self::Aarch64,
139 _ => compile_error!("unsupported target architecture"),
140 }
141 }
142}
143
144/// Trigger ADO pipelines via Continuous Integration
145#[derive(Default, Debug)]
146pub struct AdoScheduleTriggers {
147 /// Friendly name for the scheduled run
148 pub display_name: String,
149 /// Run the pipeline whenever there is a commit on these specified branches
150 /// (supports glob syntax)
151 pub branches: Vec<String>,
152 /// Specify any branches which should be filtered out from the list of
153 /// `branches` (supports glob syntax)
154 pub exclude_branches: Vec<String>,
155 /// Run the pipeline in a schedule, as specified by a cron string
156 pub cron: String,
157}
158
159/// Trigger ADO pipelines per PR
160#[derive(Debug)]
161pub struct AdoPrTriggers {
162 /// Run the pipeline whenever there is a PR to these specified branches
163 /// (supports glob syntax)
164 pub branches: Vec<String>,
165 /// Specify any branches which should be filtered out from the list of
166 /// `branches` (supports glob syntax)
167 pub exclude_branches: Vec<String>,
168 /// Run the pipeline even if the PR is a draft PR. Defaults to `false`.
169 pub run_on_draft: bool,
170 /// Automatically cancel the pipeline run if a new commit lands in the
171 /// branch. Defaults to `true`.
172 pub auto_cancel: bool,
173 /// Only run the pipeline when files matching these paths are changed
174 /// (supports glob syntax)
175 pub paths: Vec<String>,
176 /// Specify any paths which should be filtered out from the list of
177 /// `paths` (supports glob syntax)
178 pub exclude_paths: Vec<String>,
179}
180
181/// Trigger ADO pipelines per CI
182#[derive(Debug, Default)]
183pub struct AdoCiTriggers {
184 /// Run the pipeline whenever there is a change to these specified branches
185 /// (supports glob syntax)
186 pub branches: Vec<String>,
187 /// Specify any branches which should be filtered out from the list of
188 /// `branches` (supports glob syntax)
189 pub exclude_branches: Vec<String>,
190 /// Run the pipeline whenever a matching tag is created (supports glob
191 /// syntax)
192 pub tags: Vec<String>,
193 /// Specify any tags which should be filtered out from the list of `tags`
194 /// (supports glob syntax)
195 pub exclude_tags: Vec<String>,
196 /// Whether to batch changes per branch.
197 pub batch: bool,
198 /// Only run the pipeline when files matching these paths are changed
199 /// (supports glob syntax)
200 pub paths: Vec<String>,
201 /// Specify any paths which should be filtered out from the list of
202 /// `paths` (supports glob syntax)
203 pub exclude_paths: Vec<String>,
204}
205
206impl Default for AdoPrTriggers {
207 fn default() -> Self {
208 Self {
209 branches: Vec::new(),
210 exclude_branches: Vec::new(),
211 run_on_draft: false,
212 auto_cancel: true,
213 paths: Vec::new(),
214 exclude_paths: Vec::new(),
215 }
216 }
217}
218
219/// ADO repository resource.
220#[derive(Debug)]
221pub struct AdoResourcesRepository {
222 /// Type of repo that is being connected to.
223 pub repo_type: AdoResourcesRepositoryType,
224 /// Repository name. Format depends on `repo_type`.
225 pub name: String,
226 /// git ref to checkout.
227 pub git_ref: AdoResourcesRepositoryRef,
228 /// (optional) ID of the service endpoint connecting to this repository.
229 pub endpoint: Option<String>,
230}
231
232/// ADO repository resource type
233#[derive(Debug)]
234pub enum AdoResourcesRepositoryType {
235 /// Azure Repos Git repository
236 AzureReposGit,
237 /// Github repository
238 GitHub,
239}
240
241/// ADO repository ref
242#[derive(Debug)]
243pub enum AdoResourcesRepositoryRef<P = UseParameter<String>> {
244 /// Hard-coded ref (e.g: refs/heads/main)
245 Fixed(String),
246 /// Connected to pipeline-level parameter
247 Parameter(P),
248}
249
250/// Trigger Github Actions pipelines via Continuous Integration
251///
252/// NOTE: Github Actions doesn't support specifying the branch when triggered by `schedule`.
253/// To run on a specific branch, modify the branch checked out in the pipeline.
254#[derive(Default, Debug)]
255pub struct GhScheduleTriggers {
256 /// Run the pipeline in a schedule, as specified by a cron string
257 pub cron: String,
258 /// IANA timezone string
259 pub timezone: Option<String>,
260}
261
262/// Trigger Github Actions pipelines per PR
263#[derive(Debug)]
264pub struct GhPrTriggers {
265 /// Run the pipeline whenever there is a PR to these specified branches
266 /// (supports glob syntax)
267 pub branches: Vec<String>,
268 /// Specify any branches which should be filtered out from the list of
269 /// `branches` (supports glob syntax)
270 pub exclude_branches: Vec<String>,
271 /// Automatically cancel the pipeline run if a new commit lands in the
272 /// branch. Defaults to `true`.
273 pub auto_cancel: bool,
274 /// Run the pipeline whenever the PR trigger matches the specified types
275 pub types: Vec<String>,
276 /// Only run the pipeline when files matching these paths are changed
277 /// (supports glob syntax)
278 pub paths: Vec<String>,
279 /// Specify any paths which should be filtered out from the list of
280 /// `paths` (supports glob syntax)
281 pub paths_ignore: Vec<String>,
282}
283
284/// Trigger Github Actions pipelines per PR
285#[derive(Debug, Default)]
286pub struct GhCiTriggers {
287 /// Run the pipeline whenever there is a change to these specified branches
288 /// (supports glob syntax)
289 pub branches: Vec<String>,
290 /// Specify any branches which should be filtered out from the list of
291 /// `branches` (supports glob syntax)
292 pub exclude_branches: Vec<String>,
293 /// Run the pipeline whenever a matching tag is created (supports glob
294 /// syntax)
295 pub tags: Vec<String>,
296 /// Specify any tags which should be filtered out from the list of `tags`
297 /// (supports glob syntax)
298 pub exclude_tags: Vec<String>,
299 /// Only run the pipeline when files matching these paths are changed
300 /// (supports glob syntax)
301 pub paths: Vec<String>,
302 /// Specify any paths which should be filtered out from the list of
303 /// `paths` (supports glob syntax)
304 pub paths_ignore: Vec<String>,
305 /// If set, only one pipeline run in this concurrency group runs at a time.
306 pub concurrency_group: Option<GhConcurrencyGroup>,
307}
308
309/// Settings for a Github concurrency group.
310#[derive(Debug, Default, Clone)]
311pub struct GhConcurrencyGroup {
312 /// The name of the concurrency group.
313 pub name: String,
314 /// Cancel the active run when a new run joins the concurrency group.
315 /// Defaults to `false`.
316 pub cancel_in_progress: bool,
317}
318
319impl GhPrTriggers {
320 /// Triggers the pipeline on the default PR events plus when a draft is marked as ready for review.
321 pub fn new_draftable() -> Self {
322 Self {
323 branches: Vec::new(),
324 exclude_branches: Vec::new(),
325 types: vec![
326 "opened".into(),
327 "synchronize".into(),
328 "reopened".into(),
329 "ready_for_review".into(),
330 ],
331 auto_cancel: true,
332 paths: Vec::new(),
333 paths_ignore: Vec::new(),
334 }
335 }
336}
337
338#[derive(Debug, Clone, PartialEq)]
339pub enum GhRunnerOsLabel {
340 UbuntuLatest,
341 Ubuntu2404,
342 Ubuntu2204,
343 WindowsLatest,
344 Windows2025,
345 Windows2022,
346 Ubuntu2404Arm,
347 Ubuntu2204Arm,
348 Windows11Arm,
349 Custom(String),
350}
351
352/// GitHub runner type
353#[derive(Debug, Clone, PartialEq)]
354pub enum GhRunner {
355 // See <https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#choosing-github-hosted-runners>
356 // for more details.
357 GhHosted(GhRunnerOsLabel),
358 // Self hosted runners are selected by matching runner labels to <labels>.
359 // 'self-hosted' is a common label for self hosted runners, but is not required.
360 // Labels are case-insensitive and can take the form of arbitrary strings.
361 // See <https://docs.github.com/en/actions/hosting-your-own-runners> for more details.
362 SelfHosted(Vec<String>),
363 // This uses a runner belonging to <group> that matches all labels in <labels>.
364 // See <https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#choosing-github-hosted-runners>
365 // for more details.
366 RunnerGroup { group: String, labels: Vec<String> },
367}
368
369impl GhRunner {
370 /// Whether this is a self-hosted runner with the provided label
371 pub fn is_self_hosted_with_label(&self, label: &str) -> bool {
372 matches!(self, GhRunner::SelfHosted(labels) if labels.iter().any(|s| s.as_str() == label))
373 }
374}
375
376// TODO: support a more structured format for demands
377// See https://learn.microsoft.com/en-us/azure/devops/pipelines/yaml-schema/pool-demands
378#[derive(Debug, Clone)]
379pub struct AdoPool {
380 pub name: String,
381 pub demands: Vec<String>,
382}
383
384/// Parameter type (unstable / stable).
385#[derive(Debug, Clone)]
386pub enum ParameterKind {
387 // The parameter is considered an unstable API, and should not be
388 // taken as a dependency.
389 Unstable,
390 // The parameter is considered a stable API, and can be used by
391 // external pipelines to control behavior of the pipeline.
392 Stable,
393}
394
395#[derive(Clone, Debug)]
396#[must_use]
397pub struct UseParameter<T> {
398 idx: usize,
399 _kind: std::marker::PhantomData<T>,
400}
401
402/// Opaque handle to an artifact which must be published by a single job.
403#[must_use]
404pub struct PublishArtifact {
405 idx: usize,
406}
407
408/// Opaque handle to an artifact which can be used by one or more jobs.
409#[derive(Clone)]
410#[must_use]
411pub struct UseArtifact {
412 idx: usize,
413}
414
415/// Opaque handle to an artifact of type `T` which must be published by a single job.
416#[must_use]
417pub struct PublishTypedArtifact<T>(PublishArtifact, std::marker::PhantomData<fn() -> T>);
418
419/// Opaque handle to an artifact of type `T` which can be used by one or more
420/// jobs.
421#[must_use]
422pub struct UseTypedArtifact<T>(UseArtifact, std::marker::PhantomData<fn(T)>);
423
424impl<T> Clone for UseTypedArtifact<T> {
425 fn clone(&self) -> Self {
426 UseTypedArtifact(self.0.clone(), std::marker::PhantomData)
427 }
428}
429
430#[derive(Default)]
431pub struct Pipeline {
432 jobs: Vec<PipelineJobMetadata>,
433 artifacts: Vec<ArtifactMeta>,
434 parameters: Vec<ParameterMeta>,
435 extra_deps: BTreeSet<(usize, usize)>,
436 // builder internal
437 artifact_names: BTreeSet<String>,
438 dummy_done_idx: usize,
439 artifact_map_idx: usize,
440 global_patchfns: Vec<crate::patch::PatchFn>,
441 inject_all_jobs_with: Option<Box<dyn for<'a> Fn(PipelineJob<'a>) -> PipelineJob<'a>>>,
442 // backend specific
443 ado_name: Option<String>,
444 ado_job_id_overrides: BTreeMap<usize, String>,
445 ado_schedule_triggers: Vec<AdoScheduleTriggers>,
446 ado_ci_triggers: Option<AdoCiTriggers>,
447 ado_pr_triggers: Option<AdoPrTriggers>,
448 ado_resources_repository: Vec<InternalAdoResourcesRepository>,
449 ado_bootstrap_template: Option<Box<AdoBootstrapTemplate>>,
450 ado_variables: BTreeMap<String, String>,
451 ado_post_process_yaml_cb: Option<Box<dyn FnOnce(serde_yaml::Value) -> serde_yaml::Value>>,
452 gh_name: Option<String>,
453 gh_schedule_triggers: Vec<GhScheduleTriggers>,
454 gh_ci_triggers: Option<GhCiTriggers>,
455 gh_pr_triggers: Option<GhPrTriggers>,
456 gh_bootstrap_template: Option<Box<GhBootstrapTemplate>>,
457}
458
459impl Pipeline {
460 pub fn new() -> Pipeline {
461 Pipeline::default()
462 }
463
464 /// Inject all pipeline jobs with some common logic. (e.g: to resolve common
465 /// configuration requirements shared by all jobs).
466 ///
467 /// Can only be invoked once per pipeline.
468 #[track_caller]
469 pub fn inject_all_jobs_with(
470 &mut self,
471 cb: impl for<'a> Fn(PipelineJob<'a>) -> PipelineJob<'a> + 'static,
472 ) -> &mut Self {
473 if self.inject_all_jobs_with.is_some() {
474 panic!("can only call inject_all_jobs_with once!")
475 }
476 self.inject_all_jobs_with = Some(Box::new(cb));
477 self
478 }
479
480 /// (ADO only) Provide a callback to provide a job-specific YAML template used to bootstrap flowey at the start
481 /// of an ADO pipeline.
482 ///
483 /// The callback receives the job's platform and architecture and is invoked
484 /// for each job that builds flowey, but not jobs that consume a prebuilt
485 /// binary.
486 ///
487 /// The template has access to the following vars, which will be statically
488 /// interpolated into the template's text:
489 ///
490 /// - `{{FLOWEY_OUTDIR}}`
491 /// - Directory to copy artifacts into.
492 /// - NOTE: this var will include `\` on Windows, and `/` on linux!
493 /// - `{{FLOWEY_BIN_EXTENSION}}`
494 /// - Extension of the expected flowey bin (either "", or ".exe")
495 /// - `{{FLOWEY_CRATE}}`
496 /// - Name of the project-specific flowey crate to be built
497 /// - `{{FLOWEY_TARGET}}`
498 /// - The target-triple flowey is being built for
499 /// - `{{FLOWEY_PIPELINE_PATH}}`
500 /// - Repo-root relative path to the pipeline (as provided when
501 /// generating the pipeline via the flowey CLI)
502 ///
503 /// The template's sole responsibility is to copy 3 files into the
504 /// `{{FLOWEY_OUTDIR}}`:
505 ///
506 /// 1. The bootstrapped flowey binary, with the file name
507 /// `flowey{{FLOWEY_BIN_EXTENSION}}`
508 /// 2. Two files called `pipeline.yaml` and `pipeline.json`, which are
509 /// copied of the pipeline YAML and pipeline JSON currently being run.
510 /// `{{FLOWEY_PIPELINE_PATH}}` is provided as a way to disambiguate in
511 /// cases where the same template is being for multiple pipelines (e.g: a
512 /// debug vs. release pipeline).
513 pub fn ado_set_flowey_bootstrap_template_fn(
514 &mut self,
515 template: impl Fn(FlowPlatform, FlowArch) -> anyhow::Result<String> + 'static,
516 ) -> &mut Self {
517 self.ado_bootstrap_template = Some(Box::new(template));
518 self
519 }
520
521 /// (ADO only) Provide a callback function which will be used to
522 /// post-process any YAML flowey generates for the pipeline.
523 ///
524 /// Until flowey defines a stable API for maintaining out-of-tree backends,
525 /// this method can be used to integrate the output from the generic ADO
526 /// backend with any organization-specific templates that one may be
527 /// required to use (e.g: for compliance reasons).
528 pub fn ado_post_process_yaml(
529 &mut self,
530 cb: impl FnOnce(serde_yaml::Value) -> serde_yaml::Value + 'static,
531 ) -> &mut Self {
532 self.ado_post_process_yaml_cb = Some(Box::new(cb));
533 self
534 }
535
536 /// (ADO only) Add a new scheduled CI trigger. Can be called multiple times
537 /// to set up multiple schedules runs.
538 pub fn ado_add_schedule_trigger(&mut self, triggers: AdoScheduleTriggers) -> &mut Self {
539 self.ado_schedule_triggers.push(triggers);
540 self
541 }
542
543 /// (ADO only) Set a PR trigger. Calling this method multiple times will
544 /// overwrite any previously set triggers.
545 pub fn ado_set_pr_triggers(&mut self, triggers: AdoPrTriggers) -> &mut Self {
546 self.ado_pr_triggers = Some(triggers);
547 self
548 }
549
550 /// (ADO only) Set a CI trigger. Calling this method multiple times will
551 /// overwrite any previously set triggers.
552 pub fn ado_set_ci_triggers(&mut self, triggers: AdoCiTriggers) -> &mut Self {
553 self.ado_ci_triggers = Some(triggers);
554 self
555 }
556
557 /// (ADO only) Declare a new repository resource, returning a type-safe
558 /// handle which downstream ADO steps are able to consume via
559 /// [`AdoStepServices::resolve_repository_id`](crate::node::user_facing::AdoStepServices::resolve_repository_id).
560 pub fn ado_add_resources_repository(
561 &mut self,
562 repo: AdoResourcesRepository,
563 ) -> AdoResourcesRepositoryId {
564 let AdoResourcesRepository {
565 repo_type,
566 name,
567 git_ref,
568 endpoint,
569 } = repo;
570
571 let repo_id = format!("repo{}", self.ado_resources_repository.len());
572
573 self.ado_resources_repository
574 .push(InternalAdoResourcesRepository {
575 repo_id: repo_id.clone(),
576 repo_type,
577 name,
578 git_ref: match git_ref {
579 AdoResourcesRepositoryRef::Fixed(s) => AdoResourcesRepositoryRef::Fixed(s),
580 AdoResourcesRepositoryRef::Parameter(p) => {
581 AdoResourcesRepositoryRef::Parameter(p.idx)
582 }
583 },
584 endpoint,
585 });
586 AdoResourcesRepositoryId { repo_id }
587 }
588
589 /// (GitHub Actions only) Set the pipeline-level name.
590 ///
591 /// <https://docs.github.com/en/actions/writing-workflows/workflow-syntax-for-github-actions#name>
592 pub fn gh_set_name(&mut self, name: impl AsRef<str>) -> &mut Self {
593 self.gh_name = Some(name.as_ref().into());
594 self
595 }
596
597 /// Provide a callback that provides a job-specific YAML template used to
598 /// bootstrap flowey at the start of a GitHub Actions pipeline.
599 ///
600 /// The callback receives the job's platform and architecture and is invoked
601 /// for each job that builds flowey, but not jobs that consume a prebuilt
602 /// binary.
603 ///
604 /// The template has access to the following vars, which will be statically
605 /// interpolated into the template's text:
606 ///
607 /// - `{{FLOWEY_OUTDIR}}`
608 /// - Directory to copy artifacts into.
609 /// - NOTE: this var will include `\` on Windows, and `/` on linux!
610 /// - `{{FLOWEY_BIN_EXTENSION}}`
611 /// - Extension of the expected flowey bin (either "", or ".exe")
612 /// - `{{FLOWEY_CRATE}}`
613 /// - Name of the project-specific flowey crate to be built
614 /// - `{{FLOWEY_TARGET}}`
615 /// - The target-triple flowey is being built for
616 /// - `{{FLOWEY_PIPELINE_PATH}}`
617 /// - Repo-root relative path to the pipeline (as provided when
618 /// generating the pipeline via the flowey CLI)
619 ///
620 /// The template's sole responsibility is to copy 3 files into the
621 /// `{{FLOWEY_OUTDIR}}`:
622 ///
623 /// 1. The bootstrapped flowey binary, with the file name
624 /// `flowey{{FLOWEY_BIN_EXTENSION}}`
625 /// 2. Two files called `pipeline.yaml` and `pipeline.json`, which are
626 /// copied of the pipeline YAML and pipeline JSON currently being run.
627 /// `{{FLOWEY_PIPELINE_PATH}}` is provided as a way to disambiguate in
628 /// cases where the same template is being for multiple pipelines (e.g: a
629 /// debug vs. release pipeline).
630 pub fn gh_set_flowey_bootstrap_template_fn(
631 &mut self,
632 template: impl Fn(FlowPlatform, FlowArch) -> anyhow::Result<String> + 'static,
633 ) -> &mut Self {
634 self.gh_bootstrap_template = Some(Box::new(template));
635 self
636 }
637
638 /// (GitHub Actions only) Add a new scheduled CI trigger. Can be called multiple times
639 /// to set up multiple schedules runs.
640 pub fn gh_add_schedule_trigger(&mut self, triggers: GhScheduleTriggers) -> &mut Self {
641 self.gh_schedule_triggers.push(triggers);
642 self
643 }
644
645 /// (GitHub Actions only) Set a PR trigger. Calling this method multiple times will
646 /// overwrite any previously set triggers.
647 pub fn gh_set_pr_triggers(&mut self, triggers: GhPrTriggers) -> &mut Self {
648 self.gh_pr_triggers = Some(triggers);
649 self
650 }
651
652 /// (GitHub Actions only) Set a CI trigger. Calling this method multiple times will
653 /// overwrite any previously set triggers.
654 pub fn gh_set_ci_triggers(&mut self, triggers: GhCiTriggers) -> &mut Self {
655 self.gh_ci_triggers = Some(triggers);
656 self
657 }
658
659 /// (GitHub Actions only) Use a pre-defined GitHub Actions secret variable.
660 ///
661 /// For more information on defining secrets for use in GitHub Actions, see
662 /// <https://docs.github.com/en/actions/security-guides/using-secrets-in-github-actions>
663 pub fn gh_use_secret(&mut self, secret_name: impl AsRef<str>) -> GhUserSecretVar {
664 GhUserSecretVar(secret_name.as_ref().to_string())
665 }
666
667 pub fn new_job(
668 &mut self,
669 platform: FlowPlatform,
670 arch: FlowArch,
671 label: impl AsRef<str>,
672 ) -> PipelineJob<'_> {
673 let idx = self.jobs.len();
674 self.jobs.push(PipelineJobMetadata {
675 root_nodes: BTreeMap::new(),
676 root_configs: BTreeMap::new(),
677 patches: ResolvedPatches::build(),
678 label: label.as_ref().into(),
679 platform,
680 arch,
681 cond_param_idx: None,
682 timeout_minutes: None,
683 command_wrapper: None,
684 ado_pool: None,
685 ado_variables: BTreeMap::new(),
686 gh_override_if: None,
687 gh_global_env: BTreeMap::new(),
688 gh_pool: None,
689 gh_concurrency_group: None,
690 gh_permissions: BTreeMap::new(),
691 });
692
693 PipelineJob {
694 pipeline: self,
695 job_idx: idx,
696 }
697 }
698
699 /// Declare a dependency between two jobs that does is not a result of an
700 /// artifact.
701 pub fn non_artifact_dep(
702 &mut self,
703 job: &PipelineJobHandle,
704 depends_on_job: &PipelineJobHandle,
705 ) -> &mut Self {
706 self.extra_deps
707 .insert((depends_on_job.job_idx, job.job_idx));
708 self
709 }
710
711 #[track_caller]
712 pub fn new_artifact(&mut self, name: impl AsRef<str>) -> (PublishArtifact, UseArtifact) {
713 let name = name.as_ref();
714 let owned_name = name.to_string();
715
716 let not_exists = self.artifact_names.insert(owned_name.clone());
717 if !not_exists {
718 panic!("duplicate artifact name: {}", name)
719 }
720
721 let idx = self.artifacts.len();
722 self.artifacts.push(ArtifactMeta {
723 name: owned_name,
724 published_by_job: None,
725 used_by_jobs: BTreeSet::new(),
726 });
727
728 (PublishArtifact { idx }, UseArtifact { idx })
729 }
730
731 /// Returns a pair of opaque handles to a new artifact for use across jobs
732 /// in the pipeline.
733 #[track_caller]
734 pub fn new_typed_artifact<T: Artifact>(
735 &mut self,
736 name: impl AsRef<str>,
737 ) -> (PublishTypedArtifact<T>, UseTypedArtifact<T>) {
738 let (publish, use_artifact) = self.new_artifact(name);
739 (
740 PublishTypedArtifact(publish, std::marker::PhantomData),
741 UseTypedArtifact(use_artifact, std::marker::PhantomData),
742 )
743 }
744
745 /// Returns a pair of sets of opaque handles to a new artifact for use
746 /// across jobs in the pipeline. The artifact names are derived by the impl
747 /// of [`ArtifactType::name`] using common prefixes and suffixes if
748 /// specified (although the implementor can choose to use those values
749 /// differently).
750 #[track_caller]
751 pub fn new_typed_artifact_collection<T: Artifact, U: ArtifactType>(
752 &mut self,
753 artifact_types: impl IntoIterator<Item = U>,
754 prefix: Option<&str>,
755 suffix: Option<&str>,
756 ) -> (
757 BTreeMap<U, PublishTypedArtifact<T>>,
758 BTreeMap<U, UseTypedArtifact<T>>,
759 ) {
760 artifact_types
761 .into_iter()
762 .map(|artifact_type| {
763 let (pub_artifact, use_artifact) =
764 self.new_typed_artifact(artifact_type.name(prefix, suffix));
765 (
766 (artifact_type.clone(), pub_artifact),
767 (artifact_type, use_artifact),
768 )
769 })
770 .unzip()
771 }
772
773 /// (ADO only) Set the pipeline-level name.
774 ///
775 /// <https://learn.microsoft.com/en-us/azure/devops/pipelines/process/run-number?view=azure-devops&tabs=yaml>
776 pub fn ado_add_name(&mut self, name: String) -> &mut Self {
777 self.ado_name = Some(name);
778 self
779 }
780
781 /// (ADO only) Declare a pipeline-level, named, read-only ADO variable.
782 ///
783 /// `name` and `value` are both arbitrary strings.
784 ///
785 /// Returns an instance of [`AdoRuntimeVar`], which, if need be, can be
786 /// converted into a [`ReadVar<String>`] using
787 /// [`NodeCtx::get_ado_variable`].
788 ///
789 /// NOTE: Unless required by some particular third-party task, it's strongly
790 /// recommended to _avoid_ using this method, and to simply use
791 /// [`ReadVar::from_static`] to get a obtain a static variable.
792 ///
793 /// [`NodeCtx::get_ado_variable`]: crate::node::NodeCtx::get_ado_variable
794 pub fn ado_new_named_variable(
795 &mut self,
796 name: impl AsRef<str>,
797 value: impl AsRef<str>,
798 ) -> AdoRuntimeVar {
799 let name = name.as_ref();
800 let value = value.as_ref();
801
802 self.ado_variables.insert(name.into(), value.into());
803
804 // safe, since we'll ensure that the global exists in the ADO backend
805 AdoRuntimeVar::dangerous_from_global(name, false)
806 }
807
808 /// (ADO only) Declare multiple pipeline-level, named, read-only ADO
809 /// variables at once.
810 ///
811 /// This is a convenience method to streamline invoking
812 /// [`Self::ado_new_named_variable`] multiple times.
813 ///
814 /// NOTE: Unless required by some particular third-party task, it's strongly
815 /// recommended to _avoid_ using this method, and to simply use
816 /// [`ReadVar::from_static`] to get a obtain a static variable.
817 ///
818 /// DEVNOTE: In the future, this API may be updated to return a handle that
819 /// will allow resolving the resulting `AdoRuntimeVar`, but for
820 /// implementation expediency, this API does not currently do this. If you
821 /// need to read the value of this variable at runtime, you may need to
822 /// invoke [`AdoRuntimeVar::dangerous_from_global`] manually.
823 ///
824 /// [`NodeCtx::get_ado_variable`]: crate::node::NodeCtx::get_ado_variable
825 pub fn ado_new_named_variables<K, V>(
826 &mut self,
827 vars: impl IntoIterator<Item = (K, V)>,
828 ) -> &mut Self
829 where
830 K: AsRef<str>,
831 V: AsRef<str>,
832 {
833 self.ado_variables.extend(
834 vars.into_iter()
835 .map(|(k, v)| (k.as_ref().into(), v.as_ref().into())),
836 );
837 self
838 }
839
840 /// Declare a pipeline-level runtime parameter with type `bool`.
841 ///
842 /// To obtain a [`ReadVar<bool>`] that can be used within a node, use the
843 /// [`PipelineJobCtx::use_parameter`] method.
844 ///
845 /// `name` is the name of the parameter.
846 ///
847 /// `description` is an arbitrary string, which will be be shown to users.
848 ///
849 /// `kind` is the type of parameter and if it should be treated as a stable
850 /// external API to callers of the pipeline.
851 ///
852 /// `default` is the default value for the parameter. If none is provided,
853 /// the parameter _must_ be specified in order for the pipeline to run.
854 ///
855 /// `possible_values` can be used to limit the set of valid values the
856 /// parameter accepts.
857 pub fn new_parameter_bool(
858 &mut self,
859 name: impl AsRef<str>,
860 description: impl AsRef<str>,
861 kind: ParameterKind,
862 default: Option<bool>,
863 ) -> UseParameter<bool> {
864 let idx = self.parameters.len();
865 let name = new_parameter_name(name, kind.clone());
866 self.parameters.push(ParameterMeta {
867 parameter: Parameter::Bool {
868 name,
869 description: description.as_ref().into(),
870 kind,
871 default,
872 },
873 used_by_jobs: BTreeSet::new(),
874 });
875
876 UseParameter {
877 idx,
878 _kind: std::marker::PhantomData,
879 }
880 }
881
882 /// Declare a pipeline-level runtime parameter with type `i64`.
883 ///
884 /// To obtain a [`ReadVar<i64>`] that can be used within a node, use the
885 /// [`PipelineJobCtx::use_parameter`] method.
886 ///
887 /// `name` is the name of the parameter.
888 ///
889 /// `description` is an arbitrary string, which will be be shown to users.
890 ///
891 /// `kind` is the type of parameter and if it should be treated as a stable
892 /// external API to callers of the pipeline.
893 ///
894 /// `default` is the default value for the parameter. If none is provided,
895 /// the parameter _must_ be specified in order for the pipeline to run.
896 ///
897 /// `possible_values` can be used to limit the set of valid values the
898 /// parameter accepts.
899 pub fn new_parameter_num(
900 &mut self,
901 name: impl AsRef<str>,
902 description: impl AsRef<str>,
903 kind: ParameterKind,
904 default: Option<i64>,
905 possible_values: Option<Vec<i64>>,
906 ) -> UseParameter<i64> {
907 let idx = self.parameters.len();
908 let name = new_parameter_name(name, kind.clone());
909 self.parameters.push(ParameterMeta {
910 parameter: Parameter::Num {
911 name,
912 description: description.as_ref().into(),
913 kind,
914 default,
915 possible_values,
916 },
917 used_by_jobs: BTreeSet::new(),
918 });
919
920 UseParameter {
921 idx,
922 _kind: std::marker::PhantomData,
923 }
924 }
925
926 /// Declare a pipeline-level runtime parameter with type `String`.
927 ///
928 /// To obtain a [`ReadVar<String>`] that can be used within a node, use the
929 /// [`PipelineJobCtx::use_parameter`] method.
930 ///
931 /// `name` is the name of the parameter.
932 ///
933 /// `description` is an arbitrary string, which will be be shown to users.
934 ///
935 /// `kind` is the type of parameter and if it should be treated as a stable
936 /// external API to callers of the pipeline.
937 ///
938 /// `default` is the default value for the parameter. If none is provided,
939 /// the parameter _must_ be specified in order for the pipeline to run.
940 ///
941 /// `possible_values` allows restricting inputs to a set of possible values.
942 /// Depending on the backend, these options may be presented as a set of
943 /// radio buttons, a dropdown menu, or something in that vein. If `None`,
944 /// then any string is allowed.
945 pub fn new_parameter_string(
946 &mut self,
947 name: impl AsRef<str>,
948 description: impl AsRef<str>,
949 kind: ParameterKind,
950 default: Option<impl AsRef<str>>,
951 possible_values: Option<Vec<String>>,
952 ) -> UseParameter<String> {
953 let idx = self.parameters.len();
954 let name = new_parameter_name(name, kind.clone());
955 self.parameters.push(ParameterMeta {
956 parameter: Parameter::String {
957 name,
958 description: description.as_ref().into(),
959 kind,
960 default: default.map(|x| x.as_ref().into()),
961 possible_values,
962 },
963 used_by_jobs: BTreeSet::new(),
964 });
965
966 UseParameter {
967 idx,
968 _kind: std::marker::PhantomData,
969 }
970 }
971}
972
973pub struct PipelineJobCtx<'a> {
974 pipeline: &'a mut Pipeline,
975 job_idx: usize,
976}
977
978impl PipelineJobCtx<'_> {
979 /// Create a new `WriteVar<SideEffect>` anchored to the pipeline job.
980 pub fn new_done_handle(&mut self) -> WriteVar<crate::node::SideEffect> {
981 self.pipeline.dummy_done_idx += 1;
982 crate::node::thin_air_write_runtime_var(format!("start{}", self.pipeline.dummy_done_idx))
983 }
984
985 /// Claim that this job will use this artifact, obtaining a path to a folder
986 /// with the artifact's contents.
987 pub fn use_artifact(&mut self, artifact: &UseArtifact) -> ReadVar<PathBuf> {
988 self.pipeline.artifacts[artifact.idx]
989 .used_by_jobs
990 .insert(self.job_idx);
991
992 crate::node::thin_air_read_runtime_var(consistent_artifact_runtime_var_name(
993 &self.pipeline.artifacts[artifact.idx].name,
994 true,
995 ))
996 }
997
998 /// Claim that this job will publish this artifact, obtaining a path to a
999 /// fresh, empty folder which will be published as the specific artifact at
1000 /// the end of the job.
1001 pub fn publish_artifact(&mut self, artifact: PublishArtifact) -> ReadVar<PathBuf> {
1002 let existing = self.pipeline.artifacts[artifact.idx]
1003 .published_by_job
1004 .replace(self.job_idx);
1005 assert!(existing.is_none()); // PublishArtifact isn't cloneable
1006
1007 crate::node::thin_air_read_runtime_var(consistent_artifact_runtime_var_name(
1008 &self.pipeline.artifacts[artifact.idx].name,
1009 false,
1010 ))
1011 }
1012
1013 fn helper_request<R: IntoRequest>(&mut self, req: R)
1014 where
1015 R::Node: 'static,
1016 {
1017 self.pipeline.jobs[self.job_idx]
1018 .root_nodes
1019 .entry(NodeHandle::from_type::<R::Node>())
1020 .or_default()
1021 .push(serde_json::to_vec(&req.into_request()).unwrap().into());
1022 }
1023
1024 fn new_artifact_map_vars<T: Artifact>(&mut self) -> (ReadVar<T>, WriteVar<T>) {
1025 let artifact_map_idx = self.pipeline.artifact_map_idx;
1026 self.pipeline.artifact_map_idx += 1;
1027
1028 let backing_var = format!("artifact_map{}", artifact_map_idx);
1029 let read_var = crate::node::thin_air_read_runtime_var(backing_var.clone());
1030 let write_var = crate::node::thin_air_write_runtime_var(backing_var);
1031 (read_var, write_var)
1032 }
1033
1034 /// Claim that this job will use this artifact, obtaining the resolved
1035 /// contents of the artifact.
1036 pub fn use_typed_artifact<T: Artifact>(
1037 &mut self,
1038 artifact: &UseTypedArtifact<T>,
1039 ) -> ReadVar<T> {
1040 let artifact_path = self.use_artifact(&artifact.0);
1041 let (read, write) = self.new_artifact_map_vars::<T>();
1042 self.helper_request(resolve::Request::new(artifact_path, write));
1043 read
1044 }
1045
1046 /// Claim that this job will publish this artifact, obtaining a variable to
1047 /// write the artifact's contents to. The artifact will be published at
1048 /// the end of the job.
1049 pub fn publish_typed_artifact<T: Artifact>(
1050 &mut self,
1051 artifact: PublishTypedArtifact<T>,
1052 ) -> WriteVar<T> {
1053 let artifact_path = self.publish_artifact(artifact.0);
1054 let (read, write) = self.new_artifact_map_vars::<T>();
1055 let done = self.new_done_handle();
1056 self.helper_request(artifact::publish::Request::new(read, artifact_path, done));
1057 write
1058 }
1059
1060 /// Obtain a `ReadVar<T>` corresponding to a pipeline parameter which is
1061 /// specified at runtime.
1062 pub fn use_parameter<T>(&mut self, param: UseParameter<T>) -> ReadVar<T>
1063 where
1064 T: Serialize + DeserializeOwned,
1065 {
1066 self.pipeline.parameters[param.idx]
1067 .used_by_jobs
1068 .insert(self.job_idx);
1069
1070 crate::node::thin_air_read_runtime_var(
1071 self.pipeline.parameters[param.idx]
1072 .parameter
1073 .name()
1074 .to_string(),
1075 )
1076 }
1077
1078 /// Shortcut which allows defining a bool pipeline parameter within a Job.
1079 ///
1080 /// To share a single parameter between multiple jobs, don't use this method
1081 /// - use [`Pipeline::new_parameter_bool`] + [`Self::use_parameter`] instead.
1082 pub fn new_parameter_bool(
1083 &mut self,
1084 name: impl AsRef<str>,
1085 description: impl AsRef<str>,
1086 kind: ParameterKind,
1087 default: Option<bool>,
1088 ) -> ReadVar<bool> {
1089 let param = self
1090 .pipeline
1091 .new_parameter_bool(name, description, kind, default);
1092 self.use_parameter(param)
1093 }
1094
1095 /// Shortcut which allows defining a number pipeline parameter within a Job.
1096 ///
1097 /// To share a single parameter between multiple jobs, don't use this method
1098 /// - use [`Pipeline::new_parameter_num`] + [`Self::use_parameter`] instead.
1099 pub fn new_parameter_num(
1100 &mut self,
1101 name: impl AsRef<str>,
1102 description: impl AsRef<str>,
1103 kind: ParameterKind,
1104 default: Option<i64>,
1105 possible_values: Option<Vec<i64>>,
1106 ) -> ReadVar<i64> {
1107 let param =
1108 self.pipeline
1109 .new_parameter_num(name, description, kind, default, possible_values);
1110 self.use_parameter(param)
1111 }
1112
1113 /// Shortcut which allows defining a string pipeline parameter within a Job.
1114 ///
1115 /// To share a single parameter between multiple jobs, don't use this method
1116 /// - use [`Pipeline::new_parameter_string`] + [`Self::use_parameter`] instead.
1117 pub fn new_parameter_string(
1118 &mut self,
1119 name: impl AsRef<str>,
1120 description: impl AsRef<str>,
1121 kind: ParameterKind,
1122 default: Option<String>,
1123 possible_values: Option<Vec<String>>,
1124 ) -> ReadVar<String> {
1125 let param =
1126 self.pipeline
1127 .new_parameter_string(name, description, kind, default, possible_values);
1128 self.use_parameter(param)
1129 }
1130}
1131
1132#[must_use]
1133pub struct PipelineJob<'a> {
1134 pipeline: &'a mut Pipeline,
1135 job_idx: usize,
1136}
1137
1138impl PipelineJob<'_> {
1139 /// (ADO only) specify which agent pool this job will be run on.
1140 pub fn ado_set_pool(self, pool: AdoPool) -> Self {
1141 self.pipeline.jobs[self.job_idx].ado_pool = Some(pool);
1142 self
1143 }
1144
1145 /// (ADO only) specify which agent pool this job will be run on, with
1146 /// additional special runner demands.
1147 pub fn ado_set_pool_with_demands(self, pool: impl AsRef<str>, demands: Vec<String>) -> Self {
1148 self.pipeline.jobs[self.job_idx].ado_pool = Some(AdoPool {
1149 name: pool.as_ref().into(),
1150 demands,
1151 });
1152 self
1153 }
1154
1155 /// (ADO only) Declare a job-level, named, read-only ADO variable.
1156 ///
1157 /// `name` and `value` are both arbitrary strings, which may include ADO
1158 /// template expressions.
1159 ///
1160 /// NOTE: Unless required by some particular third-party task, it's strongly
1161 /// recommended to _avoid_ using this method, and to simply use
1162 /// [`ReadVar::from_static`] to get a obtain a static variable.
1163 ///
1164 /// DEVNOTE: In the future, this API may be updated to return a handle that
1165 /// will allow resolving the resulting `AdoRuntimeVar`, but for
1166 /// implementation expediency, this API does not currently do this. If you
1167 /// need to read the value of this variable at runtime, you may need to
1168 /// invoke [`AdoRuntimeVar::dangerous_from_global`] manually.
1169 ///
1170 /// [`NodeCtx::get_ado_variable`]: crate::node::NodeCtx::get_ado_variable
1171 pub fn ado_new_named_variable(self, name: impl AsRef<str>, value: impl AsRef<str>) -> Self {
1172 let name = name.as_ref();
1173 let value = value.as_ref();
1174 self.pipeline.jobs[self.job_idx]
1175 .ado_variables
1176 .insert(name.into(), value.into());
1177 self
1178 }
1179
1180 /// (ADO only) Declare multiple job-level, named, read-only ADO variables at
1181 /// once.
1182 ///
1183 /// This is a convenience method to streamline invoking
1184 /// [`Self::ado_new_named_variable`] multiple times.
1185 ///
1186 /// NOTE: Unless required by some particular third-party task, it's strongly
1187 /// recommended to _avoid_ using this method, and to simply use
1188 /// [`ReadVar::from_static`] to get a obtain a static variable.
1189 ///
1190 /// DEVNOTE: In the future, this API may be updated to return a handle that
1191 /// will allow resolving the resulting `AdoRuntimeVar`, but for
1192 /// implementation expediency, this API does not currently do this. If you
1193 /// need to read the value of this variable at runtime, you may need to
1194 /// invoke [`AdoRuntimeVar::dangerous_from_global`] manually.
1195 ///
1196 /// [`NodeCtx::get_ado_variable`]: crate::node::NodeCtx::get_ado_variable
1197 pub fn ado_new_named_variables<K, V>(self, vars: impl IntoIterator<Item = (K, V)>) -> Self
1198 where
1199 K: AsRef<str>,
1200 V: AsRef<str>,
1201 {
1202 self.pipeline.jobs[self.job_idx].ado_variables.extend(
1203 vars.into_iter()
1204 .map(|(k, v)| (k.as_ref().into(), v.as_ref().into())),
1205 );
1206 self
1207 }
1208
1209 /// Overrides the id of the job.
1210 ///
1211 /// Flowey typically generates a reasonable job ID but some use cases that depend
1212 /// on the ID may find it useful to override it to something custom.
1213 pub fn ado_override_job_id(self, name: impl AsRef<str>) -> Self {
1214 self.pipeline
1215 .ado_job_id_overrides
1216 .insert(self.job_idx, name.as_ref().into());
1217 self
1218 }
1219
1220 /// (GitHub Actions only) specify which Github runner this job will be run on.
1221 pub fn gh_set_pool(self, pool: GhRunner) -> Self {
1222 self.pipeline.jobs[self.job_idx].gh_pool = Some(pool);
1223 self
1224 }
1225
1226 /// (GitHub Actions only) Set the concurrency group for this job.
1227 pub fn gh_set_concurrency_group(self, group: GhConcurrencyGroup) -> Self {
1228 self.pipeline.jobs[self.job_idx].gh_concurrency_group = Some(group);
1229 self
1230 }
1231
1232 /// (GitHub Actions only) Manually override the `if:` condition for this
1233 /// particular job.
1234 ///
1235 /// **This is dangerous**, as an improperly set `if` condition may break
1236 /// downstream flowey jobs which assume flowey is in control of the job's
1237 /// scheduling logic.
1238 ///
1239 /// See
1240 /// <https://docs.github.com/en/actions/writing-workflows/workflow-syntax-for-github-actions#jobsjob_idif>
1241 /// for more info.
1242 pub fn gh_dangerous_override_if(self, condition: impl AsRef<str>) -> Self {
1243 self.pipeline.jobs[self.job_idx].gh_override_if = Some(condition.as_ref().into());
1244 self
1245 }
1246
1247 /// (GitHub Actions only) Declare a global job-level environment variable,
1248 /// visible to all downstream steps.
1249 ///
1250 /// `name` and `value` are both arbitrary strings, which may include GitHub
1251 /// Actions template expressions.
1252 ///
1253 /// **This is dangerous**, as it is easy to misuse this API in order to
1254 /// write a node which takes an implicit dependency on there being a global
1255 /// variable set on its behalf by the top-level pipeline code, making it
1256 /// difficult to "locally reason" about the behavior of a node simply by
1257 /// reading its code.
1258 ///
1259 /// Whenever possible, nodes should "late bind" environment variables:
1260 /// accepting a compile-time / runtime flowey parameter, and then setting it
1261 /// prior to executing a child command that requires it.
1262 ///
1263 /// Only use this API in exceptional cases, such as obtaining an environment
1264 /// variable whose value is determined by a job-level GitHub Actions
1265 /// expression evaluation.
1266 pub fn gh_dangerous_global_env_var(
1267 self,
1268 name: impl AsRef<str>,
1269 value: impl AsRef<str>,
1270 ) -> Self {
1271 let name = name.as_ref();
1272 let value = value.as_ref();
1273 self.pipeline.jobs[self.job_idx]
1274 .gh_global_env
1275 .insert(name.into(), value.into());
1276 self
1277 }
1278
1279 /// (GitHub Actions only) Grant permissions required by nodes in the job.
1280 ///
1281 /// For a given node handle, grant the specified permissions.
1282 /// The list provided must match the permissions specified within the node
1283 /// using `requires_permission`.
1284 ///
1285 /// NOTE: While this method is called at a node-level for auditability, the emitted
1286 /// yaml grants permissions at the job-level.
1287 ///
1288 /// This can lead to weird situations where node 1 might not specify a permission
1289 /// required according to Github Actions, but due to job-level granting of the permission
1290 /// by another node 2, the pipeline executes even though it wouldn't if node 2 was removed.
1291 ///
1292 /// For available permission scopes and their descriptions, see
1293 /// <https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#permissions>.
1294 pub fn gh_grant_permissions<N: FlowNodeBase + 'static>(
1295 self,
1296 permissions: impl IntoIterator<Item = (GhPermission, GhPermissionValue)>,
1297 ) -> Self {
1298 let node_handle = NodeHandle::from_type::<N>();
1299 for (permission, value) in permissions {
1300 self.pipeline.jobs[self.job_idx]
1301 .gh_permissions
1302 .entry(node_handle)
1303 .or_default()
1304 .insert(permission, value);
1305 }
1306 self
1307 }
1308
1309 pub fn apply_patchfn(self, patchfn: crate::patch::PatchFn) -> Self {
1310 self.pipeline.jobs[self.job_idx]
1311 .patches
1312 .apply_patchfn(patchfn);
1313 self
1314 }
1315
1316 /// Set a timeout for the job, in minutes.
1317 ///
1318 /// Not calling this will result in the platform's default timeout being used,
1319 /// which is typically 60 minutes, but may vary.
1320 pub fn with_timeout_in_minutes(self, timeout: u32) -> Self {
1321 self.pipeline.jobs[self.job_idx].timeout_minutes = Some(timeout);
1322 self
1323 }
1324
1325 /// (ADO+Local Only) Only run the job if the specified condition is true.
1326 pub fn with_condition(self, cond: UseParameter<bool>) -> Self {
1327 self.pipeline.jobs[self.job_idx].cond_param_idx = Some(cond.idx);
1328 self.pipeline.parameters[cond.idx]
1329 .used_by_jobs
1330 .insert(self.job_idx);
1331 self
1332 }
1333
1334 /// Set a [`CommandWrapperKind`] that will be applied to all shell
1335 /// commands executed in this job's steps.
1336 ///
1337 /// The wrapper is applied both when running locally (via direct run)
1338 /// and when running in CI (the kind is serialized into
1339 /// `pipeline.json` and reconstructed at runtime).
1340 ///
1341 /// [`CommandWrapperKind`]: crate::shell::CommandWrapperKind
1342 pub fn set_command_wrapper(self, wrapper: crate::shell::CommandWrapperKind) -> Self {
1343 self.pipeline.jobs[self.job_idx].command_wrapper = Some(wrapper);
1344 self
1345 }
1346
1347 /// Add a flow node which will be run as part of the job.
1348 pub fn dep_on<R: IntoRequest + 'static>(
1349 self,
1350 f: impl FnOnce(&mut PipelineJobCtx<'_>) -> R,
1351 ) -> Self {
1352 // JobToNodeCtx will ensure artifact deps are taken care of
1353 let req = f(&mut PipelineJobCtx {
1354 pipeline: self.pipeline,
1355 job_idx: self.job_idx,
1356 });
1357
1358 self.pipeline.jobs[self.job_idx]
1359 .root_nodes
1360 .entry(NodeHandle::from_type::<R::Node>())
1361 .or_default()
1362 .push(serde_json::to_vec(&req.into_request()).unwrap().into());
1363
1364 self
1365 }
1366
1367 /// Add a flow node whose request publishes a typed artifact.
1368 ///
1369 /// This is a shortcut for the common pattern of calling
1370 /// [`PipelineJobCtx::publish_typed_artifact`] inside a [`Self::dep_on`]
1371 /// closure and passing the resulting [`WriteVar`] into a request.
1372 pub fn publish<T: Artifact, R: IntoRequest + 'static>(
1373 self,
1374 artifact: PublishTypedArtifact<T>,
1375 f: impl FnOnce(WriteVar<T>) -> R,
1376 ) -> Self {
1377 self.dep_on(|ctx| f(ctx.publish_typed_artifact(artifact)))
1378 }
1379
1380 /// Add a flow node whose request is run purely for its side effect.
1381 ///
1382 /// This is a shortcut for the common pattern of calling
1383 /// [`PipelineJobCtx::new_done_handle`] inside a [`Self::dep_on`]
1384 /// closure and passing the resulting [`WriteVar`] into a request.
1385 pub fn side_effect<R: IntoRequest + 'static>(
1386 self,
1387 f: impl FnOnce(WriteVar<crate::node::SideEffect>) -> R,
1388 ) -> Self {
1389 self.dep_on(|ctx| f(ctx.new_done_handle()))
1390 }
1391
1392 /// Set config on a node for this job.
1393 ///
1394 /// This is the pipeline-level equivalent of [`NodeCtx::config`]. Config
1395 /// set here is merged with any config set by nodes within the job.
1396 ///
1397 /// [`NodeCtx::config`]: crate::node::NodeCtx::config
1398 pub fn config<C: IntoConfig + 'static>(self, config: C) -> Self {
1399 self.pipeline.jobs[self.job_idx]
1400 .root_configs
1401 .entry(NodeHandle::from_type::<C::Node>())
1402 .or_default()
1403 .push(serde_json::to_vec(&config).unwrap().into());
1404
1405 self
1406 }
1407
1408 /// Finish describing the pipeline job.
1409 pub fn finish(self) -> PipelineJobHandle {
1410 PipelineJobHandle {
1411 job_idx: self.job_idx,
1412 }
1413 }
1414
1415 /// Return the job's platform.
1416 pub fn get_platform(&self) -> FlowPlatform {
1417 self.pipeline.jobs[self.job_idx].platform
1418 }
1419
1420 /// Return the job's architecture.
1421 pub fn get_arch(&self) -> FlowArch {
1422 self.pipeline.jobs[self.job_idx].arch
1423 }
1424}
1425
1426#[derive(Clone)]
1427pub struct PipelineJobHandle {
1428 job_idx: usize,
1429}
1430
1431impl PipelineJobHandle {
1432 pub fn is_handle_for(&self, job: &PipelineJob<'_>) -> bool {
1433 self.job_idx == job.job_idx
1434 }
1435}
1436
1437#[derive(Clone, Copy)]
1438pub enum PipelineBackendHint {
1439 /// Pipeline is being run on the user's dev machine (via bash / direct run)
1440 Local,
1441 /// Pipeline is run on ADO
1442 Ado,
1443 /// Pipeline is run on GitHub Actions
1444 Github,
1445}
1446
1447/// Trait for types that can be converted into a [`Pipeline`].
1448///
1449/// This is the primary entry point for defining flowey pipelines. Implement this trait
1450/// to create a pipeline definition that can be executed locally or converted to CI YAML.
1451///
1452/// # Example
1453///
1454/// ```rust,no_run
1455/// use flowey_core::pipeline::{IntoPipeline, Pipeline, PipelineBackendHint};
1456/// use flowey_core::node::{FlowPlatform, FlowPlatformLinuxDistro, FlowArch};
1457///
1458/// struct MyPipeline;
1459///
1460/// impl IntoPipeline for MyPipeline {
1461/// fn into_pipeline(self, backend_hint: PipelineBackendHint) -> anyhow::Result<Pipeline> {
1462/// let mut pipeline = Pipeline::new();
1463///
1464/// // Define a job that runs on Linux x86_64
1465/// let _job = pipeline
1466/// .new_job(
1467/// FlowPlatform::Linux(FlowPlatformLinuxDistro::Ubuntu),
1468/// FlowArch::X86_64,
1469/// "build"
1470/// )
1471/// .finish();
1472///
1473/// Ok(pipeline)
1474/// }
1475/// }
1476/// ```
1477///
1478/// # Complex Example with Parameters and Artifacts
1479///
1480/// ```rust,ignore
1481/// use flowey_core::pipeline::{IntoPipeline, Pipeline, PipelineBackendHint, ParameterKind};
1482/// use flowey_core::node::{FlowPlatform, FlowPlatformLinuxDistro, FlowArch};
1483///
1484/// struct BuildPipeline;
1485///
1486/// impl IntoPipeline for BuildPipeline {
1487/// fn into_pipeline(self, backend_hint: PipelineBackendHint) -> anyhow::Result<Pipeline> {
1488/// let mut pipeline = Pipeline::new();
1489///
1490/// // Define a runtime parameter
1491/// let enable_tests = pipeline.new_parameter_bool(
1492/// "enable_tests",
1493/// "Whether to run tests",
1494/// ParameterKind::Stable,
1495/// Some(true) // default value
1496/// );
1497///
1498/// // Create an artifact for passing data between jobs
1499/// let (publish_build, use_build) = pipeline.new_artifact("build-output");
1500///
1501/// // Job 1: Build
1502/// let build_job = pipeline
1503/// .new_job(
1504/// FlowPlatform::Linux(FlowPlatformLinuxDistro::Ubuntu),
1505/// FlowArch::X86_64,
1506/// "build"
1507/// )
1508/// .with_timeout_in_minutes(30)
1509/// .dep_on(|ctx| flowey_lib_hvlite::_jobs::example_node::Request {
1510/// output_dir: ctx.publish_artifact(publish_build),
1511/// })
1512/// .finish();
1513///
1514/// // Job 2: Test (conditionally run based on parameter)
1515/// let _test_job = pipeline
1516/// .new_job(
1517/// FlowPlatform::Linux(FlowPlatformLinuxDistro::Ubuntu),
1518/// FlowArch::X86_64,
1519/// "test"
1520/// )
1521/// .with_condition(enable_tests)
1522/// .dep_on(|ctx| flowey_lib_hvlite::_jobs::example_node2::Request {
1523/// input_dir: ctx.use_artifact(&use_build),
1524/// })
1525/// .finish();
1526///
1527/// Ok(pipeline)
1528/// }
1529/// }
1530/// ```
1531pub trait IntoPipeline {
1532 fn into_pipeline(self, backend_hint: PipelineBackendHint) -> anyhow::Result<Pipeline>;
1533}
1534
1535fn new_parameter_name(name: impl AsRef<str>, kind: ParameterKind) -> String {
1536 match kind {
1537 ParameterKind::Unstable => format!("__unstable_{}", name.as_ref()),
1538 ParameterKind::Stable => name.as_ref().into(),
1539 }
1540}
1541
1542/// Structs which should only be used by top-level flowey emitters. If you're a
1543/// pipeline author, these are not types you need to care about!
1544pub mod internal {
1545 use super::*;
1546 use std::collections::BTreeMap;
1547
1548 /// Generate an ADO bootstrap template for a job's platform and architecture.
1549 pub type AdoBootstrapTemplate = dyn Fn(FlowPlatform, FlowArch) -> anyhow::Result<String>;
1550
1551 /// Generate a GitHub bootstrap template for a job's platform and architecture.
1552 pub type GhBootstrapTemplate = dyn Fn(FlowPlatform, FlowArch) -> anyhow::Result<String>;
1553
1554 pub fn consistent_artifact_runtime_var_name(artifact: impl AsRef<str>, is_use: bool) -> String {
1555 format!(
1556 "artifact_{}_{}",
1557 if is_use { "use_from" } else { "publish_from" },
1558 artifact.as_ref()
1559 )
1560 }
1561
1562 #[derive(Debug)]
1563 pub struct InternalAdoResourcesRepository {
1564 /// flowey-generated unique repo identifier
1565 pub repo_id: String,
1566 /// Type of repo that is being connected to.
1567 pub repo_type: AdoResourcesRepositoryType,
1568 /// Repository name. Format depends on `repo_type`.
1569 pub name: String,
1570 /// git ref to checkout.
1571 pub git_ref: AdoResourcesRepositoryRef<usize>,
1572 /// (optional) ID of the service endpoint connecting to this repository.
1573 pub endpoint: Option<String>,
1574 }
1575
1576 pub struct PipelineJobMetadata {
1577 pub root_nodes: BTreeMap<NodeHandle, Vec<Box<[u8]>>>,
1578 pub root_configs: BTreeMap<NodeHandle, Vec<Box<[u8]>>>,
1579 pub patches: PatchResolver,
1580 pub label: String,
1581 pub platform: FlowPlatform,
1582 pub arch: FlowArch,
1583 pub cond_param_idx: Option<usize>,
1584 pub timeout_minutes: Option<u32>,
1585 pub command_wrapper: Option<crate::shell::CommandWrapperKind>,
1586 // backend specific
1587 pub ado_pool: Option<AdoPool>,
1588 pub ado_variables: BTreeMap<String, String>,
1589 pub gh_override_if: Option<String>,
1590 pub gh_pool: Option<GhRunner>,
1591 pub gh_concurrency_group: Option<GhConcurrencyGroup>,
1592 pub gh_global_env: BTreeMap<String, String>,
1593 pub gh_permissions: BTreeMap<NodeHandle, BTreeMap<GhPermission, GhPermissionValue>>,
1594 }
1595
1596 #[derive(Debug)]
1597 pub struct ArtifactMeta {
1598 pub name: String,
1599 pub published_by_job: Option<usize>,
1600 pub used_by_jobs: BTreeSet<usize>,
1601 }
1602
1603 #[derive(Debug)]
1604 pub struct ParameterMeta {
1605 pub parameter: Parameter,
1606 pub used_by_jobs: BTreeSet<usize>,
1607 }
1608
1609 /// Mirror of [`Pipeline`], except with all field marked as `pub`.
1610 pub struct PipelineFinalized {
1611 pub jobs: Vec<PipelineJobMetadata>,
1612 pub artifacts: Vec<ArtifactMeta>,
1613 pub parameters: Vec<ParameterMeta>,
1614 pub extra_deps: BTreeSet<(usize, usize)>,
1615 // backend specific
1616 pub ado_name: Option<String>,
1617 pub ado_schedule_triggers: Vec<AdoScheduleTriggers>,
1618 pub ado_ci_triggers: Option<AdoCiTriggers>,
1619 pub ado_pr_triggers: Option<AdoPrTriggers>,
1620 pub ado_bootstrap_template: Option<Box<AdoBootstrapTemplate>>,
1621 pub ado_resources_repository: Vec<InternalAdoResourcesRepository>,
1622 pub ado_post_process_yaml_cb:
1623 Option<Box<dyn FnOnce(serde_yaml::Value) -> serde_yaml::Value>>,
1624 pub ado_variables: BTreeMap<String, String>,
1625 pub ado_job_id_overrides: BTreeMap<usize, String>,
1626 pub gh_name: Option<String>,
1627 pub gh_schedule_triggers: Vec<GhScheduleTriggers>,
1628 pub gh_ci_triggers: Option<GhCiTriggers>,
1629 pub gh_pr_triggers: Option<GhPrTriggers>,
1630 pub gh_bootstrap_template: Option<Box<GhBootstrapTemplate>>,
1631 }
1632
1633 impl PipelineFinalized {
1634 pub fn from_pipeline(mut pipeline: Pipeline) -> Self {
1635 if let Some(cb) = pipeline.inject_all_jobs_with.take() {
1636 for job_idx in 0..pipeline.jobs.len() {
1637 let _ = cb(PipelineJob {
1638 pipeline: &mut pipeline,
1639 job_idx,
1640 });
1641 }
1642 }
1643
1644 let Pipeline {
1645 mut jobs,
1646 artifacts,
1647 parameters,
1648 extra_deps,
1649 ado_name,
1650 ado_bootstrap_template,
1651 ado_schedule_triggers,
1652 ado_ci_triggers,
1653 ado_pr_triggers,
1654 ado_resources_repository,
1655 ado_post_process_yaml_cb,
1656 ado_variables,
1657 ado_job_id_overrides,
1658 gh_name,
1659 gh_schedule_triggers,
1660 gh_ci_triggers,
1661 gh_pr_triggers,
1662 gh_bootstrap_template,
1663 // not relevant to consumer code
1664 dummy_done_idx: _,
1665 artifact_map_idx: _,
1666 artifact_names: _,
1667 global_patchfns,
1668 inject_all_jobs_with: _, // processed above
1669 } = pipeline;
1670
1671 for patchfn in global_patchfns {
1672 for job in &mut jobs {
1673 job.patches.apply_patchfn(patchfn)
1674 }
1675 }
1676
1677 Self {
1678 jobs,
1679 artifacts,
1680 parameters,
1681 extra_deps,
1682 ado_name,
1683 ado_schedule_triggers,
1684 ado_ci_triggers,
1685 ado_pr_triggers,
1686 ado_bootstrap_template,
1687 ado_resources_repository,
1688 ado_post_process_yaml_cb,
1689 ado_variables,
1690 ado_job_id_overrides,
1691 gh_name,
1692 gh_schedule_triggers,
1693 gh_ci_triggers,
1694 gh_pr_triggers,
1695 gh_bootstrap_template,
1696 }
1697 }
1698 }
1699
1700 #[derive(Debug, Clone)]
1701 pub enum Parameter {
1702 Bool {
1703 name: String,
1704 description: String,
1705 kind: ParameterKind,
1706 default: Option<bool>,
1707 },
1708 String {
1709 name: String,
1710 description: String,
1711 default: Option<String>,
1712 kind: ParameterKind,
1713 possible_values: Option<Vec<String>>,
1714 },
1715 Num {
1716 name: String,
1717 description: String,
1718 default: Option<i64>,
1719 kind: ParameterKind,
1720 possible_values: Option<Vec<i64>>,
1721 },
1722 }
1723
1724 impl Parameter {
1725 pub fn name(&self) -> &str {
1726 match self {
1727 Parameter::Bool { name, .. } => name,
1728 Parameter::String { name, .. } => name,
1729 Parameter::Num { name, .. } => name,
1730 }
1731 }
1732 }
1733}