flowey_core/pipeline.rs
1// Copyright (c) Microsoft Corporation.
2// Licensed under the MIT License.
3
4//! Core types and traits used to create and work with flowey pipelines.
5
6mod artifact;
7
8pub use artifact::Artifact;
9pub use artifact::ArtifactType;
10
11use self::internal::*;
12use crate::node::FlowArch;
13use crate::node::FlowNodeBase;
14use crate::node::FlowPlatform;
15use crate::node::FlowPlatformLinuxDistro;
16use crate::node::GhUserSecretVar;
17use crate::node::IntoConfig;
18use crate::node::IntoRequest;
19use crate::node::NodeHandle;
20use crate::node::ReadVar;
21use crate::node::WriteVar;
22use crate::node::steps::ado::AdoResourcesRepositoryId;
23use crate::node::user_facing::AdoRuntimeVar;
24use crate::node::user_facing::GhPermission;
25use crate::node::user_facing::GhPermissionValue;
26use crate::patch::PatchResolver;
27use crate::patch::ResolvedPatches;
28use serde::Serialize;
29use serde::de::DeserializeOwned;
30use std::collections::BTreeMap;
31use std::collections::BTreeSet;
32use std::path::PathBuf;
33
34/// Pipeline types which are considered "user facing", and included in the
35/// `flowey` prelude.
36pub mod user_facing {
37 pub use super::AdoCiTriggers;
38 pub use super::AdoPool;
39 pub use super::AdoPrTriggers;
40 pub use super::AdoResourcesRepository;
41 pub use super::AdoResourcesRepositoryRef;
42 pub use super::AdoResourcesRepositoryType;
43 pub use super::AdoScheduleTriggers;
44 pub use super::GhCiTriggers;
45 pub use super::GhConcurrencyGroup;
46 pub use super::GhPrTriggers;
47 pub use super::GhRunner;
48 pub use super::GhRunnerOsLabel;
49 pub use super::GhScheduleTriggers;
50 pub use super::HostExt;
51 pub use super::IntoPipeline;
52 pub use super::ParameterKind;
53 pub use super::Pipeline;
54 pub use super::PipelineBackendHint;
55 pub use super::PipelineJob;
56 pub use super::PipelineJobCtx;
57 pub use super::PipelineJobHandle;
58 pub use super::PublishArtifact;
59 pub use super::PublishTypedArtifact;
60 pub use super::UseArtifact;
61 pub use super::UseParameter;
62 pub use super::UseTypedArtifact;
63 pub use crate::node::FlowArch;
64 pub use crate::node::FlowPlatform;
65}
66
67fn linux_distro() -> FlowPlatformLinuxDistro {
68 // Check for nix environment first - takes precedence over distro detection
69 if std::env::var("IN_NIX_SHELL").is_ok() {
70 return FlowPlatformLinuxDistro::Nix;
71 }
72
73 // A `nix develop` shell doesn't set `IN_NIX_SHELL`, but the PATH should include a nix store path
74 if std::env::var("PATH").is_ok_and(|path| path.contains("/nix/store")) {
75 return FlowPlatformLinuxDistro::Nix;
76 }
77
78 if let Ok(etc_os_release) = fs_err::read_to_string("/etc/os-release") {
79 if etc_os_release.contains("ID=ubuntu") {
80 FlowPlatformLinuxDistro::Ubuntu
81 } else if etc_os_release.contains("ID=fedora") {
82 FlowPlatformLinuxDistro::Fedora
83 } else if etc_os_release.contains("ID=azurelinux") || etc_os_release.contains("ID=mariner")
84 {
85 FlowPlatformLinuxDistro::AzureLinux
86 } else if etc_os_release.contains("ID=arch") {
87 FlowPlatformLinuxDistro::Arch
88 } else {
89 FlowPlatformLinuxDistro::Unknown
90 }
91 } else {
92 FlowPlatformLinuxDistro::Unknown
93 }
94}
95
96pub trait HostExt: Sized {
97 /// Return the value for the current host machine.
98 ///
99 /// Will panic on non-local backends.
100 fn host(backend_hint: PipelineBackendHint) -> Self;
101}
102
103impl HostExt for FlowPlatform {
104 /// Return the platform of the current host machine.
105 ///
106 /// Will panic on non-local backends.
107 fn host(backend_hint: PipelineBackendHint) -> Self {
108 if !matches!(backend_hint, PipelineBackendHint::Local) {
109 panic!("can only use `FlowPlatform::host` when defining a local-only pipeline");
110 }
111
112 if cfg!(target_os = "windows") {
113 Self::Windows
114 } else if cfg!(target_os = "linux") {
115 Self::Linux(linux_distro())
116 } else if cfg!(target_os = "macos") {
117 Self::MacOs
118 } else {
119 panic!("no valid host-os")
120 }
121 }
122}
123
124impl HostExt for FlowArch {
125 /// Return the arch of the current host machine.
126 ///
127 /// Will panic on non-local backends.
128 fn host(backend_hint: PipelineBackendHint) -> Self {
129 if !matches!(backend_hint, PipelineBackendHint::Local) {
130 panic!("can only use `FlowArch::host` when defining a local-only pipeline");
131 }
132
133 // xtask-fmt allow-target-arch oneoff-flowey
134 if cfg!(target_arch = "x86_64") {
135 Self::X86_64
136 // xtask-fmt allow-target-arch oneoff-flowey
137 } else if cfg!(target_arch = "aarch64") {
138 Self::Aarch64
139 } else {
140 panic!("no valid host-arch")
141 }
142 }
143}
144
145/// Trigger ADO pipelines via Continuous Integration
146#[derive(Default, Debug)]
147pub struct AdoScheduleTriggers {
148 /// Friendly name for the scheduled run
149 pub display_name: String,
150 /// Run the pipeline whenever there is a commit on these specified branches
151 /// (supports glob syntax)
152 pub branches: Vec<String>,
153 /// Specify any branches which should be filtered out from the list of
154 /// `branches` (supports glob syntax)
155 pub exclude_branches: Vec<String>,
156 /// Run the pipeline in a schedule, as specified by a cron string
157 pub cron: String,
158}
159
160/// Trigger ADO pipelines per PR
161#[derive(Debug)]
162pub struct AdoPrTriggers {
163 /// Run the pipeline whenever there is a PR to these specified branches
164 /// (supports glob syntax)
165 pub branches: Vec<String>,
166 /// Specify any branches which should be filtered out from the list of
167 /// `branches` (supports glob syntax)
168 pub exclude_branches: Vec<String>,
169 /// Run the pipeline even if the PR is a draft PR. Defaults to `false`.
170 pub run_on_draft: bool,
171 /// Automatically cancel the pipeline run if a new commit lands in the
172 /// branch. Defaults to `true`.
173 pub auto_cancel: bool,
174 /// Only run the pipeline when files matching these paths are changed
175 /// (supports glob syntax)
176 pub paths: Vec<String>,
177 /// Specify any paths which should be filtered out from the list of
178 /// `paths` (supports glob syntax)
179 pub exclude_paths: Vec<String>,
180}
181
182/// Trigger ADO pipelines per CI
183#[derive(Debug, Default)]
184pub struct AdoCiTriggers {
185 /// Run the pipeline whenever there is a change to these specified branches
186 /// (supports glob syntax)
187 pub branches: Vec<String>,
188 /// Specify any branches which should be filtered out from the list of
189 /// `branches` (supports glob syntax)
190 pub exclude_branches: Vec<String>,
191 /// Run the pipeline whenever a matching tag is created (supports glob
192 /// syntax)
193 pub tags: Vec<String>,
194 /// Specify any tags which should be filtered out from the list of `tags`
195 /// (supports glob syntax)
196 pub exclude_tags: Vec<String>,
197 /// Whether to batch changes per branch.
198 pub batch: bool,
199 /// Only run the pipeline when files matching these paths are changed
200 /// (supports glob syntax)
201 pub paths: Vec<String>,
202 /// Specify any paths which should be filtered out from the list of
203 /// `paths` (supports glob syntax)
204 pub exclude_paths: Vec<String>,
205}
206
207impl Default for AdoPrTriggers {
208 fn default() -> Self {
209 Self {
210 branches: Vec::new(),
211 exclude_branches: Vec::new(),
212 run_on_draft: false,
213 auto_cancel: true,
214 paths: Vec::new(),
215 exclude_paths: Vec::new(),
216 }
217 }
218}
219
220/// ADO repository resource.
221#[derive(Debug)]
222pub struct AdoResourcesRepository {
223 /// Type of repo that is being connected to.
224 pub repo_type: AdoResourcesRepositoryType,
225 /// Repository name. Format depends on `repo_type`.
226 pub name: String,
227 /// git ref to checkout.
228 pub git_ref: AdoResourcesRepositoryRef,
229 /// (optional) ID of the service endpoint connecting to this repository.
230 pub endpoint: Option<String>,
231}
232
233/// ADO repository resource type
234#[derive(Debug)]
235pub enum AdoResourcesRepositoryType {
236 /// Azure Repos Git repository
237 AzureReposGit,
238 /// Github repository
239 GitHub,
240}
241
242/// ADO repository ref
243#[derive(Debug)]
244pub enum AdoResourcesRepositoryRef<P = UseParameter<String>> {
245 /// Hard-coded ref (e.g: refs/heads/main)
246 Fixed(String),
247 /// Connected to pipeline-level parameter
248 Parameter(P),
249}
250
251/// Trigger Github Actions pipelines via Continuous Integration
252///
253/// NOTE: Github Actions doesn't support specifying the branch when triggered by `schedule`.
254/// To run on a specific branch, modify the branch checked out in the pipeline.
255#[derive(Default, Debug)]
256pub struct GhScheduleTriggers {
257 /// Run the pipeline in a schedule, as specified by a cron string
258 pub cron: String,
259}
260
261/// Trigger Github Actions pipelines per PR
262#[derive(Debug)]
263pub struct GhPrTriggers {
264 /// Run the pipeline whenever there is a PR to these specified branches
265 /// (supports glob syntax)
266 pub branches: Vec<String>,
267 /// Specify any branches which should be filtered out from the list of
268 /// `branches` (supports glob syntax)
269 pub exclude_branches: Vec<String>,
270 /// Automatically cancel the pipeline run if a new commit lands in the
271 /// branch. Defaults to `true`.
272 pub auto_cancel: bool,
273 /// Run the pipeline whenever the PR trigger matches the specified types
274 pub types: Vec<String>,
275 /// Only run the pipeline when files matching these paths are changed
276 /// (supports glob syntax)
277 pub paths: Vec<String>,
278 /// Specify any paths which should be filtered out from the list of
279 /// `paths` (supports glob syntax)
280 pub paths_ignore: Vec<String>,
281}
282
283/// Trigger Github Actions pipelines per PR
284#[derive(Debug, Default)]
285pub struct GhCiTriggers {
286 /// Run the pipeline whenever there is a change to these specified branches
287 /// (supports glob syntax)
288 pub branches: Vec<String>,
289 /// Specify any branches which should be filtered out from the list of
290 /// `branches` (supports glob syntax)
291 pub exclude_branches: Vec<String>,
292 /// Run the pipeline whenever a matching tag is created (supports glob
293 /// syntax)
294 pub tags: Vec<String>,
295 /// Specify any tags which should be filtered out from the list of `tags`
296 /// (supports glob syntax)
297 pub exclude_tags: Vec<String>,
298 /// Only run the pipeline when files matching these paths are changed
299 /// (supports glob syntax)
300 pub paths: Vec<String>,
301 /// Specify any paths which should be filtered out from the list of
302 /// `paths` (supports glob syntax)
303 pub paths_ignore: Vec<String>,
304 /// If set, only one pipeline run in this concurrency group runs at a time.
305 pub concurrency_group: Option<GhConcurrencyGroup>,
306}
307
308/// Settings for a Github concurrency group.
309#[derive(Debug, Default, Clone)]
310pub struct GhConcurrencyGroup {
311 /// The name of the concurrency group.
312 pub name: String,
313 /// Cancel the active run when a new run joins the concurrency group.
314 /// Defaults to `false`.
315 pub cancel_in_progress: bool,
316}
317
318impl GhPrTriggers {
319 /// Triggers the pipeline on the default PR events plus when a draft is marked as ready for review.
320 pub fn new_draftable() -> Self {
321 Self {
322 branches: Vec::new(),
323 exclude_branches: Vec::new(),
324 types: vec![
325 "opened".into(),
326 "synchronize".into(),
327 "reopened".into(),
328 "ready_for_review".into(),
329 ],
330 auto_cancel: true,
331 paths: Vec::new(),
332 paths_ignore: Vec::new(),
333 }
334 }
335}
336
337#[derive(Debug, Clone, PartialEq)]
338pub enum GhRunnerOsLabel {
339 UbuntuLatest,
340 Ubuntu2404,
341 Ubuntu2204,
342 WindowsLatest,
343 Windows2025,
344 Windows2022,
345 Ubuntu2404Arm,
346 Ubuntu2204Arm,
347 Windows11Arm,
348 Custom(String),
349}
350
351/// GitHub runner type
352#[derive(Debug, Clone, PartialEq)]
353pub enum GhRunner {
354 // See <https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#choosing-github-hosted-runners>
355 // for more details.
356 GhHosted(GhRunnerOsLabel),
357 // Self hosted runners are selected by matching runner labels to <labels>.
358 // 'self-hosted' is a common label for self hosted runners, but is not required.
359 // Labels are case-insensitive and can take the form of arbitrary strings.
360 // See <https://docs.github.com/en/actions/hosting-your-own-runners> for more details.
361 SelfHosted(Vec<String>),
362 // This uses a runner belonging to <group> that matches all labels in <labels>.
363 // See <https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#choosing-github-hosted-runners>
364 // for more details.
365 RunnerGroup { group: String, labels: Vec<String> },
366}
367
368impl GhRunner {
369 /// Whether this is a self-hosted runner with the provided label
370 pub fn is_self_hosted_with_label(&self, label: &str) -> bool {
371 matches!(self, GhRunner::SelfHosted(labels) if labels.iter().any(|s| s.as_str() == label))
372 }
373}
374
375// TODO: support a more structured format for demands
376// See https://learn.microsoft.com/en-us/azure/devops/pipelines/yaml-schema/pool-demands
377#[derive(Debug, Clone)]
378pub struct AdoPool {
379 pub name: String,
380 pub demands: Vec<String>,
381}
382
383/// Parameter type (unstable / stable).
384#[derive(Debug, Clone)]
385pub enum ParameterKind {
386 // The parameter is considered an unstable API, and should not be
387 // taken as a dependency.
388 Unstable,
389 // The parameter is considered a stable API, and can be used by
390 // external pipelines to control behavior of the pipeline.
391 Stable,
392}
393
394#[derive(Clone, Debug)]
395#[must_use]
396pub struct UseParameter<T> {
397 idx: usize,
398 _kind: std::marker::PhantomData<T>,
399}
400
401/// Opaque handle to an artifact which must be published by a single job.
402#[must_use]
403pub struct PublishArtifact {
404 idx: usize,
405}
406
407/// Opaque handle to an artifact which can be used by one or more jobs.
408#[derive(Clone)]
409#[must_use]
410pub struct UseArtifact {
411 idx: usize,
412}
413
414/// Opaque handle to an artifact of type `T` which must be published by a single job.
415#[must_use]
416pub struct PublishTypedArtifact<T>(PublishArtifact, std::marker::PhantomData<fn() -> T>);
417
418/// Opaque handle to an artifact of type `T` which can be used by one or more
419/// jobs.
420#[must_use]
421pub struct UseTypedArtifact<T>(UseArtifact, std::marker::PhantomData<fn(T)>);
422
423impl<T> Clone for UseTypedArtifact<T> {
424 fn clone(&self) -> Self {
425 UseTypedArtifact(self.0.clone(), std::marker::PhantomData)
426 }
427}
428
429#[derive(Default)]
430pub struct Pipeline {
431 jobs: Vec<PipelineJobMetadata>,
432 artifacts: Vec<ArtifactMeta>,
433 parameters: Vec<ParameterMeta>,
434 extra_deps: BTreeSet<(usize, usize)>,
435 // builder internal
436 artifact_names: BTreeSet<String>,
437 dummy_done_idx: usize,
438 artifact_map_idx: usize,
439 global_patchfns: Vec<crate::patch::PatchFn>,
440 inject_all_jobs_with: Option<Box<dyn for<'a> Fn(PipelineJob<'a>) -> PipelineJob<'a>>>,
441 // backend specific
442 ado_name: Option<String>,
443 ado_job_id_overrides: BTreeMap<usize, String>,
444 ado_schedule_triggers: Vec<AdoScheduleTriggers>,
445 ado_ci_triggers: Option<AdoCiTriggers>,
446 ado_pr_triggers: Option<AdoPrTriggers>,
447 ado_resources_repository: Vec<InternalAdoResourcesRepository>,
448 ado_bootstrap_template: String,
449 ado_variables: BTreeMap<String, String>,
450 ado_post_process_yaml_cb: Option<Box<dyn FnOnce(serde_yaml::Value) -> serde_yaml::Value>>,
451 gh_name: Option<String>,
452 gh_schedule_triggers: Vec<GhScheduleTriggers>,
453 gh_ci_triggers: Option<GhCiTriggers>,
454 gh_pr_triggers: Option<GhPrTriggers>,
455 gh_bootstrap_template: String,
456}
457
458impl Pipeline {
459 pub fn new() -> Pipeline {
460 Pipeline::default()
461 }
462
463 /// Inject all pipeline jobs with some common logic. (e.g: to resolve common
464 /// configuration requirements shared by all jobs).
465 ///
466 /// Can only be invoked once per pipeline.
467 #[track_caller]
468 pub fn inject_all_jobs_with(
469 &mut self,
470 cb: impl for<'a> Fn(PipelineJob<'a>) -> PipelineJob<'a> + 'static,
471 ) -> &mut Self {
472 if self.inject_all_jobs_with.is_some() {
473 panic!("can only call inject_all_jobs_with once!")
474 }
475 self.inject_all_jobs_with = Some(Box::new(cb));
476 self
477 }
478
479 /// (ADO only) Provide a YAML template used to bootstrap flowey at the start
480 /// of an ADO pipeline.
481 ///
482 /// The template has access to the following vars, which will be statically
483 /// interpolated into the template's text:
484 ///
485 /// - `{{FLOWEY_OUTDIR}}`
486 /// - Directory to copy artifacts into.
487 /// - NOTE: this var will include `\` on Windows, and `/` on linux!
488 /// - `{{FLOWEY_BIN_EXTENSION}}`
489 /// - Extension of the expected flowey bin (either "", or ".exe")
490 /// - `{{FLOWEY_CRATE}}`
491 /// - Name of the project-specific flowey crate to be built
492 /// - `{{FLOWEY_TARGET}}`
493 /// - The target-triple flowey is being built for
494 /// - `{{FLOWEY_PIPELINE_PATH}}`
495 /// - Repo-root relative path to the pipeline (as provided when
496 /// generating the pipeline via the flowey CLI)
497 ///
498 /// The template's sole responsibility is to copy 3 files into the
499 /// `{{FLOWEY_OUTDIR}}`:
500 ///
501 /// 1. The bootstrapped flowey binary, with the file name
502 /// `flowey{{FLOWEY_BIN_EXTENSION}}`
503 /// 2. Two files called `pipeline.yaml` and `pipeline.json`, which are
504 /// copied of the pipeline YAML and pipeline JSON currently being run.
505 /// `{{FLOWEY_PIPELINE_PATH}}` is provided as a way to disambiguate in
506 /// cases where the same template is being for multiple pipelines (e.g: a
507 /// debug vs. release pipeline).
508 pub fn ado_set_flowey_bootstrap_template(&mut self, template: String) -> &mut Self {
509 self.ado_bootstrap_template = template;
510 self
511 }
512
513 /// (ADO only) Provide a callback function which will be used to
514 /// post-process any YAML flowey generates for the pipeline.
515 ///
516 /// Until flowey defines a stable API for maintaining out-of-tree backends,
517 /// this method can be used to integrate the output from the generic ADO
518 /// backend with any organization-specific templates that one may be
519 /// required to use (e.g: for compliance reasons).
520 pub fn ado_post_process_yaml(
521 &mut self,
522 cb: impl FnOnce(serde_yaml::Value) -> serde_yaml::Value + 'static,
523 ) -> &mut Self {
524 self.ado_post_process_yaml_cb = Some(Box::new(cb));
525 self
526 }
527
528 /// (ADO only) Add a new scheduled CI trigger. Can be called multiple times
529 /// to set up multiple schedules runs.
530 pub fn ado_add_schedule_trigger(&mut self, triggers: AdoScheduleTriggers) -> &mut Self {
531 self.ado_schedule_triggers.push(triggers);
532 self
533 }
534
535 /// (ADO only) Set a PR trigger. Calling this method multiple times will
536 /// overwrite any previously set triggers.
537 pub fn ado_set_pr_triggers(&mut self, triggers: AdoPrTriggers) -> &mut Self {
538 self.ado_pr_triggers = Some(triggers);
539 self
540 }
541
542 /// (ADO only) Set a CI trigger. Calling this method multiple times will
543 /// overwrite any previously set triggers.
544 pub fn ado_set_ci_triggers(&mut self, triggers: AdoCiTriggers) -> &mut Self {
545 self.ado_ci_triggers = Some(triggers);
546 self
547 }
548
549 /// (ADO only) Declare a new repository resource, returning a type-safe
550 /// handle which downstream ADO steps are able to consume via
551 /// [`AdoStepServices::resolve_repository_id`](crate::node::user_facing::AdoStepServices::resolve_repository_id).
552 pub fn ado_add_resources_repository(
553 &mut self,
554 repo: AdoResourcesRepository,
555 ) -> AdoResourcesRepositoryId {
556 let AdoResourcesRepository {
557 repo_type,
558 name,
559 git_ref,
560 endpoint,
561 } = repo;
562
563 let repo_id = format!("repo{}", self.ado_resources_repository.len());
564
565 self.ado_resources_repository
566 .push(InternalAdoResourcesRepository {
567 repo_id: repo_id.clone(),
568 repo_type,
569 name,
570 git_ref: match git_ref {
571 AdoResourcesRepositoryRef::Fixed(s) => AdoResourcesRepositoryRef::Fixed(s),
572 AdoResourcesRepositoryRef::Parameter(p) => {
573 AdoResourcesRepositoryRef::Parameter(p.idx)
574 }
575 },
576 endpoint,
577 });
578 AdoResourcesRepositoryId { repo_id }
579 }
580
581 /// (GitHub Actions only) Set the pipeline-level name.
582 ///
583 /// <https://docs.github.com/en/actions/writing-workflows/workflow-syntax-for-github-actions#name>
584 pub fn gh_set_name(&mut self, name: impl AsRef<str>) -> &mut Self {
585 self.gh_name = Some(name.as_ref().into());
586 self
587 }
588
589 /// Provide a YAML template used to bootstrap flowey at the start of an GitHub
590 /// pipeline.
591 ///
592 /// The template has access to the following vars, which will be statically
593 /// interpolated into the template's text:
594 ///
595 /// - `{{FLOWEY_OUTDIR}}`
596 /// - Directory to copy artifacts into.
597 /// - NOTE: this var will include `\` on Windows, and `/` on linux!
598 /// - `{{FLOWEY_BIN_EXTENSION}}`
599 /// - Extension of the expected flowey bin (either "", or ".exe")
600 /// - `{{FLOWEY_CRATE}}`
601 /// - Name of the project-specific flowey crate to be built
602 /// - `{{FLOWEY_TARGET}}`
603 /// - The target-triple flowey is being built for
604 /// - `{{FLOWEY_PIPELINE_PATH}}`
605 /// - Repo-root relative path to the pipeline (as provided when
606 /// generating the pipeline via the flowey CLI)
607 ///
608 /// The template's sole responsibility is to copy 3 files into the
609 /// `{{FLOWEY_OUTDIR}}`:
610 ///
611 /// 1. The bootstrapped flowey binary, with the file name
612 /// `flowey{{FLOWEY_BIN_EXTENSION}}`
613 /// 2. Two files called `pipeline.yaml` and `pipeline.json`, which are
614 /// copied of the pipeline YAML and pipeline JSON currently being run.
615 /// `{{FLOWEY_PIPELINE_PATH}}` is provided as a way to disambiguate in
616 /// cases where the same template is being for multiple pipelines (e.g: a
617 /// debug vs. release pipeline).
618 pub fn gh_set_flowey_bootstrap_template(&mut self, template: String) -> &mut Self {
619 self.gh_bootstrap_template = template;
620 self
621 }
622
623 /// (GitHub Actions only) Add a new scheduled CI trigger. Can be called multiple times
624 /// to set up multiple schedules runs.
625 pub fn gh_add_schedule_trigger(&mut self, triggers: GhScheduleTriggers) -> &mut Self {
626 self.gh_schedule_triggers.push(triggers);
627 self
628 }
629
630 /// (GitHub Actions only) Set a PR trigger. Calling this method multiple times will
631 /// overwrite any previously set triggers.
632 pub fn gh_set_pr_triggers(&mut self, triggers: GhPrTriggers) -> &mut Self {
633 self.gh_pr_triggers = Some(triggers);
634 self
635 }
636
637 /// (GitHub Actions only) Set a CI trigger. Calling this method multiple times will
638 /// overwrite any previously set triggers.
639 pub fn gh_set_ci_triggers(&mut self, triggers: GhCiTriggers) -> &mut Self {
640 self.gh_ci_triggers = Some(triggers);
641 self
642 }
643
644 /// (GitHub Actions only) Use a pre-defined GitHub Actions secret variable.
645 ///
646 /// For more information on defining secrets for use in GitHub Actions, see
647 /// <https://docs.github.com/en/actions/security-guides/using-secrets-in-github-actions>
648 pub fn gh_use_secret(&mut self, secret_name: impl AsRef<str>) -> GhUserSecretVar {
649 GhUserSecretVar(secret_name.as_ref().to_string())
650 }
651
652 pub fn new_job(
653 &mut self,
654 platform: FlowPlatform,
655 arch: FlowArch,
656 label: impl AsRef<str>,
657 ) -> PipelineJob<'_> {
658 let idx = self.jobs.len();
659 self.jobs.push(PipelineJobMetadata {
660 root_nodes: BTreeMap::new(),
661 root_configs: BTreeMap::new(),
662 patches: ResolvedPatches::build(),
663 label: label.as_ref().into(),
664 platform,
665 arch,
666 cond_param_idx: None,
667 timeout_minutes: None,
668 command_wrapper: None,
669 ado_pool: None,
670 ado_variables: BTreeMap::new(),
671 gh_override_if: None,
672 gh_global_env: BTreeMap::new(),
673 gh_pool: None,
674 gh_concurrency_group: None,
675 gh_permissions: BTreeMap::new(),
676 });
677
678 PipelineJob {
679 pipeline: self,
680 job_idx: idx,
681 }
682 }
683
684 /// Declare a dependency between two jobs that does is not a result of an
685 /// artifact.
686 pub fn non_artifact_dep(
687 &mut self,
688 job: &PipelineJobHandle,
689 depends_on_job: &PipelineJobHandle,
690 ) -> &mut Self {
691 self.extra_deps
692 .insert((depends_on_job.job_idx, job.job_idx));
693 self
694 }
695
696 #[track_caller]
697 pub fn new_artifact(&mut self, name: impl AsRef<str>) -> (PublishArtifact, UseArtifact) {
698 let name = name.as_ref();
699 let owned_name = name.to_string();
700
701 let not_exists = self.artifact_names.insert(owned_name.clone());
702 if !not_exists {
703 panic!("duplicate artifact name: {}", name)
704 }
705
706 let idx = self.artifacts.len();
707 self.artifacts.push(ArtifactMeta {
708 name: owned_name,
709 published_by_job: None,
710 used_by_jobs: BTreeSet::new(),
711 });
712
713 (PublishArtifact { idx }, UseArtifact { idx })
714 }
715
716 /// Returns a pair of opaque handles to a new artifact for use across jobs
717 /// in the pipeline.
718 #[track_caller]
719 pub fn new_typed_artifact<T: Artifact>(
720 &mut self,
721 name: impl AsRef<str>,
722 ) -> (PublishTypedArtifact<T>, UseTypedArtifact<T>) {
723 let (publish, use_artifact) = self.new_artifact(name);
724 (
725 PublishTypedArtifact(publish, std::marker::PhantomData),
726 UseTypedArtifact(use_artifact, std::marker::PhantomData),
727 )
728 }
729
730 /// Returns a pair of sets of opaque handles to a new artifact for use
731 /// across jobs in the pipeline. The artifact names are derived by the impl
732 /// of [`ArtifactType::name`] using common prefixes and suffixes if
733 /// specified (although the implementor can choose to use those values
734 /// differently).
735 #[track_caller]
736 pub fn new_typed_artifact_collection<T: Artifact, U: ArtifactType>(
737 &mut self,
738 artifact_types: impl IntoIterator<Item = U>,
739 prefix: Option<&str>,
740 suffix: Option<&str>,
741 ) -> (
742 BTreeMap<U, PublishTypedArtifact<T>>,
743 BTreeMap<U, UseTypedArtifact<T>>,
744 ) {
745 artifact_types
746 .into_iter()
747 .map(|artifact_type| {
748 let (pub_artifact, use_artifact) =
749 self.new_typed_artifact(artifact_type.name(prefix, suffix));
750 (
751 (artifact_type.clone(), pub_artifact),
752 (artifact_type, use_artifact),
753 )
754 })
755 .unzip()
756 }
757
758 /// (ADO only) Set the pipeline-level name.
759 ///
760 /// <https://learn.microsoft.com/en-us/azure/devops/pipelines/process/run-number?view=azure-devops&tabs=yaml>
761 pub fn ado_add_name(&mut self, name: String) -> &mut Self {
762 self.ado_name = Some(name);
763 self
764 }
765
766 /// (ADO only) Declare a pipeline-level, named, read-only ADO variable.
767 ///
768 /// `name` and `value` are both arbitrary strings.
769 ///
770 /// Returns an instance of [`AdoRuntimeVar`], which, if need be, can be
771 /// converted into a [`ReadVar<String>`] using
772 /// [`NodeCtx::get_ado_variable`].
773 ///
774 /// NOTE: Unless required by some particular third-party task, it's strongly
775 /// recommended to _avoid_ using this method, and to simply use
776 /// [`ReadVar::from_static`] to get a obtain a static variable.
777 ///
778 /// [`NodeCtx::get_ado_variable`]: crate::node::NodeCtx::get_ado_variable
779 pub fn ado_new_named_variable(
780 &mut self,
781 name: impl AsRef<str>,
782 value: impl AsRef<str>,
783 ) -> AdoRuntimeVar {
784 let name = name.as_ref();
785 let value = value.as_ref();
786
787 self.ado_variables.insert(name.into(), value.into());
788
789 // safe, since we'll ensure that the global exists in the ADO backend
790 AdoRuntimeVar::dangerous_from_global(name, false)
791 }
792
793 /// (ADO only) Declare multiple pipeline-level, named, read-only ADO
794 /// variables at once.
795 ///
796 /// This is a convenience method to streamline invoking
797 /// [`Self::ado_new_named_variable`] multiple times.
798 ///
799 /// NOTE: Unless required by some particular third-party task, it's strongly
800 /// recommended to _avoid_ using this method, and to simply use
801 /// [`ReadVar::from_static`] to get a obtain a static variable.
802 ///
803 /// DEVNOTE: In the future, this API may be updated to return a handle that
804 /// will allow resolving the resulting `AdoRuntimeVar`, but for
805 /// implementation expediency, this API does not currently do this. If you
806 /// need to read the value of this variable at runtime, you may need to
807 /// invoke [`AdoRuntimeVar::dangerous_from_global`] manually.
808 ///
809 /// [`NodeCtx::get_ado_variable`]: crate::node::NodeCtx::get_ado_variable
810 pub fn ado_new_named_variables<K, V>(
811 &mut self,
812 vars: impl IntoIterator<Item = (K, V)>,
813 ) -> &mut Self
814 where
815 K: AsRef<str>,
816 V: AsRef<str>,
817 {
818 self.ado_variables.extend(
819 vars.into_iter()
820 .map(|(k, v)| (k.as_ref().into(), v.as_ref().into())),
821 );
822 self
823 }
824
825 /// Declare a pipeline-level runtime parameter with type `bool`.
826 ///
827 /// To obtain a [`ReadVar<bool>`] that can be used within a node, use the
828 /// [`PipelineJobCtx::use_parameter`] method.
829 ///
830 /// `name` is the name of the parameter.
831 ///
832 /// `description` is an arbitrary string, which will be be shown to users.
833 ///
834 /// `kind` is the type of parameter and if it should be treated as a stable
835 /// external API to callers of the pipeline.
836 ///
837 /// `default` is the default value for the parameter. If none is provided,
838 /// the parameter _must_ be specified in order for the pipeline to run.
839 ///
840 /// `possible_values` can be used to limit the set of valid values the
841 /// parameter accepts.
842 pub fn new_parameter_bool(
843 &mut self,
844 name: impl AsRef<str>,
845 description: impl AsRef<str>,
846 kind: ParameterKind,
847 default: Option<bool>,
848 ) -> UseParameter<bool> {
849 let idx = self.parameters.len();
850 let name = new_parameter_name(name, kind.clone());
851 self.parameters.push(ParameterMeta {
852 parameter: Parameter::Bool {
853 name,
854 description: description.as_ref().into(),
855 kind,
856 default,
857 },
858 used_by_jobs: BTreeSet::new(),
859 });
860
861 UseParameter {
862 idx,
863 _kind: std::marker::PhantomData,
864 }
865 }
866
867 /// Declare a pipeline-level runtime parameter with type `i64`.
868 ///
869 /// To obtain a [`ReadVar<i64>`] that can be used within a node, use the
870 /// [`PipelineJobCtx::use_parameter`] method.
871 ///
872 /// `name` is the name of the parameter.
873 ///
874 /// `description` is an arbitrary string, which will be be shown to users.
875 ///
876 /// `kind` is the type of parameter and if it should be treated as a stable
877 /// external API to callers of the pipeline.
878 ///
879 /// `default` is the default value for the parameter. If none is provided,
880 /// the parameter _must_ be specified in order for the pipeline to run.
881 ///
882 /// `possible_values` can be used to limit the set of valid values the
883 /// parameter accepts.
884 pub fn new_parameter_num(
885 &mut self,
886 name: impl AsRef<str>,
887 description: impl AsRef<str>,
888 kind: ParameterKind,
889 default: Option<i64>,
890 possible_values: Option<Vec<i64>>,
891 ) -> UseParameter<i64> {
892 let idx = self.parameters.len();
893 let name = new_parameter_name(name, kind.clone());
894 self.parameters.push(ParameterMeta {
895 parameter: Parameter::Num {
896 name,
897 description: description.as_ref().into(),
898 kind,
899 default,
900 possible_values,
901 },
902 used_by_jobs: BTreeSet::new(),
903 });
904
905 UseParameter {
906 idx,
907 _kind: std::marker::PhantomData,
908 }
909 }
910
911 /// Declare a pipeline-level runtime parameter with type `String`.
912 ///
913 /// To obtain a [`ReadVar<String>`] that can be used within a node, use the
914 /// [`PipelineJobCtx::use_parameter`] method.
915 ///
916 /// `name` is the name of the parameter.
917 ///
918 /// `description` is an arbitrary string, which will be be shown to users.
919 ///
920 /// `kind` is the type of parameter and if it should be treated as a stable
921 /// external API to callers of the pipeline.
922 ///
923 /// `default` is the default value for the parameter. If none is provided,
924 /// the parameter _must_ be specified in order for the pipeline to run.
925 ///
926 /// `possible_values` allows restricting inputs to a set of possible values.
927 /// Depending on the backend, these options may be presented as a set of
928 /// radio buttons, a dropdown menu, or something in that vein. If `None`,
929 /// then any string is allowed.
930 pub fn new_parameter_string(
931 &mut self,
932 name: impl AsRef<str>,
933 description: impl AsRef<str>,
934 kind: ParameterKind,
935 default: Option<impl AsRef<str>>,
936 possible_values: Option<Vec<String>>,
937 ) -> UseParameter<String> {
938 let idx = self.parameters.len();
939 let name = new_parameter_name(name, kind.clone());
940 self.parameters.push(ParameterMeta {
941 parameter: Parameter::String {
942 name,
943 description: description.as_ref().into(),
944 kind,
945 default: default.map(|x| x.as_ref().into()),
946 possible_values,
947 },
948 used_by_jobs: BTreeSet::new(),
949 });
950
951 UseParameter {
952 idx,
953 _kind: std::marker::PhantomData,
954 }
955 }
956}
957
958pub struct PipelineJobCtx<'a> {
959 pipeline: &'a mut Pipeline,
960 job_idx: usize,
961}
962
963impl PipelineJobCtx<'_> {
964 /// Create a new `WriteVar<SideEffect>` anchored to the pipeline job.
965 pub fn new_done_handle(&mut self) -> WriteVar<crate::node::SideEffect> {
966 self.pipeline.dummy_done_idx += 1;
967 crate::node::thin_air_write_runtime_var(format!("start{}", self.pipeline.dummy_done_idx))
968 }
969
970 /// Claim that this job will use this artifact, obtaining a path to a folder
971 /// with the artifact's contents.
972 pub fn use_artifact(&mut self, artifact: &UseArtifact) -> ReadVar<PathBuf> {
973 self.pipeline.artifacts[artifact.idx]
974 .used_by_jobs
975 .insert(self.job_idx);
976
977 crate::node::thin_air_read_runtime_var(consistent_artifact_runtime_var_name(
978 &self.pipeline.artifacts[artifact.idx].name,
979 true,
980 ))
981 }
982
983 /// Claim that this job will publish this artifact, obtaining a path to a
984 /// fresh, empty folder which will be published as the specific artifact at
985 /// the end of the job.
986 pub fn publish_artifact(&mut self, artifact: PublishArtifact) -> ReadVar<PathBuf> {
987 let existing = self.pipeline.artifacts[artifact.idx]
988 .published_by_job
989 .replace(self.job_idx);
990 assert!(existing.is_none()); // PublishArtifact isn't cloneable
991
992 crate::node::thin_air_read_runtime_var(consistent_artifact_runtime_var_name(
993 &self.pipeline.artifacts[artifact.idx].name,
994 false,
995 ))
996 }
997
998 fn helper_request<R: IntoRequest>(&mut self, req: R)
999 where
1000 R::Node: 'static,
1001 {
1002 self.pipeline.jobs[self.job_idx]
1003 .root_nodes
1004 .entry(NodeHandle::from_type::<R::Node>())
1005 .or_default()
1006 .push(serde_json::to_vec(&req.into_request()).unwrap().into());
1007 }
1008
1009 fn new_artifact_map_vars<T: Artifact>(&mut self) -> (ReadVar<T>, WriteVar<T>) {
1010 let artifact_map_idx = self.pipeline.artifact_map_idx;
1011 self.pipeline.artifact_map_idx += 1;
1012
1013 let backing_var = format!("artifact_map{}", artifact_map_idx);
1014 let read_var = crate::node::thin_air_read_runtime_var(backing_var.clone());
1015 let write_var = crate::node::thin_air_write_runtime_var(backing_var);
1016 (read_var, write_var)
1017 }
1018
1019 /// Claim that this job will use this artifact, obtaining the resolved
1020 /// contents of the artifact.
1021 pub fn use_typed_artifact<T: Artifact>(
1022 &mut self,
1023 artifact: &UseTypedArtifact<T>,
1024 ) -> ReadVar<T> {
1025 let artifact_path = self.use_artifact(&artifact.0);
1026 let (read, write) = self.new_artifact_map_vars::<T>();
1027 self.helper_request(artifact::resolve::Request::new(artifact_path, write));
1028 read
1029 }
1030
1031 /// Claim that this job will publish this artifact, obtaining a variable to
1032 /// write the artifact's contents to. The artifact will be published at
1033 /// the end of the job.
1034 pub fn publish_typed_artifact<T: Artifact>(
1035 &mut self,
1036 artifact: PublishTypedArtifact<T>,
1037 ) -> WriteVar<T> {
1038 let artifact_path = self.publish_artifact(artifact.0);
1039 let (read, write) = self.new_artifact_map_vars::<T>();
1040 let done = self.new_done_handle();
1041 self.helper_request(artifact::publish::Request::new(read, artifact_path, done));
1042 write
1043 }
1044
1045 /// Obtain a `ReadVar<T>` corresponding to a pipeline parameter which is
1046 /// specified at runtime.
1047 pub fn use_parameter<T>(&mut self, param: UseParameter<T>) -> ReadVar<T>
1048 where
1049 T: Serialize + DeserializeOwned,
1050 {
1051 self.pipeline.parameters[param.idx]
1052 .used_by_jobs
1053 .insert(self.job_idx);
1054
1055 crate::node::thin_air_read_runtime_var(
1056 self.pipeline.parameters[param.idx]
1057 .parameter
1058 .name()
1059 .to_string(),
1060 )
1061 }
1062
1063 /// Shortcut which allows defining a bool pipeline parameter within a Job.
1064 ///
1065 /// To share a single parameter between multiple jobs, don't use this method
1066 /// - use [`Pipeline::new_parameter_bool`] + [`Self::use_parameter`] instead.
1067 pub fn new_parameter_bool(
1068 &mut self,
1069 name: impl AsRef<str>,
1070 description: impl AsRef<str>,
1071 kind: ParameterKind,
1072 default: Option<bool>,
1073 ) -> ReadVar<bool> {
1074 let param = self
1075 .pipeline
1076 .new_parameter_bool(name, description, kind, default);
1077 self.use_parameter(param)
1078 }
1079
1080 /// Shortcut which allows defining a number pipeline parameter within a Job.
1081 ///
1082 /// To share a single parameter between multiple jobs, don't use this method
1083 /// - use [`Pipeline::new_parameter_num`] + [`Self::use_parameter`] instead.
1084 pub fn new_parameter_num(
1085 &mut self,
1086 name: impl AsRef<str>,
1087 description: impl AsRef<str>,
1088 kind: ParameterKind,
1089 default: Option<i64>,
1090 possible_values: Option<Vec<i64>>,
1091 ) -> ReadVar<i64> {
1092 let param =
1093 self.pipeline
1094 .new_parameter_num(name, description, kind, default, possible_values);
1095 self.use_parameter(param)
1096 }
1097
1098 /// Shortcut which allows defining a string pipeline parameter within a Job.
1099 ///
1100 /// To share a single parameter between multiple jobs, don't use this method
1101 /// - use [`Pipeline::new_parameter_string`] + [`Self::use_parameter`] instead.
1102 pub fn new_parameter_string(
1103 &mut self,
1104 name: impl AsRef<str>,
1105 description: impl AsRef<str>,
1106 kind: ParameterKind,
1107 default: Option<String>,
1108 possible_values: Option<Vec<String>>,
1109 ) -> ReadVar<String> {
1110 let param =
1111 self.pipeline
1112 .new_parameter_string(name, description, kind, default, possible_values);
1113 self.use_parameter(param)
1114 }
1115}
1116
1117#[must_use]
1118pub struct PipelineJob<'a> {
1119 pipeline: &'a mut Pipeline,
1120 job_idx: usize,
1121}
1122
1123impl PipelineJob<'_> {
1124 /// (ADO only) specify which agent pool this job will be run on.
1125 pub fn ado_set_pool(self, pool: AdoPool) -> Self {
1126 self.pipeline.jobs[self.job_idx].ado_pool = Some(pool);
1127 self
1128 }
1129
1130 /// (ADO only) specify which agent pool this job will be run on, with
1131 /// additional special runner demands.
1132 pub fn ado_set_pool_with_demands(self, pool: impl AsRef<str>, demands: Vec<String>) -> Self {
1133 self.pipeline.jobs[self.job_idx].ado_pool = Some(AdoPool {
1134 name: pool.as_ref().into(),
1135 demands,
1136 });
1137 self
1138 }
1139
1140 /// (ADO only) Declare a job-level, named, read-only ADO variable.
1141 ///
1142 /// `name` and `value` are both arbitrary strings, which may include ADO
1143 /// template expressions.
1144 ///
1145 /// NOTE: Unless required by some particular third-party task, it's strongly
1146 /// recommended to _avoid_ using this method, and to simply use
1147 /// [`ReadVar::from_static`] to get a obtain a static variable.
1148 ///
1149 /// DEVNOTE: In the future, this API may be updated to return a handle that
1150 /// will allow resolving the resulting `AdoRuntimeVar`, but for
1151 /// implementation expediency, this API does not currently do this. If you
1152 /// need to read the value of this variable at runtime, you may need to
1153 /// invoke [`AdoRuntimeVar::dangerous_from_global`] manually.
1154 ///
1155 /// [`NodeCtx::get_ado_variable`]: crate::node::NodeCtx::get_ado_variable
1156 pub fn ado_new_named_variable(self, name: impl AsRef<str>, value: impl AsRef<str>) -> Self {
1157 let name = name.as_ref();
1158 let value = value.as_ref();
1159 self.pipeline.jobs[self.job_idx]
1160 .ado_variables
1161 .insert(name.into(), value.into());
1162 self
1163 }
1164
1165 /// (ADO only) Declare multiple job-level, named, read-only ADO variables at
1166 /// once.
1167 ///
1168 /// This is a convenience method to streamline invoking
1169 /// [`Self::ado_new_named_variable`] multiple times.
1170 ///
1171 /// NOTE: Unless required by some particular third-party task, it's strongly
1172 /// recommended to _avoid_ using this method, and to simply use
1173 /// [`ReadVar::from_static`] to get a obtain a static variable.
1174 ///
1175 /// DEVNOTE: In the future, this API may be updated to return a handle that
1176 /// will allow resolving the resulting `AdoRuntimeVar`, but for
1177 /// implementation expediency, this API does not currently do this. If you
1178 /// need to read the value of this variable at runtime, you may need to
1179 /// invoke [`AdoRuntimeVar::dangerous_from_global`] manually.
1180 ///
1181 /// [`NodeCtx::get_ado_variable`]: crate::node::NodeCtx::get_ado_variable
1182 pub fn ado_new_named_variables<K, V>(self, vars: impl IntoIterator<Item = (K, V)>) -> Self
1183 where
1184 K: AsRef<str>,
1185 V: AsRef<str>,
1186 {
1187 self.pipeline.jobs[self.job_idx].ado_variables.extend(
1188 vars.into_iter()
1189 .map(|(k, v)| (k.as_ref().into(), v.as_ref().into())),
1190 );
1191 self
1192 }
1193
1194 /// Overrides the id of the job.
1195 ///
1196 /// Flowey typically generates a reasonable job ID but some use cases that depend
1197 /// on the ID may find it useful to override it to something custom.
1198 pub fn ado_override_job_id(self, name: impl AsRef<str>) -> Self {
1199 self.pipeline
1200 .ado_job_id_overrides
1201 .insert(self.job_idx, name.as_ref().into());
1202 self
1203 }
1204
1205 /// (GitHub Actions only) specify which Github runner this job will be run on.
1206 pub fn gh_set_pool(self, pool: GhRunner) -> Self {
1207 self.pipeline.jobs[self.job_idx].gh_pool = Some(pool);
1208 self
1209 }
1210
1211 /// (GitHub Actions only) Set the concurrency group for this job.
1212 pub fn gh_set_concurrency_group(self, group: GhConcurrencyGroup) -> Self {
1213 self.pipeline.jobs[self.job_idx].gh_concurrency_group = Some(group);
1214 self
1215 }
1216
1217 /// (GitHub Actions only) Manually override the `if:` condition for this
1218 /// particular job.
1219 ///
1220 /// **This is dangerous**, as an improperly set `if` condition may break
1221 /// downstream flowey jobs which assume flowey is in control of the job's
1222 /// scheduling logic.
1223 ///
1224 /// See
1225 /// <https://docs.github.com/en/actions/writing-workflows/workflow-syntax-for-github-actions#jobsjob_idif>
1226 /// for more info.
1227 pub fn gh_dangerous_override_if(self, condition: impl AsRef<str>) -> Self {
1228 self.pipeline.jobs[self.job_idx].gh_override_if = Some(condition.as_ref().into());
1229 self
1230 }
1231
1232 /// (GitHub Actions only) Declare a global job-level environment variable,
1233 /// visible to all downstream steps.
1234 ///
1235 /// `name` and `value` are both arbitrary strings, which may include GitHub
1236 /// Actions template expressions.
1237 ///
1238 /// **This is dangerous**, as it is easy to misuse this API in order to
1239 /// write a node which takes an implicit dependency on there being a global
1240 /// variable set on its behalf by the top-level pipeline code, making it
1241 /// difficult to "locally reason" about the behavior of a node simply by
1242 /// reading its code.
1243 ///
1244 /// Whenever possible, nodes should "late bind" environment variables:
1245 /// accepting a compile-time / runtime flowey parameter, and then setting it
1246 /// prior to executing a child command that requires it.
1247 ///
1248 /// Only use this API in exceptional cases, such as obtaining an environment
1249 /// variable whose value is determined by a job-level GitHub Actions
1250 /// expression evaluation.
1251 pub fn gh_dangerous_global_env_var(
1252 self,
1253 name: impl AsRef<str>,
1254 value: impl AsRef<str>,
1255 ) -> Self {
1256 let name = name.as_ref();
1257 let value = value.as_ref();
1258 self.pipeline.jobs[self.job_idx]
1259 .gh_global_env
1260 .insert(name.into(), value.into());
1261 self
1262 }
1263
1264 /// (GitHub Actions only) Grant permissions required by nodes in the job.
1265 ///
1266 /// For a given node handle, grant the specified permissions.
1267 /// The list provided must match the permissions specified within the node
1268 /// using `requires_permission`.
1269 ///
1270 /// NOTE: While this method is called at a node-level for auditability, the emitted
1271 /// yaml grants permissions at the job-level.
1272 ///
1273 /// This can lead to weird situations where node 1 might not specify a permission
1274 /// required according to Github Actions, but due to job-level granting of the permission
1275 /// by another node 2, the pipeline executes even though it wouldn't if node 2 was removed.
1276 ///
1277 /// For available permission scopes and their descriptions, see
1278 /// <https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#permissions>.
1279 pub fn gh_grant_permissions<N: FlowNodeBase + 'static>(
1280 self,
1281 permissions: impl IntoIterator<Item = (GhPermission, GhPermissionValue)>,
1282 ) -> Self {
1283 let node_handle = NodeHandle::from_type::<N>();
1284 for (permission, value) in permissions {
1285 self.pipeline.jobs[self.job_idx]
1286 .gh_permissions
1287 .entry(node_handle)
1288 .or_default()
1289 .insert(permission, value);
1290 }
1291 self
1292 }
1293
1294 pub fn apply_patchfn(self, patchfn: crate::patch::PatchFn) -> Self {
1295 self.pipeline.jobs[self.job_idx]
1296 .patches
1297 .apply_patchfn(patchfn);
1298 self
1299 }
1300
1301 /// Set a timeout for the job, in minutes.
1302 ///
1303 /// Not calling this will result in the platform's default timeout being used,
1304 /// which is typically 60 minutes, but may vary.
1305 pub fn with_timeout_in_minutes(self, timeout: u32) -> Self {
1306 self.pipeline.jobs[self.job_idx].timeout_minutes = Some(timeout);
1307 self
1308 }
1309
1310 /// (ADO+Local Only) Only run the job if the specified condition is true.
1311 pub fn with_condition(self, cond: UseParameter<bool>) -> Self {
1312 self.pipeline.jobs[self.job_idx].cond_param_idx = Some(cond.idx);
1313 self.pipeline.parameters[cond.idx]
1314 .used_by_jobs
1315 .insert(self.job_idx);
1316 self
1317 }
1318
1319 /// Set a [`CommandWrapperKind`] that will be applied to all shell
1320 /// commands executed in this job's steps.
1321 ///
1322 /// The wrapper is applied both when running locally (via direct run)
1323 /// and when running in CI (the kind is serialized into
1324 /// `pipeline.json` and reconstructed at runtime).
1325 ///
1326 /// [`CommandWrapperKind`]: crate::shell::CommandWrapperKind
1327 pub fn set_command_wrapper(self, wrapper: crate::shell::CommandWrapperKind) -> Self {
1328 self.pipeline.jobs[self.job_idx].command_wrapper = Some(wrapper);
1329 self
1330 }
1331
1332 /// Add a flow node which will be run as part of the job.
1333 pub fn dep_on<R: IntoRequest + 'static>(
1334 self,
1335 f: impl FnOnce(&mut PipelineJobCtx<'_>) -> R,
1336 ) -> Self {
1337 // JobToNodeCtx will ensure artifact deps are taken care of
1338 let req = f(&mut PipelineJobCtx {
1339 pipeline: self.pipeline,
1340 job_idx: self.job_idx,
1341 });
1342
1343 self.pipeline.jobs[self.job_idx]
1344 .root_nodes
1345 .entry(NodeHandle::from_type::<R::Node>())
1346 .or_default()
1347 .push(serde_json::to_vec(&req.into_request()).unwrap().into());
1348
1349 self
1350 }
1351
1352 /// Add a flow node whose request publishes a typed artifact.
1353 ///
1354 /// This is a shortcut for the common pattern of calling
1355 /// [`PipelineJobCtx::publish_typed_artifact`] inside a [`Self::dep_on`]
1356 /// closure and passing the resulting [`WriteVar`] into a request.
1357 pub fn publish<T: Artifact, R: IntoRequest + 'static>(
1358 self,
1359 artifact: PublishTypedArtifact<T>,
1360 f: impl FnOnce(WriteVar<T>) -> R,
1361 ) -> Self {
1362 self.dep_on(|ctx| f(ctx.publish_typed_artifact(artifact)))
1363 }
1364
1365 /// Add a flow node whose request is run purely for its side effect.
1366 ///
1367 /// This is a shortcut for the common pattern of calling
1368 /// [`PipelineJobCtx::new_done_handle`] inside a [`Self::dep_on`]
1369 /// closure and passing the resulting [`WriteVar`] into a request.
1370 pub fn side_effect<R: IntoRequest + 'static>(
1371 self,
1372 f: impl FnOnce(WriteVar<crate::node::SideEffect>) -> R,
1373 ) -> Self {
1374 self.dep_on(|ctx| f(ctx.new_done_handle()))
1375 }
1376
1377 /// Set config on a node for this job.
1378 ///
1379 /// This is the pipeline-level equivalent of [`NodeCtx::config`]. Config
1380 /// set here is merged with any config set by nodes within the job.
1381 ///
1382 /// [`NodeCtx::config`]: crate::node::NodeCtx::config
1383 pub fn config<C: IntoConfig + 'static>(self, config: C) -> Self {
1384 self.pipeline.jobs[self.job_idx]
1385 .root_configs
1386 .entry(NodeHandle::from_type::<C::Node>())
1387 .or_default()
1388 .push(serde_json::to_vec(&config).unwrap().into());
1389
1390 self
1391 }
1392
1393 /// Finish describing the pipeline job.
1394 pub fn finish(self) -> PipelineJobHandle {
1395 PipelineJobHandle {
1396 job_idx: self.job_idx,
1397 }
1398 }
1399
1400 /// Return the job's platform.
1401 pub fn get_platform(&self) -> FlowPlatform {
1402 self.pipeline.jobs[self.job_idx].platform
1403 }
1404
1405 /// Return the job's architecture.
1406 pub fn get_arch(&self) -> FlowArch {
1407 self.pipeline.jobs[self.job_idx].arch
1408 }
1409}
1410
1411#[derive(Clone)]
1412pub struct PipelineJobHandle {
1413 job_idx: usize,
1414}
1415
1416impl PipelineJobHandle {
1417 pub fn is_handle_for(&self, job: &PipelineJob<'_>) -> bool {
1418 self.job_idx == job.job_idx
1419 }
1420}
1421
1422#[derive(Clone, Copy)]
1423pub enum PipelineBackendHint {
1424 /// Pipeline is being run on the user's dev machine (via bash / direct run)
1425 Local,
1426 /// Pipeline is run on ADO
1427 Ado,
1428 /// Pipeline is run on GitHub Actions
1429 Github,
1430}
1431
1432/// Trait for types that can be converted into a [`Pipeline`].
1433///
1434/// This is the primary entry point for defining flowey pipelines. Implement this trait
1435/// to create a pipeline definition that can be executed locally or converted to CI YAML.
1436///
1437/// # Example
1438///
1439/// ```rust,no_run
1440/// use flowey_core::pipeline::{IntoPipeline, Pipeline, PipelineBackendHint};
1441/// use flowey_core::node::{FlowPlatform, FlowPlatformLinuxDistro, FlowArch};
1442///
1443/// struct MyPipeline;
1444///
1445/// impl IntoPipeline for MyPipeline {
1446/// fn into_pipeline(self, backend_hint: PipelineBackendHint) -> anyhow::Result<Pipeline> {
1447/// let mut pipeline = Pipeline::new();
1448///
1449/// // Define a job that runs on Linux x86_64
1450/// let _job = pipeline
1451/// .new_job(
1452/// FlowPlatform::Linux(FlowPlatformLinuxDistro::Ubuntu),
1453/// FlowArch::X86_64,
1454/// "build"
1455/// )
1456/// .finish();
1457///
1458/// Ok(pipeline)
1459/// }
1460/// }
1461/// ```
1462///
1463/// # Complex Example with Parameters and Artifacts
1464///
1465/// ```rust,ignore
1466/// use flowey_core::pipeline::{IntoPipeline, Pipeline, PipelineBackendHint, ParameterKind};
1467/// use flowey_core::node::{FlowPlatform, FlowPlatformLinuxDistro, FlowArch};
1468///
1469/// struct BuildPipeline;
1470///
1471/// impl IntoPipeline for BuildPipeline {
1472/// fn into_pipeline(self, backend_hint: PipelineBackendHint) -> anyhow::Result<Pipeline> {
1473/// let mut pipeline = Pipeline::new();
1474///
1475/// // Define a runtime parameter
1476/// let enable_tests = pipeline.new_parameter_bool(
1477/// "enable_tests",
1478/// "Whether to run tests",
1479/// ParameterKind::Stable,
1480/// Some(true) // default value
1481/// );
1482///
1483/// // Create an artifact for passing data between jobs
1484/// let (publish_build, use_build) = pipeline.new_artifact("build-output");
1485///
1486/// // Job 1: Build
1487/// let build_job = pipeline
1488/// .new_job(
1489/// FlowPlatform::Linux(FlowPlatformLinuxDistro::Ubuntu),
1490/// FlowArch::X86_64,
1491/// "build"
1492/// )
1493/// .with_timeout_in_minutes(30)
1494/// .dep_on(|ctx| flowey_lib_hvlite::_jobs::example_node::Request {
1495/// output_dir: ctx.publish_artifact(publish_build),
1496/// })
1497/// .finish();
1498///
1499/// // Job 2: Test (conditionally run based on parameter)
1500/// let _test_job = pipeline
1501/// .new_job(
1502/// FlowPlatform::Linux(FlowPlatformLinuxDistro::Ubuntu),
1503/// FlowArch::X86_64,
1504/// "test"
1505/// )
1506/// .with_condition(enable_tests)
1507/// .dep_on(|ctx| flowey_lib_hvlite::_jobs::example_node2::Request {
1508/// input_dir: ctx.use_artifact(&use_build),
1509/// })
1510/// .finish();
1511///
1512/// Ok(pipeline)
1513/// }
1514/// }
1515/// ```
1516pub trait IntoPipeline {
1517 fn into_pipeline(self, backend_hint: PipelineBackendHint) -> anyhow::Result<Pipeline>;
1518}
1519
1520fn new_parameter_name(name: impl AsRef<str>, kind: ParameterKind) -> String {
1521 match kind {
1522 ParameterKind::Unstable => format!("__unstable_{}", name.as_ref()),
1523 ParameterKind::Stable => name.as_ref().into(),
1524 }
1525}
1526
1527/// Structs which should only be used by top-level flowey emitters. If you're a
1528/// pipeline author, these are not types you need to care about!
1529pub mod internal {
1530 use super::*;
1531 use std::collections::BTreeMap;
1532
1533 pub fn consistent_artifact_runtime_var_name(artifact: impl AsRef<str>, is_use: bool) -> String {
1534 format!(
1535 "artifact_{}_{}",
1536 if is_use { "use_from" } else { "publish_from" },
1537 artifact.as_ref()
1538 )
1539 }
1540
1541 #[derive(Debug)]
1542 pub struct InternalAdoResourcesRepository {
1543 /// flowey-generated unique repo identifier
1544 pub repo_id: String,
1545 /// Type of repo that is being connected to.
1546 pub repo_type: AdoResourcesRepositoryType,
1547 /// Repository name. Format depends on `repo_type`.
1548 pub name: String,
1549 /// git ref to checkout.
1550 pub git_ref: AdoResourcesRepositoryRef<usize>,
1551 /// (optional) ID of the service endpoint connecting to this repository.
1552 pub endpoint: Option<String>,
1553 }
1554
1555 pub struct PipelineJobMetadata {
1556 pub root_nodes: BTreeMap<NodeHandle, Vec<Box<[u8]>>>,
1557 pub root_configs: BTreeMap<NodeHandle, Vec<Box<[u8]>>>,
1558 pub patches: PatchResolver,
1559 pub label: String,
1560 pub platform: FlowPlatform,
1561 pub arch: FlowArch,
1562 pub cond_param_idx: Option<usize>,
1563 pub timeout_minutes: Option<u32>,
1564 pub command_wrapper: Option<crate::shell::CommandWrapperKind>,
1565 // backend specific
1566 pub ado_pool: Option<AdoPool>,
1567 pub ado_variables: BTreeMap<String, String>,
1568 pub gh_override_if: Option<String>,
1569 pub gh_pool: Option<GhRunner>,
1570 pub gh_concurrency_group: Option<GhConcurrencyGroup>,
1571 pub gh_global_env: BTreeMap<String, String>,
1572 pub gh_permissions: BTreeMap<NodeHandle, BTreeMap<GhPermission, GhPermissionValue>>,
1573 }
1574
1575 #[derive(Debug)]
1576 pub struct ArtifactMeta {
1577 pub name: String,
1578 pub published_by_job: Option<usize>,
1579 pub used_by_jobs: BTreeSet<usize>,
1580 }
1581
1582 #[derive(Debug)]
1583 pub struct ParameterMeta {
1584 pub parameter: Parameter,
1585 pub used_by_jobs: BTreeSet<usize>,
1586 }
1587
1588 /// Mirror of [`Pipeline`], except with all field marked as `pub`.
1589 pub struct PipelineFinalized {
1590 pub jobs: Vec<PipelineJobMetadata>,
1591 pub artifacts: Vec<ArtifactMeta>,
1592 pub parameters: Vec<ParameterMeta>,
1593 pub extra_deps: BTreeSet<(usize, usize)>,
1594 // backend specific
1595 pub ado_name: Option<String>,
1596 pub ado_schedule_triggers: Vec<AdoScheduleTriggers>,
1597 pub ado_ci_triggers: Option<AdoCiTriggers>,
1598 pub ado_pr_triggers: Option<AdoPrTriggers>,
1599 pub ado_bootstrap_template: String,
1600 pub ado_resources_repository: Vec<InternalAdoResourcesRepository>,
1601 pub ado_post_process_yaml_cb:
1602 Option<Box<dyn FnOnce(serde_yaml::Value) -> serde_yaml::Value>>,
1603 pub ado_variables: BTreeMap<String, String>,
1604 pub ado_job_id_overrides: BTreeMap<usize, String>,
1605 pub gh_name: Option<String>,
1606 pub gh_schedule_triggers: Vec<GhScheduleTriggers>,
1607 pub gh_ci_triggers: Option<GhCiTriggers>,
1608 pub gh_pr_triggers: Option<GhPrTriggers>,
1609 pub gh_bootstrap_template: String,
1610 }
1611
1612 impl PipelineFinalized {
1613 pub fn from_pipeline(mut pipeline: Pipeline) -> Self {
1614 if let Some(cb) = pipeline.inject_all_jobs_with.take() {
1615 for job_idx in 0..pipeline.jobs.len() {
1616 let _ = cb(PipelineJob {
1617 pipeline: &mut pipeline,
1618 job_idx,
1619 });
1620 }
1621 }
1622
1623 let Pipeline {
1624 mut jobs,
1625 artifacts,
1626 parameters,
1627 extra_deps,
1628 ado_name,
1629 ado_bootstrap_template,
1630 ado_schedule_triggers,
1631 ado_ci_triggers,
1632 ado_pr_triggers,
1633 ado_resources_repository,
1634 ado_post_process_yaml_cb,
1635 ado_variables,
1636 ado_job_id_overrides,
1637 gh_name,
1638 gh_schedule_triggers,
1639 gh_ci_triggers,
1640 gh_pr_triggers,
1641 gh_bootstrap_template,
1642 // not relevant to consumer code
1643 dummy_done_idx: _,
1644 artifact_map_idx: _,
1645 artifact_names: _,
1646 global_patchfns,
1647 inject_all_jobs_with: _, // processed above
1648 } = pipeline;
1649
1650 for patchfn in global_patchfns {
1651 for job in &mut jobs {
1652 job.patches.apply_patchfn(patchfn)
1653 }
1654 }
1655
1656 Self {
1657 jobs,
1658 artifacts,
1659 parameters,
1660 extra_deps,
1661 ado_name,
1662 ado_schedule_triggers,
1663 ado_ci_triggers,
1664 ado_pr_triggers,
1665 ado_bootstrap_template,
1666 ado_resources_repository,
1667 ado_post_process_yaml_cb,
1668 ado_variables,
1669 ado_job_id_overrides,
1670 gh_name,
1671 gh_schedule_triggers,
1672 gh_ci_triggers,
1673 gh_pr_triggers,
1674 gh_bootstrap_template,
1675 }
1676 }
1677 }
1678
1679 #[derive(Debug, Clone)]
1680 pub enum Parameter {
1681 Bool {
1682 name: String,
1683 description: String,
1684 kind: ParameterKind,
1685 default: Option<bool>,
1686 },
1687 String {
1688 name: String,
1689 description: String,
1690 default: Option<String>,
1691 kind: ParameterKind,
1692 possible_values: Option<Vec<String>>,
1693 },
1694 Num {
1695 name: String,
1696 description: String,
1697 default: Option<i64>,
1698 kind: ParameterKind,
1699 possible_values: Option<Vec<i64>>,
1700 },
1701 }
1702
1703 impl Parameter {
1704 pub fn name(&self) -> &str {
1705 match self {
1706 Parameter::Bool { name, .. } => name,
1707 Parameter::String { name, .. } => name,
1708 Parameter::Num { name, .. } => name,
1709 }
1710 }
1711 }
1712}