flowey_core/pipeline.rs
1// Copyright (c) Microsoft Corporation.
2// Licensed under the MIT License.
3
4//! Core types and traits used to create and work with flowey pipelines.
5
6mod artifact;
7
8pub use artifact::Artifact;
9pub use artifact::ArtifactType;
10pub use artifact::resolve;
11
12use self::internal::*;
13use crate::node::FlowArch;
14use crate::node::FlowNodeBase;
15use crate::node::FlowPlatform;
16use crate::node::FlowPlatformLinuxDistro;
17use crate::node::GhUserSecretVar;
18use crate::node::IntoConfig;
19use crate::node::IntoRequest;
20use crate::node::NodeHandle;
21use crate::node::ReadVar;
22use crate::node::WriteVar;
23use crate::node::steps::ado::AdoResourcesRepositoryId;
24use crate::node::user_facing::AdoRuntimeVar;
25use crate::node::user_facing::GhPermission;
26use crate::node::user_facing::GhPermissionValue;
27use crate::patch::PatchResolver;
28use crate::patch::ResolvedPatches;
29use serde::Serialize;
30use serde::de::DeserializeOwned;
31use std::collections::BTreeMap;
32use std::collections::BTreeSet;
33use std::path::PathBuf;
34
35/// Pipeline types which are considered "user facing", and included in the
36/// `flowey` prelude.
37pub mod user_facing {
38 pub use super::AdoCiTriggers;
39 pub use super::AdoPool;
40 pub use super::AdoPrTriggers;
41 pub use super::AdoResourcesRepository;
42 pub use super::AdoResourcesRepositoryRef;
43 pub use super::AdoResourcesRepositoryType;
44 pub use super::AdoScheduleTriggers;
45 pub use super::GhCiTriggers;
46 pub use super::GhConcurrencyGroup;
47 pub use super::GhPrTriggers;
48 pub use super::GhRunner;
49 pub use super::GhRunnerOsLabel;
50 pub use super::GhScheduleTriggers;
51 pub use super::HostExt;
52 pub use super::IntoPipeline;
53 pub use super::ParameterKind;
54 pub use super::Pipeline;
55 pub use super::PipelineBackendHint;
56 pub use super::PipelineJob;
57 pub use super::PipelineJobCtx;
58 pub use super::PipelineJobHandle;
59 pub use super::PublishArtifact;
60 pub use super::PublishTypedArtifact;
61 pub use super::UseArtifact;
62 pub use super::UseParameter;
63 pub use super::UseTypedArtifact;
64 pub use crate::node::FlowArch;
65 pub use crate::node::FlowPlatform;
66}
67
68fn linux_distro() -> FlowPlatformLinuxDistro {
69 // Check for nix environment first - takes precedence over distro detection
70 if std::env::var("IN_NIX_SHELL").is_ok() {
71 return FlowPlatformLinuxDistro::Nix;
72 }
73
74 // A `nix develop` shell doesn't set `IN_NIX_SHELL`, but the PATH should include a nix store path
75 if std::env::var("PATH").is_ok_and(|path| path.contains("/nix/store")) {
76 return FlowPlatformLinuxDistro::Nix;
77 }
78
79 if let Ok(etc_os_release) = fs_err::read_to_string("/etc/os-release") {
80 if etc_os_release.contains("ID=ubuntu") {
81 FlowPlatformLinuxDistro::Ubuntu
82 } else if etc_os_release.contains("ID=fedora") {
83 FlowPlatformLinuxDistro::Fedora
84 } else if etc_os_release.contains("ID=azurelinux") || etc_os_release.contains("ID=mariner")
85 {
86 FlowPlatformLinuxDistro::AzureLinux
87 } else if etc_os_release.contains("ID=arch") {
88 FlowPlatformLinuxDistro::Arch
89 } else {
90 FlowPlatformLinuxDistro::Unknown
91 }
92 } else {
93 FlowPlatformLinuxDistro::Unknown
94 }
95}
96
97pub trait HostExt: Sized {
98 /// Return the value for the current host machine.
99 ///
100 /// Will panic on non-local backends.
101 fn host(backend_hint: PipelineBackendHint) -> Self;
102}
103
104impl HostExt for FlowPlatform {
105 /// Return the platform of the current host machine.
106 ///
107 /// Will panic on non-local backends.
108 fn host(backend_hint: PipelineBackendHint) -> Self {
109 if !matches!(backend_hint, PipelineBackendHint::Local) {
110 panic!("can only use `FlowPlatform::host` when defining a local-only pipeline");
111 }
112
113 if cfg!(target_os = "windows") {
114 Self::Windows
115 } else if cfg!(target_os = "linux") {
116 Self::Linux(linux_distro())
117 } else if cfg!(target_os = "macos") {
118 Self::MacOs
119 } else {
120 panic!("no valid host-os")
121 }
122 }
123}
124
125impl HostExt for FlowArch {
126 /// Return the arch of the current host machine.
127 ///
128 /// Will panic on non-local backends.
129 fn host(backend_hint: PipelineBackendHint) -> Self {
130 if !matches!(backend_hint, PipelineBackendHint::Local) {
131 panic!("can only use `FlowArch::host` when defining a local-only pipeline");
132 }
133
134 // xtask-fmt allow-target-arch oneoff-flowey
135 if cfg!(target_arch = "x86_64") {
136 Self::X86_64
137 // xtask-fmt allow-target-arch oneoff-flowey
138 } else if cfg!(target_arch = "aarch64") {
139 Self::Aarch64
140 } else {
141 panic!("no valid host-arch")
142 }
143 }
144}
145
146/// Trigger ADO pipelines via Continuous Integration
147#[derive(Default, Debug)]
148pub struct AdoScheduleTriggers {
149 /// Friendly name for the scheduled run
150 pub display_name: String,
151 /// Run the pipeline whenever there is a commit on these specified branches
152 /// (supports glob syntax)
153 pub branches: Vec<String>,
154 /// Specify any branches which should be filtered out from the list of
155 /// `branches` (supports glob syntax)
156 pub exclude_branches: Vec<String>,
157 /// Run the pipeline in a schedule, as specified by a cron string
158 pub cron: String,
159}
160
161/// Trigger ADO pipelines per PR
162#[derive(Debug)]
163pub struct AdoPrTriggers {
164 /// Run the pipeline whenever there is a PR to these specified branches
165 /// (supports glob syntax)
166 pub branches: Vec<String>,
167 /// Specify any branches which should be filtered out from the list of
168 /// `branches` (supports glob syntax)
169 pub exclude_branches: Vec<String>,
170 /// Run the pipeline even if the PR is a draft PR. Defaults to `false`.
171 pub run_on_draft: bool,
172 /// Automatically cancel the pipeline run if a new commit lands in the
173 /// branch. Defaults to `true`.
174 pub auto_cancel: bool,
175 /// Only run the pipeline when files matching these paths are changed
176 /// (supports glob syntax)
177 pub paths: Vec<String>,
178 /// Specify any paths which should be filtered out from the list of
179 /// `paths` (supports glob syntax)
180 pub exclude_paths: Vec<String>,
181}
182
183/// Trigger ADO pipelines per CI
184#[derive(Debug, Default)]
185pub struct AdoCiTriggers {
186 /// Run the pipeline whenever there is a change to these specified branches
187 /// (supports glob syntax)
188 pub branches: Vec<String>,
189 /// Specify any branches which should be filtered out from the list of
190 /// `branches` (supports glob syntax)
191 pub exclude_branches: Vec<String>,
192 /// Run the pipeline whenever a matching tag is created (supports glob
193 /// syntax)
194 pub tags: Vec<String>,
195 /// Specify any tags which should be filtered out from the list of `tags`
196 /// (supports glob syntax)
197 pub exclude_tags: Vec<String>,
198 /// Whether to batch changes per branch.
199 pub batch: bool,
200 /// Only run the pipeline when files matching these paths are changed
201 /// (supports glob syntax)
202 pub paths: Vec<String>,
203 /// Specify any paths which should be filtered out from the list of
204 /// `paths` (supports glob syntax)
205 pub exclude_paths: Vec<String>,
206}
207
208impl Default for AdoPrTriggers {
209 fn default() -> Self {
210 Self {
211 branches: Vec::new(),
212 exclude_branches: Vec::new(),
213 run_on_draft: false,
214 auto_cancel: true,
215 paths: Vec::new(),
216 exclude_paths: Vec::new(),
217 }
218 }
219}
220
221/// ADO repository resource.
222#[derive(Debug)]
223pub struct AdoResourcesRepository {
224 /// Type of repo that is being connected to.
225 pub repo_type: AdoResourcesRepositoryType,
226 /// Repository name. Format depends on `repo_type`.
227 pub name: String,
228 /// git ref to checkout.
229 pub git_ref: AdoResourcesRepositoryRef,
230 /// (optional) ID of the service endpoint connecting to this repository.
231 pub endpoint: Option<String>,
232}
233
234/// ADO repository resource type
235#[derive(Debug)]
236pub enum AdoResourcesRepositoryType {
237 /// Azure Repos Git repository
238 AzureReposGit,
239 /// Github repository
240 GitHub,
241}
242
243/// ADO repository ref
244#[derive(Debug)]
245pub enum AdoResourcesRepositoryRef<P = UseParameter<String>> {
246 /// Hard-coded ref (e.g: refs/heads/main)
247 Fixed(String),
248 /// Connected to pipeline-level parameter
249 Parameter(P),
250}
251
252/// Trigger Github Actions pipelines via Continuous Integration
253///
254/// NOTE: Github Actions doesn't support specifying the branch when triggered by `schedule`.
255/// To run on a specific branch, modify the branch checked out in the pipeline.
256#[derive(Default, Debug)]
257pub struct GhScheduleTriggers {
258 /// Run the pipeline in a schedule, as specified by a cron string
259 pub cron: String,
260 /// IANA timezone string
261 pub timezone: Option<String>,
262}
263
264/// Trigger Github Actions pipelines per PR
265#[derive(Debug)]
266pub struct GhPrTriggers {
267 /// Run the pipeline whenever there is a PR to these specified branches
268 /// (supports glob syntax)
269 pub branches: Vec<String>,
270 /// Specify any branches which should be filtered out from the list of
271 /// `branches` (supports glob syntax)
272 pub exclude_branches: Vec<String>,
273 /// Automatically cancel the pipeline run if a new commit lands in the
274 /// branch. Defaults to `true`.
275 pub auto_cancel: bool,
276 /// Run the pipeline whenever the PR trigger matches the specified types
277 pub types: Vec<String>,
278 /// Only run the pipeline when files matching these paths are changed
279 /// (supports glob syntax)
280 pub paths: Vec<String>,
281 /// Specify any paths which should be filtered out from the list of
282 /// `paths` (supports glob syntax)
283 pub paths_ignore: Vec<String>,
284}
285
286/// Trigger Github Actions pipelines per PR
287#[derive(Debug, Default)]
288pub struct GhCiTriggers {
289 /// Run the pipeline whenever there is a change to these specified branches
290 /// (supports glob syntax)
291 pub branches: Vec<String>,
292 /// Specify any branches which should be filtered out from the list of
293 /// `branches` (supports glob syntax)
294 pub exclude_branches: Vec<String>,
295 /// Run the pipeline whenever a matching tag is created (supports glob
296 /// syntax)
297 pub tags: Vec<String>,
298 /// Specify any tags which should be filtered out from the list of `tags`
299 /// (supports glob syntax)
300 pub exclude_tags: Vec<String>,
301 /// Only run the pipeline when files matching these paths are changed
302 /// (supports glob syntax)
303 pub paths: Vec<String>,
304 /// Specify any paths which should be filtered out from the list of
305 /// `paths` (supports glob syntax)
306 pub paths_ignore: Vec<String>,
307 /// If set, only one pipeline run in this concurrency group runs at a time.
308 pub concurrency_group: Option<GhConcurrencyGroup>,
309}
310
311/// Settings for a Github concurrency group.
312#[derive(Debug, Default, Clone)]
313pub struct GhConcurrencyGroup {
314 /// The name of the concurrency group.
315 pub name: String,
316 /// Cancel the active run when a new run joins the concurrency group.
317 /// Defaults to `false`.
318 pub cancel_in_progress: bool,
319}
320
321impl GhPrTriggers {
322 /// Triggers the pipeline on the default PR events plus when a draft is marked as ready for review.
323 pub fn new_draftable() -> Self {
324 Self {
325 branches: Vec::new(),
326 exclude_branches: Vec::new(),
327 types: vec![
328 "opened".into(),
329 "synchronize".into(),
330 "reopened".into(),
331 "ready_for_review".into(),
332 ],
333 auto_cancel: true,
334 paths: Vec::new(),
335 paths_ignore: Vec::new(),
336 }
337 }
338}
339
340#[derive(Debug, Clone, PartialEq)]
341pub enum GhRunnerOsLabel {
342 UbuntuLatest,
343 Ubuntu2404,
344 Ubuntu2204,
345 WindowsLatest,
346 Windows2025,
347 Windows2022,
348 Ubuntu2404Arm,
349 Ubuntu2204Arm,
350 Windows11Arm,
351 Custom(String),
352}
353
354/// GitHub runner type
355#[derive(Debug, Clone, PartialEq)]
356pub enum GhRunner {
357 // See <https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#choosing-github-hosted-runners>
358 // for more details.
359 GhHosted(GhRunnerOsLabel),
360 // Self hosted runners are selected by matching runner labels to <labels>.
361 // 'self-hosted' is a common label for self hosted runners, but is not required.
362 // Labels are case-insensitive and can take the form of arbitrary strings.
363 // See <https://docs.github.com/en/actions/hosting-your-own-runners> for more details.
364 SelfHosted(Vec<String>),
365 // This uses a runner belonging to <group> that matches all labels in <labels>.
366 // See <https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#choosing-github-hosted-runners>
367 // for more details.
368 RunnerGroup { group: String, labels: Vec<String> },
369}
370
371impl GhRunner {
372 /// Whether this is a self-hosted runner with the provided label
373 pub fn is_self_hosted_with_label(&self, label: &str) -> bool {
374 matches!(self, GhRunner::SelfHosted(labels) if labels.iter().any(|s| s.as_str() == label))
375 }
376}
377
378// TODO: support a more structured format for demands
379// See https://learn.microsoft.com/en-us/azure/devops/pipelines/yaml-schema/pool-demands
380#[derive(Debug, Clone)]
381pub struct AdoPool {
382 pub name: String,
383 pub demands: Vec<String>,
384}
385
386/// Parameter type (unstable / stable).
387#[derive(Debug, Clone)]
388pub enum ParameterKind {
389 // The parameter is considered an unstable API, and should not be
390 // taken as a dependency.
391 Unstable,
392 // The parameter is considered a stable API, and can be used by
393 // external pipelines to control behavior of the pipeline.
394 Stable,
395}
396
397#[derive(Clone, Debug)]
398#[must_use]
399pub struct UseParameter<T> {
400 idx: usize,
401 _kind: std::marker::PhantomData<T>,
402}
403
404/// Opaque handle to an artifact which must be published by a single job.
405#[must_use]
406pub struct PublishArtifact {
407 idx: usize,
408}
409
410/// Opaque handle to an artifact which can be used by one or more jobs.
411#[derive(Clone)]
412#[must_use]
413pub struct UseArtifact {
414 idx: usize,
415}
416
417/// Opaque handle to an artifact of type `T` which must be published by a single job.
418#[must_use]
419pub struct PublishTypedArtifact<T>(PublishArtifact, std::marker::PhantomData<fn() -> T>);
420
421/// Opaque handle to an artifact of type `T` which can be used by one or more
422/// jobs.
423#[must_use]
424pub struct UseTypedArtifact<T>(UseArtifact, std::marker::PhantomData<fn(T)>);
425
426impl<T> Clone for UseTypedArtifact<T> {
427 fn clone(&self) -> Self {
428 UseTypedArtifact(self.0.clone(), std::marker::PhantomData)
429 }
430}
431
432#[derive(Default)]
433pub struct Pipeline {
434 jobs: Vec<PipelineJobMetadata>,
435 artifacts: Vec<ArtifactMeta>,
436 parameters: Vec<ParameterMeta>,
437 extra_deps: BTreeSet<(usize, usize)>,
438 // builder internal
439 artifact_names: BTreeSet<String>,
440 dummy_done_idx: usize,
441 artifact_map_idx: usize,
442 global_patchfns: Vec<crate::patch::PatchFn>,
443 inject_all_jobs_with: Option<Box<dyn for<'a> Fn(PipelineJob<'a>) -> PipelineJob<'a>>>,
444 // backend specific
445 ado_name: Option<String>,
446 ado_job_id_overrides: BTreeMap<usize, String>,
447 ado_schedule_triggers: Vec<AdoScheduleTriggers>,
448 ado_ci_triggers: Option<AdoCiTriggers>,
449 ado_pr_triggers: Option<AdoPrTriggers>,
450 ado_resources_repository: Vec<InternalAdoResourcesRepository>,
451 ado_bootstrap_template: String,
452 ado_variables: BTreeMap<String, String>,
453 ado_post_process_yaml_cb: Option<Box<dyn FnOnce(serde_yaml::Value) -> serde_yaml::Value>>,
454 gh_name: Option<String>,
455 gh_schedule_triggers: Vec<GhScheduleTriggers>,
456 gh_ci_triggers: Option<GhCiTriggers>,
457 gh_pr_triggers: Option<GhPrTriggers>,
458 gh_bootstrap_template: String,
459}
460
461impl Pipeline {
462 pub fn new() -> Pipeline {
463 Pipeline::default()
464 }
465
466 /// Inject all pipeline jobs with some common logic. (e.g: to resolve common
467 /// configuration requirements shared by all jobs).
468 ///
469 /// Can only be invoked once per pipeline.
470 #[track_caller]
471 pub fn inject_all_jobs_with(
472 &mut self,
473 cb: impl for<'a> Fn(PipelineJob<'a>) -> PipelineJob<'a> + 'static,
474 ) -> &mut Self {
475 if self.inject_all_jobs_with.is_some() {
476 panic!("can only call inject_all_jobs_with once!")
477 }
478 self.inject_all_jobs_with = Some(Box::new(cb));
479 self
480 }
481
482 /// (ADO only) Provide a YAML template used to bootstrap flowey at the start
483 /// of an ADO pipeline.
484 ///
485 /// The template has access to the following vars, which will be statically
486 /// interpolated into the template's text:
487 ///
488 /// - `{{FLOWEY_OUTDIR}}`
489 /// - Directory to copy artifacts into.
490 /// - NOTE: this var will include `\` on Windows, and `/` on linux!
491 /// - `{{FLOWEY_BIN_EXTENSION}}`
492 /// - Extension of the expected flowey bin (either "", or ".exe")
493 /// - `{{FLOWEY_CRATE}}`
494 /// - Name of the project-specific flowey crate to be built
495 /// - `{{FLOWEY_TARGET}}`
496 /// - The target-triple flowey is being built for
497 /// - `{{FLOWEY_PIPELINE_PATH}}`
498 /// - Repo-root relative path to the pipeline (as provided when
499 /// generating the pipeline via the flowey CLI)
500 ///
501 /// The template's sole responsibility is to copy 3 files into the
502 /// `{{FLOWEY_OUTDIR}}`:
503 ///
504 /// 1. The bootstrapped flowey binary, with the file name
505 /// `flowey{{FLOWEY_BIN_EXTENSION}}`
506 /// 2. Two files called `pipeline.yaml` and `pipeline.json`, which are
507 /// copied of the pipeline YAML and pipeline JSON currently being run.
508 /// `{{FLOWEY_PIPELINE_PATH}}` is provided as a way to disambiguate in
509 /// cases where the same template is being for multiple pipelines (e.g: a
510 /// debug vs. release pipeline).
511 pub fn ado_set_flowey_bootstrap_template(&mut self, template: String) -> &mut Self {
512 self.ado_bootstrap_template = template;
513 self
514 }
515
516 /// (ADO only) Provide a callback function which will be used to
517 /// post-process any YAML flowey generates for the pipeline.
518 ///
519 /// Until flowey defines a stable API for maintaining out-of-tree backends,
520 /// this method can be used to integrate the output from the generic ADO
521 /// backend with any organization-specific templates that one may be
522 /// required to use (e.g: for compliance reasons).
523 pub fn ado_post_process_yaml(
524 &mut self,
525 cb: impl FnOnce(serde_yaml::Value) -> serde_yaml::Value + 'static,
526 ) -> &mut Self {
527 self.ado_post_process_yaml_cb = Some(Box::new(cb));
528 self
529 }
530
531 /// (ADO only) Add a new scheduled CI trigger. Can be called multiple times
532 /// to set up multiple schedules runs.
533 pub fn ado_add_schedule_trigger(&mut self, triggers: AdoScheduleTriggers) -> &mut Self {
534 self.ado_schedule_triggers.push(triggers);
535 self
536 }
537
538 /// (ADO only) Set a PR trigger. Calling this method multiple times will
539 /// overwrite any previously set triggers.
540 pub fn ado_set_pr_triggers(&mut self, triggers: AdoPrTriggers) -> &mut Self {
541 self.ado_pr_triggers = Some(triggers);
542 self
543 }
544
545 /// (ADO only) Set a CI trigger. Calling this method multiple times will
546 /// overwrite any previously set triggers.
547 pub fn ado_set_ci_triggers(&mut self, triggers: AdoCiTriggers) -> &mut Self {
548 self.ado_ci_triggers = Some(triggers);
549 self
550 }
551
552 /// (ADO only) Declare a new repository resource, returning a type-safe
553 /// handle which downstream ADO steps are able to consume via
554 /// [`AdoStepServices::resolve_repository_id`](crate::node::user_facing::AdoStepServices::resolve_repository_id).
555 pub fn ado_add_resources_repository(
556 &mut self,
557 repo: AdoResourcesRepository,
558 ) -> AdoResourcesRepositoryId {
559 let AdoResourcesRepository {
560 repo_type,
561 name,
562 git_ref,
563 endpoint,
564 } = repo;
565
566 let repo_id = format!("repo{}", self.ado_resources_repository.len());
567
568 self.ado_resources_repository
569 .push(InternalAdoResourcesRepository {
570 repo_id: repo_id.clone(),
571 repo_type,
572 name,
573 git_ref: match git_ref {
574 AdoResourcesRepositoryRef::Fixed(s) => AdoResourcesRepositoryRef::Fixed(s),
575 AdoResourcesRepositoryRef::Parameter(p) => {
576 AdoResourcesRepositoryRef::Parameter(p.idx)
577 }
578 },
579 endpoint,
580 });
581 AdoResourcesRepositoryId { repo_id }
582 }
583
584 /// (GitHub Actions only) Set the pipeline-level name.
585 ///
586 /// <https://docs.github.com/en/actions/writing-workflows/workflow-syntax-for-github-actions#name>
587 pub fn gh_set_name(&mut self, name: impl AsRef<str>) -> &mut Self {
588 self.gh_name = Some(name.as_ref().into());
589 self
590 }
591
592 /// Provide a YAML template used to bootstrap flowey at the start of an GitHub
593 /// pipeline.
594 ///
595 /// The template has access to the following vars, which will be statically
596 /// interpolated into the template's text:
597 ///
598 /// - `{{FLOWEY_OUTDIR}}`
599 /// - Directory to copy artifacts into.
600 /// - NOTE: this var will include `\` on Windows, and `/` on linux!
601 /// - `{{FLOWEY_BIN_EXTENSION}}`
602 /// - Extension of the expected flowey bin (either "", or ".exe")
603 /// - `{{FLOWEY_CRATE}}`
604 /// - Name of the project-specific flowey crate to be built
605 /// - `{{FLOWEY_TARGET}}`
606 /// - The target-triple flowey is being built for
607 /// - `{{FLOWEY_PIPELINE_PATH}}`
608 /// - Repo-root relative path to the pipeline (as provided when
609 /// generating the pipeline via the flowey CLI)
610 ///
611 /// The template's sole responsibility is to copy 3 files into the
612 /// `{{FLOWEY_OUTDIR}}`:
613 ///
614 /// 1. The bootstrapped flowey binary, with the file name
615 /// `flowey{{FLOWEY_BIN_EXTENSION}}`
616 /// 2. Two files called `pipeline.yaml` and `pipeline.json`, which are
617 /// copied of the pipeline YAML and pipeline JSON currently being run.
618 /// `{{FLOWEY_PIPELINE_PATH}}` is provided as a way to disambiguate in
619 /// cases where the same template is being for multiple pipelines (e.g: a
620 /// debug vs. release pipeline).
621 pub fn gh_set_flowey_bootstrap_template(&mut self, template: String) -> &mut Self {
622 self.gh_bootstrap_template = template;
623 self
624 }
625
626 /// (GitHub Actions only) Add a new scheduled CI trigger. Can be called multiple times
627 /// to set up multiple schedules runs.
628 pub fn gh_add_schedule_trigger(&mut self, triggers: GhScheduleTriggers) -> &mut Self {
629 self.gh_schedule_triggers.push(triggers);
630 self
631 }
632
633 /// (GitHub Actions only) Set a PR trigger. Calling this method multiple times will
634 /// overwrite any previously set triggers.
635 pub fn gh_set_pr_triggers(&mut self, triggers: GhPrTriggers) -> &mut Self {
636 self.gh_pr_triggers = Some(triggers);
637 self
638 }
639
640 /// (GitHub Actions only) Set a CI trigger. Calling this method multiple times will
641 /// overwrite any previously set triggers.
642 pub fn gh_set_ci_triggers(&mut self, triggers: GhCiTriggers) -> &mut Self {
643 self.gh_ci_triggers = Some(triggers);
644 self
645 }
646
647 /// (GitHub Actions only) Use a pre-defined GitHub Actions secret variable.
648 ///
649 /// For more information on defining secrets for use in GitHub Actions, see
650 /// <https://docs.github.com/en/actions/security-guides/using-secrets-in-github-actions>
651 pub fn gh_use_secret(&mut self, secret_name: impl AsRef<str>) -> GhUserSecretVar {
652 GhUserSecretVar(secret_name.as_ref().to_string())
653 }
654
655 pub fn new_job(
656 &mut self,
657 platform: FlowPlatform,
658 arch: FlowArch,
659 label: impl AsRef<str>,
660 ) -> PipelineJob<'_> {
661 let idx = self.jobs.len();
662 self.jobs.push(PipelineJobMetadata {
663 root_nodes: BTreeMap::new(),
664 root_configs: BTreeMap::new(),
665 patches: ResolvedPatches::build(),
666 label: label.as_ref().into(),
667 platform,
668 arch,
669 cond_param_idx: None,
670 timeout_minutes: None,
671 command_wrapper: None,
672 ado_pool: None,
673 ado_variables: BTreeMap::new(),
674 gh_override_if: None,
675 gh_global_env: BTreeMap::new(),
676 gh_pool: None,
677 gh_concurrency_group: None,
678 gh_permissions: BTreeMap::new(),
679 });
680
681 PipelineJob {
682 pipeline: self,
683 job_idx: idx,
684 }
685 }
686
687 /// Declare a dependency between two jobs that does is not a result of an
688 /// artifact.
689 pub fn non_artifact_dep(
690 &mut self,
691 job: &PipelineJobHandle,
692 depends_on_job: &PipelineJobHandle,
693 ) -> &mut Self {
694 self.extra_deps
695 .insert((depends_on_job.job_idx, job.job_idx));
696 self
697 }
698
699 #[track_caller]
700 pub fn new_artifact(&mut self, name: impl AsRef<str>) -> (PublishArtifact, UseArtifact) {
701 let name = name.as_ref();
702 let owned_name = name.to_string();
703
704 let not_exists = self.artifact_names.insert(owned_name.clone());
705 if !not_exists {
706 panic!("duplicate artifact name: {}", name)
707 }
708
709 let idx = self.artifacts.len();
710 self.artifacts.push(ArtifactMeta {
711 name: owned_name,
712 published_by_job: None,
713 used_by_jobs: BTreeSet::new(),
714 });
715
716 (PublishArtifact { idx }, UseArtifact { idx })
717 }
718
719 /// Returns a pair of opaque handles to a new artifact for use across jobs
720 /// in the pipeline.
721 #[track_caller]
722 pub fn new_typed_artifact<T: Artifact>(
723 &mut self,
724 name: impl AsRef<str>,
725 ) -> (PublishTypedArtifact<T>, UseTypedArtifact<T>) {
726 let (publish, use_artifact) = self.new_artifact(name);
727 (
728 PublishTypedArtifact(publish, std::marker::PhantomData),
729 UseTypedArtifact(use_artifact, std::marker::PhantomData),
730 )
731 }
732
733 /// Returns a pair of sets of opaque handles to a new artifact for use
734 /// across jobs in the pipeline. The artifact names are derived by the impl
735 /// of [`ArtifactType::name`] using common prefixes and suffixes if
736 /// specified (although the implementor can choose to use those values
737 /// differently).
738 #[track_caller]
739 pub fn new_typed_artifact_collection<T: Artifact, U: ArtifactType>(
740 &mut self,
741 artifact_types: impl IntoIterator<Item = U>,
742 prefix: Option<&str>,
743 suffix: Option<&str>,
744 ) -> (
745 BTreeMap<U, PublishTypedArtifact<T>>,
746 BTreeMap<U, UseTypedArtifact<T>>,
747 ) {
748 artifact_types
749 .into_iter()
750 .map(|artifact_type| {
751 let (pub_artifact, use_artifact) =
752 self.new_typed_artifact(artifact_type.name(prefix, suffix));
753 (
754 (artifact_type.clone(), pub_artifact),
755 (artifact_type, use_artifact),
756 )
757 })
758 .unzip()
759 }
760
761 /// (ADO only) Set the pipeline-level name.
762 ///
763 /// <https://learn.microsoft.com/en-us/azure/devops/pipelines/process/run-number?view=azure-devops&tabs=yaml>
764 pub fn ado_add_name(&mut self, name: String) -> &mut Self {
765 self.ado_name = Some(name);
766 self
767 }
768
769 /// (ADO only) Declare a pipeline-level, named, read-only ADO variable.
770 ///
771 /// `name` and `value` are both arbitrary strings.
772 ///
773 /// Returns an instance of [`AdoRuntimeVar`], which, if need be, can be
774 /// converted into a [`ReadVar<String>`] using
775 /// [`NodeCtx::get_ado_variable`].
776 ///
777 /// NOTE: Unless required by some particular third-party task, it's strongly
778 /// recommended to _avoid_ using this method, and to simply use
779 /// [`ReadVar::from_static`] to get a obtain a static variable.
780 ///
781 /// [`NodeCtx::get_ado_variable`]: crate::node::NodeCtx::get_ado_variable
782 pub fn ado_new_named_variable(
783 &mut self,
784 name: impl AsRef<str>,
785 value: impl AsRef<str>,
786 ) -> AdoRuntimeVar {
787 let name = name.as_ref();
788 let value = value.as_ref();
789
790 self.ado_variables.insert(name.into(), value.into());
791
792 // safe, since we'll ensure that the global exists in the ADO backend
793 AdoRuntimeVar::dangerous_from_global(name, false)
794 }
795
796 /// (ADO only) Declare multiple pipeline-level, named, read-only ADO
797 /// variables at once.
798 ///
799 /// This is a convenience method to streamline invoking
800 /// [`Self::ado_new_named_variable`] multiple times.
801 ///
802 /// NOTE: Unless required by some particular third-party task, it's strongly
803 /// recommended to _avoid_ using this method, and to simply use
804 /// [`ReadVar::from_static`] to get a obtain a static variable.
805 ///
806 /// DEVNOTE: In the future, this API may be updated to return a handle that
807 /// will allow resolving the resulting `AdoRuntimeVar`, but for
808 /// implementation expediency, this API does not currently do this. If you
809 /// need to read the value of this variable at runtime, you may need to
810 /// invoke [`AdoRuntimeVar::dangerous_from_global`] manually.
811 ///
812 /// [`NodeCtx::get_ado_variable`]: crate::node::NodeCtx::get_ado_variable
813 pub fn ado_new_named_variables<K, V>(
814 &mut self,
815 vars: impl IntoIterator<Item = (K, V)>,
816 ) -> &mut Self
817 where
818 K: AsRef<str>,
819 V: AsRef<str>,
820 {
821 self.ado_variables.extend(
822 vars.into_iter()
823 .map(|(k, v)| (k.as_ref().into(), v.as_ref().into())),
824 );
825 self
826 }
827
828 /// Declare a pipeline-level runtime parameter with type `bool`.
829 ///
830 /// To obtain a [`ReadVar<bool>`] that can be used within a node, use the
831 /// [`PipelineJobCtx::use_parameter`] method.
832 ///
833 /// `name` is the name of the parameter.
834 ///
835 /// `description` is an arbitrary string, which will be be shown to users.
836 ///
837 /// `kind` is the type of parameter and if it should be treated as a stable
838 /// external API to callers of the pipeline.
839 ///
840 /// `default` is the default value for the parameter. If none is provided,
841 /// the parameter _must_ be specified in order for the pipeline to run.
842 ///
843 /// `possible_values` can be used to limit the set of valid values the
844 /// parameter accepts.
845 pub fn new_parameter_bool(
846 &mut self,
847 name: impl AsRef<str>,
848 description: impl AsRef<str>,
849 kind: ParameterKind,
850 default: Option<bool>,
851 ) -> UseParameter<bool> {
852 let idx = self.parameters.len();
853 let name = new_parameter_name(name, kind.clone());
854 self.parameters.push(ParameterMeta {
855 parameter: Parameter::Bool {
856 name,
857 description: description.as_ref().into(),
858 kind,
859 default,
860 },
861 used_by_jobs: BTreeSet::new(),
862 });
863
864 UseParameter {
865 idx,
866 _kind: std::marker::PhantomData,
867 }
868 }
869
870 /// Declare a pipeline-level runtime parameter with type `i64`.
871 ///
872 /// To obtain a [`ReadVar<i64>`] that can be used within a node, use the
873 /// [`PipelineJobCtx::use_parameter`] method.
874 ///
875 /// `name` is the name of the parameter.
876 ///
877 /// `description` is an arbitrary string, which will be be shown to users.
878 ///
879 /// `kind` is the type of parameter and if it should be treated as a stable
880 /// external API to callers of the pipeline.
881 ///
882 /// `default` is the default value for the parameter. If none is provided,
883 /// the parameter _must_ be specified in order for the pipeline to run.
884 ///
885 /// `possible_values` can be used to limit the set of valid values the
886 /// parameter accepts.
887 pub fn new_parameter_num(
888 &mut self,
889 name: impl AsRef<str>,
890 description: impl AsRef<str>,
891 kind: ParameterKind,
892 default: Option<i64>,
893 possible_values: Option<Vec<i64>>,
894 ) -> UseParameter<i64> {
895 let idx = self.parameters.len();
896 let name = new_parameter_name(name, kind.clone());
897 self.parameters.push(ParameterMeta {
898 parameter: Parameter::Num {
899 name,
900 description: description.as_ref().into(),
901 kind,
902 default,
903 possible_values,
904 },
905 used_by_jobs: BTreeSet::new(),
906 });
907
908 UseParameter {
909 idx,
910 _kind: std::marker::PhantomData,
911 }
912 }
913
914 /// Declare a pipeline-level runtime parameter with type `String`.
915 ///
916 /// To obtain a [`ReadVar<String>`] that can be used within a node, use the
917 /// [`PipelineJobCtx::use_parameter`] method.
918 ///
919 /// `name` is the name of the parameter.
920 ///
921 /// `description` is an arbitrary string, which will be be shown to users.
922 ///
923 /// `kind` is the type of parameter and if it should be treated as a stable
924 /// external API to callers of the pipeline.
925 ///
926 /// `default` is the default value for the parameter. If none is provided,
927 /// the parameter _must_ be specified in order for the pipeline to run.
928 ///
929 /// `possible_values` allows restricting inputs to a set of possible values.
930 /// Depending on the backend, these options may be presented as a set of
931 /// radio buttons, a dropdown menu, or something in that vein. If `None`,
932 /// then any string is allowed.
933 pub fn new_parameter_string(
934 &mut self,
935 name: impl AsRef<str>,
936 description: impl AsRef<str>,
937 kind: ParameterKind,
938 default: Option<impl AsRef<str>>,
939 possible_values: Option<Vec<String>>,
940 ) -> UseParameter<String> {
941 let idx = self.parameters.len();
942 let name = new_parameter_name(name, kind.clone());
943 self.parameters.push(ParameterMeta {
944 parameter: Parameter::String {
945 name,
946 description: description.as_ref().into(),
947 kind,
948 default: default.map(|x| x.as_ref().into()),
949 possible_values,
950 },
951 used_by_jobs: BTreeSet::new(),
952 });
953
954 UseParameter {
955 idx,
956 _kind: std::marker::PhantomData,
957 }
958 }
959}
960
961pub struct PipelineJobCtx<'a> {
962 pipeline: &'a mut Pipeline,
963 job_idx: usize,
964}
965
966impl PipelineJobCtx<'_> {
967 /// Create a new `WriteVar<SideEffect>` anchored to the pipeline job.
968 pub fn new_done_handle(&mut self) -> WriteVar<crate::node::SideEffect> {
969 self.pipeline.dummy_done_idx += 1;
970 crate::node::thin_air_write_runtime_var(format!("start{}", self.pipeline.dummy_done_idx))
971 }
972
973 /// Claim that this job will use this artifact, obtaining a path to a folder
974 /// with the artifact's contents.
975 pub fn use_artifact(&mut self, artifact: &UseArtifact) -> ReadVar<PathBuf> {
976 self.pipeline.artifacts[artifact.idx]
977 .used_by_jobs
978 .insert(self.job_idx);
979
980 crate::node::thin_air_read_runtime_var(consistent_artifact_runtime_var_name(
981 &self.pipeline.artifacts[artifact.idx].name,
982 true,
983 ))
984 }
985
986 /// Claim that this job will publish this artifact, obtaining a path to a
987 /// fresh, empty folder which will be published as the specific artifact at
988 /// the end of the job.
989 pub fn publish_artifact(&mut self, artifact: PublishArtifact) -> ReadVar<PathBuf> {
990 let existing = self.pipeline.artifacts[artifact.idx]
991 .published_by_job
992 .replace(self.job_idx);
993 assert!(existing.is_none()); // PublishArtifact isn't cloneable
994
995 crate::node::thin_air_read_runtime_var(consistent_artifact_runtime_var_name(
996 &self.pipeline.artifacts[artifact.idx].name,
997 false,
998 ))
999 }
1000
1001 fn helper_request<R: IntoRequest>(&mut self, req: R)
1002 where
1003 R::Node: 'static,
1004 {
1005 self.pipeline.jobs[self.job_idx]
1006 .root_nodes
1007 .entry(NodeHandle::from_type::<R::Node>())
1008 .or_default()
1009 .push(serde_json::to_vec(&req.into_request()).unwrap().into());
1010 }
1011
1012 fn new_artifact_map_vars<T: Artifact>(&mut self) -> (ReadVar<T>, WriteVar<T>) {
1013 let artifact_map_idx = self.pipeline.artifact_map_idx;
1014 self.pipeline.artifact_map_idx += 1;
1015
1016 let backing_var = format!("artifact_map{}", artifact_map_idx);
1017 let read_var = crate::node::thin_air_read_runtime_var(backing_var.clone());
1018 let write_var = crate::node::thin_air_write_runtime_var(backing_var);
1019 (read_var, write_var)
1020 }
1021
1022 /// Claim that this job will use this artifact, obtaining the resolved
1023 /// contents of the artifact.
1024 pub fn use_typed_artifact<T: Artifact>(
1025 &mut self,
1026 artifact: &UseTypedArtifact<T>,
1027 ) -> ReadVar<T> {
1028 let artifact_path = self.use_artifact(&artifact.0);
1029 let (read, write) = self.new_artifact_map_vars::<T>();
1030 self.helper_request(resolve::Request::new(artifact_path, write));
1031 read
1032 }
1033
1034 /// Claim that this job will publish this artifact, obtaining a variable to
1035 /// write the artifact's contents to. The artifact will be published at
1036 /// the end of the job.
1037 pub fn publish_typed_artifact<T: Artifact>(
1038 &mut self,
1039 artifact: PublishTypedArtifact<T>,
1040 ) -> WriteVar<T> {
1041 let artifact_path = self.publish_artifact(artifact.0);
1042 let (read, write) = self.new_artifact_map_vars::<T>();
1043 let done = self.new_done_handle();
1044 self.helper_request(artifact::publish::Request::new(read, artifact_path, done));
1045 write
1046 }
1047
1048 /// Obtain a `ReadVar<T>` corresponding to a pipeline parameter which is
1049 /// specified at runtime.
1050 pub fn use_parameter<T>(&mut self, param: UseParameter<T>) -> ReadVar<T>
1051 where
1052 T: Serialize + DeserializeOwned,
1053 {
1054 self.pipeline.parameters[param.idx]
1055 .used_by_jobs
1056 .insert(self.job_idx);
1057
1058 crate::node::thin_air_read_runtime_var(
1059 self.pipeline.parameters[param.idx]
1060 .parameter
1061 .name()
1062 .to_string(),
1063 )
1064 }
1065
1066 /// Shortcut which allows defining a bool pipeline parameter within a Job.
1067 ///
1068 /// To share a single parameter between multiple jobs, don't use this method
1069 /// - use [`Pipeline::new_parameter_bool`] + [`Self::use_parameter`] instead.
1070 pub fn new_parameter_bool(
1071 &mut self,
1072 name: impl AsRef<str>,
1073 description: impl AsRef<str>,
1074 kind: ParameterKind,
1075 default: Option<bool>,
1076 ) -> ReadVar<bool> {
1077 let param = self
1078 .pipeline
1079 .new_parameter_bool(name, description, kind, default);
1080 self.use_parameter(param)
1081 }
1082
1083 /// Shortcut which allows defining a number pipeline parameter within a Job.
1084 ///
1085 /// To share a single parameter between multiple jobs, don't use this method
1086 /// - use [`Pipeline::new_parameter_num`] + [`Self::use_parameter`] instead.
1087 pub fn new_parameter_num(
1088 &mut self,
1089 name: impl AsRef<str>,
1090 description: impl AsRef<str>,
1091 kind: ParameterKind,
1092 default: Option<i64>,
1093 possible_values: Option<Vec<i64>>,
1094 ) -> ReadVar<i64> {
1095 let param =
1096 self.pipeline
1097 .new_parameter_num(name, description, kind, default, possible_values);
1098 self.use_parameter(param)
1099 }
1100
1101 /// Shortcut which allows defining a string pipeline parameter within a Job.
1102 ///
1103 /// To share a single parameter between multiple jobs, don't use this method
1104 /// - use [`Pipeline::new_parameter_string`] + [`Self::use_parameter`] instead.
1105 pub fn new_parameter_string(
1106 &mut self,
1107 name: impl AsRef<str>,
1108 description: impl AsRef<str>,
1109 kind: ParameterKind,
1110 default: Option<String>,
1111 possible_values: Option<Vec<String>>,
1112 ) -> ReadVar<String> {
1113 let param =
1114 self.pipeline
1115 .new_parameter_string(name, description, kind, default, possible_values);
1116 self.use_parameter(param)
1117 }
1118}
1119
1120#[must_use]
1121pub struct PipelineJob<'a> {
1122 pipeline: &'a mut Pipeline,
1123 job_idx: usize,
1124}
1125
1126impl PipelineJob<'_> {
1127 /// (ADO only) specify which agent pool this job will be run on.
1128 pub fn ado_set_pool(self, pool: AdoPool) -> Self {
1129 self.pipeline.jobs[self.job_idx].ado_pool = Some(pool);
1130 self
1131 }
1132
1133 /// (ADO only) specify which agent pool this job will be run on, with
1134 /// additional special runner demands.
1135 pub fn ado_set_pool_with_demands(self, pool: impl AsRef<str>, demands: Vec<String>) -> Self {
1136 self.pipeline.jobs[self.job_idx].ado_pool = Some(AdoPool {
1137 name: pool.as_ref().into(),
1138 demands,
1139 });
1140 self
1141 }
1142
1143 /// (ADO only) Declare a job-level, named, read-only ADO variable.
1144 ///
1145 /// `name` and `value` are both arbitrary strings, which may include ADO
1146 /// template expressions.
1147 ///
1148 /// NOTE: Unless required by some particular third-party task, it's strongly
1149 /// recommended to _avoid_ using this method, and to simply use
1150 /// [`ReadVar::from_static`] to get a obtain a static variable.
1151 ///
1152 /// DEVNOTE: In the future, this API may be updated to return a handle that
1153 /// will allow resolving the resulting `AdoRuntimeVar`, but for
1154 /// implementation expediency, this API does not currently do this. If you
1155 /// need to read the value of this variable at runtime, you may need to
1156 /// invoke [`AdoRuntimeVar::dangerous_from_global`] manually.
1157 ///
1158 /// [`NodeCtx::get_ado_variable`]: crate::node::NodeCtx::get_ado_variable
1159 pub fn ado_new_named_variable(self, name: impl AsRef<str>, value: impl AsRef<str>) -> Self {
1160 let name = name.as_ref();
1161 let value = value.as_ref();
1162 self.pipeline.jobs[self.job_idx]
1163 .ado_variables
1164 .insert(name.into(), value.into());
1165 self
1166 }
1167
1168 /// (ADO only) Declare multiple job-level, named, read-only ADO variables at
1169 /// once.
1170 ///
1171 /// This is a convenience method to streamline invoking
1172 /// [`Self::ado_new_named_variable`] multiple times.
1173 ///
1174 /// NOTE: Unless required by some particular third-party task, it's strongly
1175 /// recommended to _avoid_ using this method, and to simply use
1176 /// [`ReadVar::from_static`] to get a obtain a static variable.
1177 ///
1178 /// DEVNOTE: In the future, this API may be updated to return a handle that
1179 /// will allow resolving the resulting `AdoRuntimeVar`, but for
1180 /// implementation expediency, this API does not currently do this. If you
1181 /// need to read the value of this variable at runtime, you may need to
1182 /// invoke [`AdoRuntimeVar::dangerous_from_global`] manually.
1183 ///
1184 /// [`NodeCtx::get_ado_variable`]: crate::node::NodeCtx::get_ado_variable
1185 pub fn ado_new_named_variables<K, V>(self, vars: impl IntoIterator<Item = (K, V)>) -> Self
1186 where
1187 K: AsRef<str>,
1188 V: AsRef<str>,
1189 {
1190 self.pipeline.jobs[self.job_idx].ado_variables.extend(
1191 vars.into_iter()
1192 .map(|(k, v)| (k.as_ref().into(), v.as_ref().into())),
1193 );
1194 self
1195 }
1196
1197 /// Overrides the id of the job.
1198 ///
1199 /// Flowey typically generates a reasonable job ID but some use cases that depend
1200 /// on the ID may find it useful to override it to something custom.
1201 pub fn ado_override_job_id(self, name: impl AsRef<str>) -> Self {
1202 self.pipeline
1203 .ado_job_id_overrides
1204 .insert(self.job_idx, name.as_ref().into());
1205 self
1206 }
1207
1208 /// (GitHub Actions only) specify which Github runner this job will be run on.
1209 pub fn gh_set_pool(self, pool: GhRunner) -> Self {
1210 self.pipeline.jobs[self.job_idx].gh_pool = Some(pool);
1211 self
1212 }
1213
1214 /// (GitHub Actions only) Set the concurrency group for this job.
1215 pub fn gh_set_concurrency_group(self, group: GhConcurrencyGroup) -> Self {
1216 self.pipeline.jobs[self.job_idx].gh_concurrency_group = Some(group);
1217 self
1218 }
1219
1220 /// (GitHub Actions only) Manually override the `if:` condition for this
1221 /// particular job.
1222 ///
1223 /// **This is dangerous**, as an improperly set `if` condition may break
1224 /// downstream flowey jobs which assume flowey is in control of the job's
1225 /// scheduling logic.
1226 ///
1227 /// See
1228 /// <https://docs.github.com/en/actions/writing-workflows/workflow-syntax-for-github-actions#jobsjob_idif>
1229 /// for more info.
1230 pub fn gh_dangerous_override_if(self, condition: impl AsRef<str>) -> Self {
1231 self.pipeline.jobs[self.job_idx].gh_override_if = Some(condition.as_ref().into());
1232 self
1233 }
1234
1235 /// (GitHub Actions only) Declare a global job-level environment variable,
1236 /// visible to all downstream steps.
1237 ///
1238 /// `name` and `value` are both arbitrary strings, which may include GitHub
1239 /// Actions template expressions.
1240 ///
1241 /// **This is dangerous**, as it is easy to misuse this API in order to
1242 /// write a node which takes an implicit dependency on there being a global
1243 /// variable set on its behalf by the top-level pipeline code, making it
1244 /// difficult to "locally reason" about the behavior of a node simply by
1245 /// reading its code.
1246 ///
1247 /// Whenever possible, nodes should "late bind" environment variables:
1248 /// accepting a compile-time / runtime flowey parameter, and then setting it
1249 /// prior to executing a child command that requires it.
1250 ///
1251 /// Only use this API in exceptional cases, such as obtaining an environment
1252 /// variable whose value is determined by a job-level GitHub Actions
1253 /// expression evaluation.
1254 pub fn gh_dangerous_global_env_var(
1255 self,
1256 name: impl AsRef<str>,
1257 value: impl AsRef<str>,
1258 ) -> Self {
1259 let name = name.as_ref();
1260 let value = value.as_ref();
1261 self.pipeline.jobs[self.job_idx]
1262 .gh_global_env
1263 .insert(name.into(), value.into());
1264 self
1265 }
1266
1267 /// (GitHub Actions only) Grant permissions required by nodes in the job.
1268 ///
1269 /// For a given node handle, grant the specified permissions.
1270 /// The list provided must match the permissions specified within the node
1271 /// using `requires_permission`.
1272 ///
1273 /// NOTE: While this method is called at a node-level for auditability, the emitted
1274 /// yaml grants permissions at the job-level.
1275 ///
1276 /// This can lead to weird situations where node 1 might not specify a permission
1277 /// required according to Github Actions, but due to job-level granting of the permission
1278 /// by another node 2, the pipeline executes even though it wouldn't if node 2 was removed.
1279 ///
1280 /// For available permission scopes and their descriptions, see
1281 /// <https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#permissions>.
1282 pub fn gh_grant_permissions<N: FlowNodeBase + 'static>(
1283 self,
1284 permissions: impl IntoIterator<Item = (GhPermission, GhPermissionValue)>,
1285 ) -> Self {
1286 let node_handle = NodeHandle::from_type::<N>();
1287 for (permission, value) in permissions {
1288 self.pipeline.jobs[self.job_idx]
1289 .gh_permissions
1290 .entry(node_handle)
1291 .or_default()
1292 .insert(permission, value);
1293 }
1294 self
1295 }
1296
1297 pub fn apply_patchfn(self, patchfn: crate::patch::PatchFn) -> Self {
1298 self.pipeline.jobs[self.job_idx]
1299 .patches
1300 .apply_patchfn(patchfn);
1301 self
1302 }
1303
1304 /// Set a timeout for the job, in minutes.
1305 ///
1306 /// Not calling this will result in the platform's default timeout being used,
1307 /// which is typically 60 minutes, but may vary.
1308 pub fn with_timeout_in_minutes(self, timeout: u32) -> Self {
1309 self.pipeline.jobs[self.job_idx].timeout_minutes = Some(timeout);
1310 self
1311 }
1312
1313 /// (ADO+Local Only) Only run the job if the specified condition is true.
1314 pub fn with_condition(self, cond: UseParameter<bool>) -> Self {
1315 self.pipeline.jobs[self.job_idx].cond_param_idx = Some(cond.idx);
1316 self.pipeline.parameters[cond.idx]
1317 .used_by_jobs
1318 .insert(self.job_idx);
1319 self
1320 }
1321
1322 /// Set a [`CommandWrapperKind`] that will be applied to all shell
1323 /// commands executed in this job's steps.
1324 ///
1325 /// The wrapper is applied both when running locally (via direct run)
1326 /// and when running in CI (the kind is serialized into
1327 /// `pipeline.json` and reconstructed at runtime).
1328 ///
1329 /// [`CommandWrapperKind`]: crate::shell::CommandWrapperKind
1330 pub fn set_command_wrapper(self, wrapper: crate::shell::CommandWrapperKind) -> Self {
1331 self.pipeline.jobs[self.job_idx].command_wrapper = Some(wrapper);
1332 self
1333 }
1334
1335 /// Add a flow node which will be run as part of the job.
1336 pub fn dep_on<R: IntoRequest + 'static>(
1337 self,
1338 f: impl FnOnce(&mut PipelineJobCtx<'_>) -> R,
1339 ) -> Self {
1340 // JobToNodeCtx will ensure artifact deps are taken care of
1341 let req = f(&mut PipelineJobCtx {
1342 pipeline: self.pipeline,
1343 job_idx: self.job_idx,
1344 });
1345
1346 self.pipeline.jobs[self.job_idx]
1347 .root_nodes
1348 .entry(NodeHandle::from_type::<R::Node>())
1349 .or_default()
1350 .push(serde_json::to_vec(&req.into_request()).unwrap().into());
1351
1352 self
1353 }
1354
1355 /// Add a flow node whose request publishes a typed artifact.
1356 ///
1357 /// This is a shortcut for the common pattern of calling
1358 /// [`PipelineJobCtx::publish_typed_artifact`] inside a [`Self::dep_on`]
1359 /// closure and passing the resulting [`WriteVar`] into a request.
1360 pub fn publish<T: Artifact, R: IntoRequest + 'static>(
1361 self,
1362 artifact: PublishTypedArtifact<T>,
1363 f: impl FnOnce(WriteVar<T>) -> R,
1364 ) -> Self {
1365 self.dep_on(|ctx| f(ctx.publish_typed_artifact(artifact)))
1366 }
1367
1368 /// Add a flow node whose request is run purely for its side effect.
1369 ///
1370 /// This is a shortcut for the common pattern of calling
1371 /// [`PipelineJobCtx::new_done_handle`] inside a [`Self::dep_on`]
1372 /// closure and passing the resulting [`WriteVar`] into a request.
1373 pub fn side_effect<R: IntoRequest + 'static>(
1374 self,
1375 f: impl FnOnce(WriteVar<crate::node::SideEffect>) -> R,
1376 ) -> Self {
1377 self.dep_on(|ctx| f(ctx.new_done_handle()))
1378 }
1379
1380 /// Set config on a node for this job.
1381 ///
1382 /// This is the pipeline-level equivalent of [`NodeCtx::config`]. Config
1383 /// set here is merged with any config set by nodes within the job.
1384 ///
1385 /// [`NodeCtx::config`]: crate::node::NodeCtx::config
1386 pub fn config<C: IntoConfig + 'static>(self, config: C) -> Self {
1387 self.pipeline.jobs[self.job_idx]
1388 .root_configs
1389 .entry(NodeHandle::from_type::<C::Node>())
1390 .or_default()
1391 .push(serde_json::to_vec(&config).unwrap().into());
1392
1393 self
1394 }
1395
1396 /// Finish describing the pipeline job.
1397 pub fn finish(self) -> PipelineJobHandle {
1398 PipelineJobHandle {
1399 job_idx: self.job_idx,
1400 }
1401 }
1402
1403 /// Return the job's platform.
1404 pub fn get_platform(&self) -> FlowPlatform {
1405 self.pipeline.jobs[self.job_idx].platform
1406 }
1407
1408 /// Return the job's architecture.
1409 pub fn get_arch(&self) -> FlowArch {
1410 self.pipeline.jobs[self.job_idx].arch
1411 }
1412}
1413
1414#[derive(Clone)]
1415pub struct PipelineJobHandle {
1416 job_idx: usize,
1417}
1418
1419impl PipelineJobHandle {
1420 pub fn is_handle_for(&self, job: &PipelineJob<'_>) -> bool {
1421 self.job_idx == job.job_idx
1422 }
1423}
1424
1425#[derive(Clone, Copy)]
1426pub enum PipelineBackendHint {
1427 /// Pipeline is being run on the user's dev machine (via bash / direct run)
1428 Local,
1429 /// Pipeline is run on ADO
1430 Ado,
1431 /// Pipeline is run on GitHub Actions
1432 Github,
1433}
1434
1435/// Trait for types that can be converted into a [`Pipeline`].
1436///
1437/// This is the primary entry point for defining flowey pipelines. Implement this trait
1438/// to create a pipeline definition that can be executed locally or converted to CI YAML.
1439///
1440/// # Example
1441///
1442/// ```rust,no_run
1443/// use flowey_core::pipeline::{IntoPipeline, Pipeline, PipelineBackendHint};
1444/// use flowey_core::node::{FlowPlatform, FlowPlatformLinuxDistro, FlowArch};
1445///
1446/// struct MyPipeline;
1447///
1448/// impl IntoPipeline for MyPipeline {
1449/// fn into_pipeline(self, backend_hint: PipelineBackendHint) -> anyhow::Result<Pipeline> {
1450/// let mut pipeline = Pipeline::new();
1451///
1452/// // Define a job that runs on Linux x86_64
1453/// let _job = pipeline
1454/// .new_job(
1455/// FlowPlatform::Linux(FlowPlatformLinuxDistro::Ubuntu),
1456/// FlowArch::X86_64,
1457/// "build"
1458/// )
1459/// .finish();
1460///
1461/// Ok(pipeline)
1462/// }
1463/// }
1464/// ```
1465///
1466/// # Complex Example with Parameters and Artifacts
1467///
1468/// ```rust,ignore
1469/// use flowey_core::pipeline::{IntoPipeline, Pipeline, PipelineBackendHint, ParameterKind};
1470/// use flowey_core::node::{FlowPlatform, FlowPlatformLinuxDistro, FlowArch};
1471///
1472/// struct BuildPipeline;
1473///
1474/// impl IntoPipeline for BuildPipeline {
1475/// fn into_pipeline(self, backend_hint: PipelineBackendHint) -> anyhow::Result<Pipeline> {
1476/// let mut pipeline = Pipeline::new();
1477///
1478/// // Define a runtime parameter
1479/// let enable_tests = pipeline.new_parameter_bool(
1480/// "enable_tests",
1481/// "Whether to run tests",
1482/// ParameterKind::Stable,
1483/// Some(true) // default value
1484/// );
1485///
1486/// // Create an artifact for passing data between jobs
1487/// let (publish_build, use_build) = pipeline.new_artifact("build-output");
1488///
1489/// // Job 1: Build
1490/// let build_job = pipeline
1491/// .new_job(
1492/// FlowPlatform::Linux(FlowPlatformLinuxDistro::Ubuntu),
1493/// FlowArch::X86_64,
1494/// "build"
1495/// )
1496/// .with_timeout_in_minutes(30)
1497/// .dep_on(|ctx| flowey_lib_hvlite::_jobs::example_node::Request {
1498/// output_dir: ctx.publish_artifact(publish_build),
1499/// })
1500/// .finish();
1501///
1502/// // Job 2: Test (conditionally run based on parameter)
1503/// let _test_job = pipeline
1504/// .new_job(
1505/// FlowPlatform::Linux(FlowPlatformLinuxDistro::Ubuntu),
1506/// FlowArch::X86_64,
1507/// "test"
1508/// )
1509/// .with_condition(enable_tests)
1510/// .dep_on(|ctx| flowey_lib_hvlite::_jobs::example_node2::Request {
1511/// input_dir: ctx.use_artifact(&use_build),
1512/// })
1513/// .finish();
1514///
1515/// Ok(pipeline)
1516/// }
1517/// }
1518/// ```
1519pub trait IntoPipeline {
1520 fn into_pipeline(self, backend_hint: PipelineBackendHint) -> anyhow::Result<Pipeline>;
1521}
1522
1523fn new_parameter_name(name: impl AsRef<str>, kind: ParameterKind) -> String {
1524 match kind {
1525 ParameterKind::Unstable => format!("__unstable_{}", name.as_ref()),
1526 ParameterKind::Stable => name.as_ref().into(),
1527 }
1528}
1529
1530/// Structs which should only be used by top-level flowey emitters. If you're a
1531/// pipeline author, these are not types you need to care about!
1532pub mod internal {
1533 use super::*;
1534 use std::collections::BTreeMap;
1535
1536 pub fn consistent_artifact_runtime_var_name(artifact: impl AsRef<str>, is_use: bool) -> String {
1537 format!(
1538 "artifact_{}_{}",
1539 if is_use { "use_from" } else { "publish_from" },
1540 artifact.as_ref()
1541 )
1542 }
1543
1544 #[derive(Debug)]
1545 pub struct InternalAdoResourcesRepository {
1546 /// flowey-generated unique repo identifier
1547 pub repo_id: String,
1548 /// Type of repo that is being connected to.
1549 pub repo_type: AdoResourcesRepositoryType,
1550 /// Repository name. Format depends on `repo_type`.
1551 pub name: String,
1552 /// git ref to checkout.
1553 pub git_ref: AdoResourcesRepositoryRef<usize>,
1554 /// (optional) ID of the service endpoint connecting to this repository.
1555 pub endpoint: Option<String>,
1556 }
1557
1558 pub struct PipelineJobMetadata {
1559 pub root_nodes: BTreeMap<NodeHandle, Vec<Box<[u8]>>>,
1560 pub root_configs: BTreeMap<NodeHandle, Vec<Box<[u8]>>>,
1561 pub patches: PatchResolver,
1562 pub label: String,
1563 pub platform: FlowPlatform,
1564 pub arch: FlowArch,
1565 pub cond_param_idx: Option<usize>,
1566 pub timeout_minutes: Option<u32>,
1567 pub command_wrapper: Option<crate::shell::CommandWrapperKind>,
1568 // backend specific
1569 pub ado_pool: Option<AdoPool>,
1570 pub ado_variables: BTreeMap<String, String>,
1571 pub gh_override_if: Option<String>,
1572 pub gh_pool: Option<GhRunner>,
1573 pub gh_concurrency_group: Option<GhConcurrencyGroup>,
1574 pub gh_global_env: BTreeMap<String, String>,
1575 pub gh_permissions: BTreeMap<NodeHandle, BTreeMap<GhPermission, GhPermissionValue>>,
1576 }
1577
1578 #[derive(Debug)]
1579 pub struct ArtifactMeta {
1580 pub name: String,
1581 pub published_by_job: Option<usize>,
1582 pub used_by_jobs: BTreeSet<usize>,
1583 }
1584
1585 #[derive(Debug)]
1586 pub struct ParameterMeta {
1587 pub parameter: Parameter,
1588 pub used_by_jobs: BTreeSet<usize>,
1589 }
1590
1591 /// Mirror of [`Pipeline`], except with all field marked as `pub`.
1592 pub struct PipelineFinalized {
1593 pub jobs: Vec<PipelineJobMetadata>,
1594 pub artifacts: Vec<ArtifactMeta>,
1595 pub parameters: Vec<ParameterMeta>,
1596 pub extra_deps: BTreeSet<(usize, usize)>,
1597 // backend specific
1598 pub ado_name: Option<String>,
1599 pub ado_schedule_triggers: Vec<AdoScheduleTriggers>,
1600 pub ado_ci_triggers: Option<AdoCiTriggers>,
1601 pub ado_pr_triggers: Option<AdoPrTriggers>,
1602 pub ado_bootstrap_template: String,
1603 pub ado_resources_repository: Vec<InternalAdoResourcesRepository>,
1604 pub ado_post_process_yaml_cb:
1605 Option<Box<dyn FnOnce(serde_yaml::Value) -> serde_yaml::Value>>,
1606 pub ado_variables: BTreeMap<String, String>,
1607 pub ado_job_id_overrides: BTreeMap<usize, String>,
1608 pub gh_name: Option<String>,
1609 pub gh_schedule_triggers: Vec<GhScheduleTriggers>,
1610 pub gh_ci_triggers: Option<GhCiTriggers>,
1611 pub gh_pr_triggers: Option<GhPrTriggers>,
1612 pub gh_bootstrap_template: String,
1613 }
1614
1615 impl PipelineFinalized {
1616 pub fn from_pipeline(mut pipeline: Pipeline) -> Self {
1617 if let Some(cb) = pipeline.inject_all_jobs_with.take() {
1618 for job_idx in 0..pipeline.jobs.len() {
1619 let _ = cb(PipelineJob {
1620 pipeline: &mut pipeline,
1621 job_idx,
1622 });
1623 }
1624 }
1625
1626 let Pipeline {
1627 mut jobs,
1628 artifacts,
1629 parameters,
1630 extra_deps,
1631 ado_name,
1632 ado_bootstrap_template,
1633 ado_schedule_triggers,
1634 ado_ci_triggers,
1635 ado_pr_triggers,
1636 ado_resources_repository,
1637 ado_post_process_yaml_cb,
1638 ado_variables,
1639 ado_job_id_overrides,
1640 gh_name,
1641 gh_schedule_triggers,
1642 gh_ci_triggers,
1643 gh_pr_triggers,
1644 gh_bootstrap_template,
1645 // not relevant to consumer code
1646 dummy_done_idx: _,
1647 artifact_map_idx: _,
1648 artifact_names: _,
1649 global_patchfns,
1650 inject_all_jobs_with: _, // processed above
1651 } = pipeline;
1652
1653 for patchfn in global_patchfns {
1654 for job in &mut jobs {
1655 job.patches.apply_patchfn(patchfn)
1656 }
1657 }
1658
1659 Self {
1660 jobs,
1661 artifacts,
1662 parameters,
1663 extra_deps,
1664 ado_name,
1665 ado_schedule_triggers,
1666 ado_ci_triggers,
1667 ado_pr_triggers,
1668 ado_bootstrap_template,
1669 ado_resources_repository,
1670 ado_post_process_yaml_cb,
1671 ado_variables,
1672 ado_job_id_overrides,
1673 gh_name,
1674 gh_schedule_triggers,
1675 gh_ci_triggers,
1676 gh_pr_triggers,
1677 gh_bootstrap_template,
1678 }
1679 }
1680 }
1681
1682 #[derive(Debug, Clone)]
1683 pub enum Parameter {
1684 Bool {
1685 name: String,
1686 description: String,
1687 kind: ParameterKind,
1688 default: Option<bool>,
1689 },
1690 String {
1691 name: String,
1692 description: String,
1693 default: Option<String>,
1694 kind: ParameterKind,
1695 possible_values: Option<Vec<String>>,
1696 },
1697 Num {
1698 name: String,
1699 description: String,
1700 default: Option<i64>,
1701 kind: ParameterKind,
1702 possible_values: Option<Vec<i64>>,
1703 },
1704 }
1705
1706 impl Parameter {
1707 pub fn name(&self) -> &str {
1708 match self {
1709 Parameter::Bool { name, .. } => name,
1710 Parameter::String { name, .. } => name,
1711 Parameter::Num { name, .. } => name,
1712 }
1713 }
1714 }
1715}