flowey_core/pipeline.rs
1// Copyright (c) Microsoft Corporation.
2// Licensed under the MIT License.
3
4//! Core types and traits used to create and work with flowey pipelines.
5
6mod artifact;
7
8pub use artifact::Artifact;
9pub use artifact::ArtifactType;
10
11use self::internal::*;
12use crate::node::FlowArch;
13use crate::node::FlowNodeBase;
14use crate::node::FlowPlatform;
15use crate::node::FlowPlatformLinuxDistro;
16use crate::node::GhUserSecretVar;
17use crate::node::IntoConfig;
18use crate::node::IntoRequest;
19use crate::node::NodeHandle;
20use crate::node::ReadVar;
21use crate::node::WriteVar;
22use crate::node::steps::ado::AdoResourcesRepositoryId;
23use crate::node::user_facing::AdoRuntimeVar;
24use crate::node::user_facing::GhPermission;
25use crate::node::user_facing::GhPermissionValue;
26use crate::patch::PatchResolver;
27use crate::patch::ResolvedPatches;
28use serde::Serialize;
29use serde::de::DeserializeOwned;
30use std::collections::BTreeMap;
31use std::collections::BTreeSet;
32use std::path::PathBuf;
33
34/// Pipeline types which are considered "user facing", and included in the
35/// `flowey` prelude.
36pub mod user_facing {
37 pub use super::AdoCiTriggers;
38 pub use super::AdoPool;
39 pub use super::AdoPrTriggers;
40 pub use super::AdoResourcesRepository;
41 pub use super::AdoResourcesRepositoryRef;
42 pub use super::AdoResourcesRepositoryType;
43 pub use super::AdoScheduleTriggers;
44 pub use super::GhCiTriggers;
45 pub use super::GhPrTriggers;
46 pub use super::GhRunner;
47 pub use super::GhRunnerOsLabel;
48 pub use super::GhScheduleTriggers;
49 pub use super::HostExt;
50 pub use super::IntoPipeline;
51 pub use super::ParameterKind;
52 pub use super::Pipeline;
53 pub use super::PipelineBackendHint;
54 pub use super::PipelineJob;
55 pub use super::PipelineJobCtx;
56 pub use super::PipelineJobHandle;
57 pub use super::PublishArtifact;
58 pub use super::PublishTypedArtifact;
59 pub use super::UseArtifact;
60 pub use super::UseParameter;
61 pub use super::UseTypedArtifact;
62 pub use crate::node::FlowArch;
63 pub use crate::node::FlowPlatform;
64}
65
66fn linux_distro() -> FlowPlatformLinuxDistro {
67 // Check for nix environment first - takes precedence over distro detection
68 if std::env::var("IN_NIX_SHELL").is_ok() {
69 return FlowPlatformLinuxDistro::Nix;
70 }
71
72 // A `nix develop` shell doesn't set `IN_NIX_SHELL`, but the PATH should include a nix store path
73 if std::env::var("PATH").is_ok_and(|path| path.contains("/nix/store")) {
74 return FlowPlatformLinuxDistro::Nix;
75 }
76
77 if let Ok(etc_os_release) = fs_err::read_to_string("/etc/os-release") {
78 if etc_os_release.contains("ID=ubuntu") {
79 FlowPlatformLinuxDistro::Ubuntu
80 } else if etc_os_release.contains("ID=fedora") {
81 FlowPlatformLinuxDistro::Fedora
82 } else if etc_os_release.contains("ID=azurelinux") || etc_os_release.contains("ID=mariner")
83 {
84 FlowPlatformLinuxDistro::AzureLinux
85 } else if etc_os_release.contains("ID=arch") {
86 FlowPlatformLinuxDistro::Arch
87 } else {
88 FlowPlatformLinuxDistro::Unknown
89 }
90 } else {
91 FlowPlatformLinuxDistro::Unknown
92 }
93}
94
95pub trait HostExt: Sized {
96 /// Return the value for the current host machine.
97 ///
98 /// Will panic on non-local backends.
99 fn host(backend_hint: PipelineBackendHint) -> Self;
100}
101
102impl HostExt for FlowPlatform {
103 /// Return the platform of the current host machine.
104 ///
105 /// Will panic on non-local backends.
106 fn host(backend_hint: PipelineBackendHint) -> Self {
107 if !matches!(backend_hint, PipelineBackendHint::Local) {
108 panic!("can only use `FlowPlatform::host` when defining a local-only pipeline");
109 }
110
111 if cfg!(target_os = "windows") {
112 Self::Windows
113 } else if cfg!(target_os = "linux") {
114 Self::Linux(linux_distro())
115 } else if cfg!(target_os = "macos") {
116 Self::MacOs
117 } else {
118 panic!("no valid host-os")
119 }
120 }
121}
122
123impl HostExt for FlowArch {
124 /// Return the arch of the current host machine.
125 ///
126 /// Will panic on non-local backends.
127 fn host(backend_hint: PipelineBackendHint) -> Self {
128 if !matches!(backend_hint, PipelineBackendHint::Local) {
129 panic!("can only use `FlowArch::host` when defining a local-only pipeline");
130 }
131
132 // xtask-fmt allow-target-arch oneoff-flowey
133 if cfg!(target_arch = "x86_64") {
134 Self::X86_64
135 // xtask-fmt allow-target-arch oneoff-flowey
136 } else if cfg!(target_arch = "aarch64") {
137 Self::Aarch64
138 } else {
139 panic!("no valid host-arch")
140 }
141 }
142}
143
144/// Trigger ADO pipelines via Continuous Integration
145#[derive(Default, Debug)]
146pub struct AdoScheduleTriggers {
147 /// Friendly name for the scheduled run
148 pub display_name: String,
149 /// Run the pipeline whenever there is a commit on these specified branches
150 /// (supports glob syntax)
151 pub branches: Vec<String>,
152 /// Specify any branches which should be filtered out from the list of
153 /// `branches` (supports glob syntax)
154 pub exclude_branches: Vec<String>,
155 /// Run the pipeline in a schedule, as specified by a cron string
156 pub cron: String,
157}
158
159/// Trigger ADO pipelines per PR
160#[derive(Debug)]
161pub struct AdoPrTriggers {
162 /// Run the pipeline whenever there is a PR to these specified branches
163 /// (supports glob syntax)
164 pub branches: Vec<String>,
165 /// Specify any branches which should be filtered out from the list of
166 /// `branches` (supports glob syntax)
167 pub exclude_branches: Vec<String>,
168 /// Run the pipeline even if the PR is a draft PR. Defaults to `false`.
169 pub run_on_draft: bool,
170 /// Automatically cancel the pipeline run if a new commit lands in the
171 /// branch. Defaults to `true`.
172 pub auto_cancel: bool,
173 /// Only run the pipeline when files matching these paths are changed
174 /// (supports glob syntax)
175 pub paths: Vec<String>,
176 /// Specify any paths which should be filtered out from the list of
177 /// `paths` (supports glob syntax)
178 pub exclude_paths: Vec<String>,
179}
180
181/// Trigger ADO pipelines per CI
182#[derive(Debug, Default)]
183pub struct AdoCiTriggers {
184 /// Run the pipeline whenever there is a change to these specified branches
185 /// (supports glob syntax)
186 pub branches: Vec<String>,
187 /// Specify any branches which should be filtered out from the list of
188 /// `branches` (supports glob syntax)
189 pub exclude_branches: Vec<String>,
190 /// Run the pipeline whenever a matching tag is created (supports glob
191 /// syntax)
192 pub tags: Vec<String>,
193 /// Specify any tags which should be filtered out from the list of `tags`
194 /// (supports glob syntax)
195 pub exclude_tags: Vec<String>,
196 /// Whether to batch changes per branch.
197 pub batch: bool,
198 /// Only run the pipeline when files matching these paths are changed
199 /// (supports glob syntax)
200 pub paths: Vec<String>,
201 /// Specify any paths which should be filtered out from the list of
202 /// `paths` (supports glob syntax)
203 pub exclude_paths: Vec<String>,
204}
205
206impl Default for AdoPrTriggers {
207 fn default() -> Self {
208 Self {
209 branches: Vec::new(),
210 exclude_branches: Vec::new(),
211 run_on_draft: false,
212 auto_cancel: true,
213 paths: Vec::new(),
214 exclude_paths: Vec::new(),
215 }
216 }
217}
218
219/// ADO repository resource.
220#[derive(Debug)]
221pub struct AdoResourcesRepository {
222 /// Type of repo that is being connected to.
223 pub repo_type: AdoResourcesRepositoryType,
224 /// Repository name. Format depends on `repo_type`.
225 pub name: String,
226 /// git ref to checkout.
227 pub git_ref: AdoResourcesRepositoryRef,
228 /// (optional) ID of the service endpoint connecting to this repository.
229 pub endpoint: Option<String>,
230}
231
232/// ADO repository resource type
233#[derive(Debug)]
234pub enum AdoResourcesRepositoryType {
235 /// Azure Repos Git repository
236 AzureReposGit,
237 /// Github repository
238 GitHub,
239}
240
241/// ADO repository ref
242#[derive(Debug)]
243pub enum AdoResourcesRepositoryRef<P = UseParameter<String>> {
244 /// Hard-coded ref (e.g: refs/heads/main)
245 Fixed(String),
246 /// Connected to pipeline-level parameter
247 Parameter(P),
248}
249
250/// Trigger Github Actions pipelines via Continuous Integration
251///
252/// NOTE: Github Actions doesn't support specifying the branch when triggered by `schedule`.
253/// To run on a specific branch, modify the branch checked out in the pipeline.
254#[derive(Default, Debug)]
255pub struct GhScheduleTriggers {
256 /// Run the pipeline in a schedule, as specified by a cron string
257 pub cron: String,
258}
259
260/// Trigger Github Actions pipelines per PR
261#[derive(Debug)]
262pub struct GhPrTriggers {
263 /// Run the pipeline whenever there is a PR to these specified branches
264 /// (supports glob syntax)
265 pub branches: Vec<String>,
266 /// Specify any branches which should be filtered out from the list of
267 /// `branches` (supports glob syntax)
268 pub exclude_branches: Vec<String>,
269 /// Automatically cancel the pipeline run if a new commit lands in the
270 /// branch. Defaults to `true`.
271 pub auto_cancel: bool,
272 /// Run the pipeline whenever the PR trigger matches the specified types
273 pub types: Vec<String>,
274 /// Only run the pipeline when files matching these paths are changed
275 /// (supports glob syntax)
276 pub paths: Vec<String>,
277 /// Specify any paths which should be filtered out from the list of
278 /// `paths` (supports glob syntax)
279 pub paths_ignore: Vec<String>,
280}
281
282/// Trigger Github Actions pipelines per PR
283#[derive(Debug, Default)]
284pub struct GhCiTriggers {
285 /// Run the pipeline whenever there is a change to these specified branches
286 /// (supports glob syntax)
287 pub branches: Vec<String>,
288 /// Specify any branches which should be filtered out from the list of
289 /// `branches` (supports glob syntax)
290 pub exclude_branches: Vec<String>,
291 /// Run the pipeline whenever a matching tag is created (supports glob
292 /// syntax)
293 pub tags: Vec<String>,
294 /// Specify any tags which should be filtered out from the list of `tags`
295 /// (supports glob syntax)
296 pub exclude_tags: Vec<String>,
297 /// Only run the pipeline when files matching these paths are changed
298 /// (supports glob syntax)
299 pub paths: Vec<String>,
300 /// Specify any paths which should be filtered out from the list of
301 /// `paths` (supports glob syntax)
302 pub paths_ignore: Vec<String>,
303}
304
305impl GhPrTriggers {
306 /// Triggers the pipeline on the default PR events plus when a draft is marked as ready for review.
307 pub fn new_draftable() -> Self {
308 Self {
309 branches: Vec::new(),
310 exclude_branches: Vec::new(),
311 types: vec![
312 "opened".into(),
313 "synchronize".into(),
314 "reopened".into(),
315 "ready_for_review".into(),
316 ],
317 auto_cancel: true,
318 paths: Vec::new(),
319 paths_ignore: Vec::new(),
320 }
321 }
322}
323
324#[derive(Debug, Clone, PartialEq)]
325pub enum GhRunnerOsLabel {
326 UbuntuLatest,
327 Ubuntu2404,
328 Ubuntu2204,
329 WindowsLatest,
330 Windows2025,
331 Windows2022,
332 Ubuntu2404Arm,
333 Ubuntu2204Arm,
334 Windows11Arm,
335 Custom(String),
336}
337
338/// GitHub runner type
339#[derive(Debug, Clone, PartialEq)]
340pub enum GhRunner {
341 // See <https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#choosing-github-hosted-runners>
342 // for more details.
343 GhHosted(GhRunnerOsLabel),
344 // Self hosted runners are selected by matching runner labels to <labels>.
345 // 'self-hosted' is a common label for self hosted runners, but is not required.
346 // Labels are case-insensitive and can take the form of arbitrary strings.
347 // See <https://docs.github.com/en/actions/hosting-your-own-runners> for more details.
348 SelfHosted(Vec<String>),
349 // This uses a runner belonging to <group> that matches all labels in <labels>.
350 // See <https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#choosing-github-hosted-runners>
351 // for more details.
352 RunnerGroup { group: String, labels: Vec<String> },
353}
354
355impl GhRunner {
356 /// Whether this is a self-hosted runner with the provided label
357 pub fn is_self_hosted_with_label(&self, label: &str) -> bool {
358 matches!(self, GhRunner::SelfHosted(labels) if labels.iter().any(|s| s.as_str() == label))
359 }
360}
361
362// TODO: support a more structured format for demands
363// See https://learn.microsoft.com/en-us/azure/devops/pipelines/yaml-schema/pool-demands
364#[derive(Debug, Clone)]
365pub struct AdoPool {
366 pub name: String,
367 pub demands: Vec<String>,
368}
369
370/// Parameter type (unstable / stable).
371#[derive(Debug, Clone)]
372pub enum ParameterKind {
373 // The parameter is considered an unstable API, and should not be
374 // taken as a dependency.
375 Unstable,
376 // The parameter is considered a stable API, and can be used by
377 // external pipelines to control behavior of the pipeline.
378 Stable,
379}
380
381#[derive(Clone, Debug)]
382#[must_use]
383pub struct UseParameter<T> {
384 idx: usize,
385 _kind: std::marker::PhantomData<T>,
386}
387
388/// Opaque handle to an artifact which must be published by a single job.
389#[must_use]
390pub struct PublishArtifact {
391 idx: usize,
392}
393
394/// Opaque handle to an artifact which can be used by one or more jobs.
395#[derive(Clone)]
396#[must_use]
397pub struct UseArtifact {
398 idx: usize,
399}
400
401/// Opaque handle to an artifact of type `T` which must be published by a single job.
402#[must_use]
403pub struct PublishTypedArtifact<T>(PublishArtifact, std::marker::PhantomData<fn() -> T>);
404
405/// Opaque handle to an artifact of type `T` which can be used by one or more
406/// jobs.
407#[must_use]
408pub struct UseTypedArtifact<T>(UseArtifact, std::marker::PhantomData<fn(T)>);
409
410impl<T> Clone for UseTypedArtifact<T> {
411 fn clone(&self) -> Self {
412 UseTypedArtifact(self.0.clone(), std::marker::PhantomData)
413 }
414}
415
416#[derive(Default)]
417pub struct Pipeline {
418 jobs: Vec<PipelineJobMetadata>,
419 artifacts: Vec<ArtifactMeta>,
420 parameters: Vec<ParameterMeta>,
421 extra_deps: BTreeSet<(usize, usize)>,
422 // builder internal
423 artifact_names: BTreeSet<String>,
424 dummy_done_idx: usize,
425 artifact_map_idx: usize,
426 global_patchfns: Vec<crate::patch::PatchFn>,
427 inject_all_jobs_with: Option<Box<dyn for<'a> Fn(PipelineJob<'a>) -> PipelineJob<'a>>>,
428 // backend specific
429 ado_name: Option<String>,
430 ado_job_id_overrides: BTreeMap<usize, String>,
431 ado_schedule_triggers: Vec<AdoScheduleTriggers>,
432 ado_ci_triggers: Option<AdoCiTriggers>,
433 ado_pr_triggers: Option<AdoPrTriggers>,
434 ado_resources_repository: Vec<InternalAdoResourcesRepository>,
435 ado_bootstrap_template: String,
436 ado_variables: BTreeMap<String, String>,
437 ado_post_process_yaml_cb: Option<Box<dyn FnOnce(serde_yaml::Value) -> serde_yaml::Value>>,
438 gh_name: Option<String>,
439 gh_schedule_triggers: Vec<GhScheduleTriggers>,
440 gh_ci_triggers: Option<GhCiTriggers>,
441 gh_pr_triggers: Option<GhPrTriggers>,
442 gh_bootstrap_template: String,
443}
444
445impl Pipeline {
446 pub fn new() -> Pipeline {
447 Pipeline::default()
448 }
449
450 /// Inject all pipeline jobs with some common logic. (e.g: to resolve common
451 /// configuration requirements shared by all jobs).
452 ///
453 /// Can only be invoked once per pipeline.
454 #[track_caller]
455 pub fn inject_all_jobs_with(
456 &mut self,
457 cb: impl for<'a> Fn(PipelineJob<'a>) -> PipelineJob<'a> + 'static,
458 ) -> &mut Self {
459 if self.inject_all_jobs_with.is_some() {
460 panic!("can only call inject_all_jobs_with once!")
461 }
462 self.inject_all_jobs_with = Some(Box::new(cb));
463 self
464 }
465
466 /// (ADO only) Provide a YAML template used to bootstrap flowey at the start
467 /// of an ADO pipeline.
468 ///
469 /// The template has access to the following vars, which will be statically
470 /// interpolated into the template's text:
471 ///
472 /// - `{{FLOWEY_OUTDIR}}`
473 /// - Directory to copy artifacts into.
474 /// - NOTE: this var will include `\` on Windows, and `/` on linux!
475 /// - `{{FLOWEY_BIN_EXTENSION}}`
476 /// - Extension of the expected flowey bin (either "", or ".exe")
477 /// - `{{FLOWEY_CRATE}}`
478 /// - Name of the project-specific flowey crate to be built
479 /// - `{{FLOWEY_TARGET}}`
480 /// - The target-triple flowey is being built for
481 /// - `{{FLOWEY_PIPELINE_PATH}}`
482 /// - Repo-root relative path to the pipeline (as provided when
483 /// generating the pipeline via the flowey CLI)
484 ///
485 /// The template's sole responsibility is to copy 3 files into the
486 /// `{{FLOWEY_OUTDIR}}`:
487 ///
488 /// 1. The bootstrapped flowey binary, with the file name
489 /// `flowey{{FLOWEY_BIN_EXTENSION}}`
490 /// 2. Two files called `pipeline.yaml` and `pipeline.json`, which are
491 /// copied of the pipeline YAML and pipeline JSON currently being run.
492 /// `{{FLOWEY_PIPELINE_PATH}}` is provided as a way to disambiguate in
493 /// cases where the same template is being for multiple pipelines (e.g: a
494 /// debug vs. release pipeline).
495 pub fn ado_set_flowey_bootstrap_template(&mut self, template: String) -> &mut Self {
496 self.ado_bootstrap_template = template;
497 self
498 }
499
500 /// (ADO only) Provide a callback function which will be used to
501 /// post-process any YAML flowey generates for the pipeline.
502 ///
503 /// Until flowey defines a stable API for maintaining out-of-tree backends,
504 /// this method can be used to integrate the output from the generic ADO
505 /// backend with any organization-specific templates that one may be
506 /// required to use (e.g: for compliance reasons).
507 pub fn ado_post_process_yaml(
508 &mut self,
509 cb: impl FnOnce(serde_yaml::Value) -> serde_yaml::Value + 'static,
510 ) -> &mut Self {
511 self.ado_post_process_yaml_cb = Some(Box::new(cb));
512 self
513 }
514
515 /// (ADO only) Add a new scheduled CI trigger. Can be called multiple times
516 /// to set up multiple schedules runs.
517 pub fn ado_add_schedule_trigger(&mut self, triggers: AdoScheduleTriggers) -> &mut Self {
518 self.ado_schedule_triggers.push(triggers);
519 self
520 }
521
522 /// (ADO only) Set a PR trigger. Calling this method multiple times will
523 /// overwrite any previously set triggers.
524 pub fn ado_set_pr_triggers(&mut self, triggers: AdoPrTriggers) -> &mut Self {
525 self.ado_pr_triggers = Some(triggers);
526 self
527 }
528
529 /// (ADO only) Set a CI trigger. Calling this method multiple times will
530 /// overwrite any previously set triggers.
531 pub fn ado_set_ci_triggers(&mut self, triggers: AdoCiTriggers) -> &mut Self {
532 self.ado_ci_triggers = Some(triggers);
533 self
534 }
535
536 /// (ADO only) Declare a new repository resource, returning a type-safe
537 /// handle which downstream ADO steps are able to consume via
538 /// [`AdoStepServices::resolve_repository_id`](crate::node::user_facing::AdoStepServices::resolve_repository_id).
539 pub fn ado_add_resources_repository(
540 &mut self,
541 repo: AdoResourcesRepository,
542 ) -> AdoResourcesRepositoryId {
543 let AdoResourcesRepository {
544 repo_type,
545 name,
546 git_ref,
547 endpoint,
548 } = repo;
549
550 let repo_id = format!("repo{}", self.ado_resources_repository.len());
551
552 self.ado_resources_repository
553 .push(InternalAdoResourcesRepository {
554 repo_id: repo_id.clone(),
555 repo_type,
556 name,
557 git_ref: match git_ref {
558 AdoResourcesRepositoryRef::Fixed(s) => AdoResourcesRepositoryRef::Fixed(s),
559 AdoResourcesRepositoryRef::Parameter(p) => {
560 AdoResourcesRepositoryRef::Parameter(p.idx)
561 }
562 },
563 endpoint,
564 });
565 AdoResourcesRepositoryId { repo_id }
566 }
567
568 /// (GitHub Actions only) Set the pipeline-level name.
569 ///
570 /// <https://docs.github.com/en/actions/writing-workflows/workflow-syntax-for-github-actions#name>
571 pub fn gh_set_name(&mut self, name: impl AsRef<str>) -> &mut Self {
572 self.gh_name = Some(name.as_ref().into());
573 self
574 }
575
576 /// Provide a YAML template used to bootstrap flowey at the start of an GitHub
577 /// pipeline.
578 ///
579 /// The template has access to the following vars, which will be statically
580 /// interpolated into the template's text:
581 ///
582 /// - `{{FLOWEY_OUTDIR}}`
583 /// - Directory to copy artifacts into.
584 /// - NOTE: this var will include `\` on Windows, and `/` on linux!
585 /// - `{{FLOWEY_BIN_EXTENSION}}`
586 /// - Extension of the expected flowey bin (either "", or ".exe")
587 /// - `{{FLOWEY_CRATE}}`
588 /// - Name of the project-specific flowey crate to be built
589 /// - `{{FLOWEY_TARGET}}`
590 /// - The target-triple flowey is being built for
591 /// - `{{FLOWEY_PIPELINE_PATH}}`
592 /// - Repo-root relative path to the pipeline (as provided when
593 /// generating the pipeline via the flowey CLI)
594 ///
595 /// The template's sole responsibility is to copy 3 files into the
596 /// `{{FLOWEY_OUTDIR}}`:
597 ///
598 /// 1. The bootstrapped flowey binary, with the file name
599 /// `flowey{{FLOWEY_BIN_EXTENSION}}`
600 /// 2. Two files called `pipeline.yaml` and `pipeline.json`, which are
601 /// copied of the pipeline YAML and pipeline JSON currently being run.
602 /// `{{FLOWEY_PIPELINE_PATH}}` is provided as a way to disambiguate in
603 /// cases where the same template is being for multiple pipelines (e.g: a
604 /// debug vs. release pipeline).
605 pub fn gh_set_flowey_bootstrap_template(&mut self, template: String) -> &mut Self {
606 self.gh_bootstrap_template = template;
607 self
608 }
609
610 /// (GitHub Actions only) Add a new scheduled CI trigger. Can be called multiple times
611 /// to set up multiple schedules runs.
612 pub fn gh_add_schedule_trigger(&mut self, triggers: GhScheduleTriggers) -> &mut Self {
613 self.gh_schedule_triggers.push(triggers);
614 self
615 }
616
617 /// (GitHub Actions only) Set a PR trigger. Calling this method multiple times will
618 /// overwrite any previously set triggers.
619 pub fn gh_set_pr_triggers(&mut self, triggers: GhPrTriggers) -> &mut Self {
620 self.gh_pr_triggers = Some(triggers);
621 self
622 }
623
624 /// (GitHub Actions only) Set a CI trigger. Calling this method multiple times will
625 /// overwrite any previously set triggers.
626 pub fn gh_set_ci_triggers(&mut self, triggers: GhCiTriggers) -> &mut Self {
627 self.gh_ci_triggers = Some(triggers);
628 self
629 }
630
631 /// (GitHub Actions only) Use a pre-defined GitHub Actions secret variable.
632 ///
633 /// For more information on defining secrets for use in GitHub Actions, see
634 /// <https://docs.github.com/en/actions/security-guides/using-secrets-in-github-actions>
635 pub fn gh_use_secret(&mut self, secret_name: impl AsRef<str>) -> GhUserSecretVar {
636 GhUserSecretVar(secret_name.as_ref().to_string())
637 }
638
639 pub fn new_job(
640 &mut self,
641 platform: FlowPlatform,
642 arch: FlowArch,
643 label: impl AsRef<str>,
644 ) -> PipelineJob<'_> {
645 let idx = self.jobs.len();
646 self.jobs.push(PipelineJobMetadata {
647 root_nodes: BTreeMap::new(),
648 root_configs: BTreeMap::new(),
649 patches: ResolvedPatches::build(),
650 label: label.as_ref().into(),
651 platform,
652 arch,
653 cond_param_idx: None,
654 timeout_minutes: None,
655 command_wrapper: None,
656 ado_pool: None,
657 ado_variables: BTreeMap::new(),
658 gh_override_if: None,
659 gh_global_env: BTreeMap::new(),
660 gh_pool: None,
661 gh_permissions: BTreeMap::new(),
662 });
663
664 PipelineJob {
665 pipeline: self,
666 job_idx: idx,
667 }
668 }
669
670 /// Declare a dependency between two jobs that does is not a result of an
671 /// artifact.
672 pub fn non_artifact_dep(
673 &mut self,
674 job: &PipelineJobHandle,
675 depends_on_job: &PipelineJobHandle,
676 ) -> &mut Self {
677 self.extra_deps
678 .insert((depends_on_job.job_idx, job.job_idx));
679 self
680 }
681
682 #[track_caller]
683 pub fn new_artifact(&mut self, name: impl AsRef<str>) -> (PublishArtifact, UseArtifact) {
684 let name = name.as_ref();
685 let owned_name = name.to_string();
686
687 let not_exists = self.artifact_names.insert(owned_name.clone());
688 if !not_exists {
689 panic!("duplicate artifact name: {}", name)
690 }
691
692 let idx = self.artifacts.len();
693 self.artifacts.push(ArtifactMeta {
694 name: owned_name,
695 published_by_job: None,
696 used_by_jobs: BTreeSet::new(),
697 });
698
699 (PublishArtifact { idx }, UseArtifact { idx })
700 }
701
702 /// Returns a pair of opaque handles to a new artifact for use across jobs
703 /// in the pipeline.
704 #[track_caller]
705 pub fn new_typed_artifact<T: Artifact>(
706 &mut self,
707 name: impl AsRef<str>,
708 ) -> (PublishTypedArtifact<T>, UseTypedArtifact<T>) {
709 let (publish, use_artifact) = self.new_artifact(name);
710 (
711 PublishTypedArtifact(publish, std::marker::PhantomData),
712 UseTypedArtifact(use_artifact, std::marker::PhantomData),
713 )
714 }
715
716 /// Returns a pair of sets of opaque handles to a new artifact for use
717 /// across jobs in the pipeline. The artifact names are derived by the impl
718 /// of [`ArtifactType::name`] using common prefixes and suffixes if
719 /// specified (although the implementor can choose to use those values
720 /// differently).
721 #[track_caller]
722 pub fn new_typed_artifact_collection<T: Artifact, U: ArtifactType>(
723 &mut self,
724 artifact_types: impl IntoIterator<Item = U>,
725 prefix: Option<&str>,
726 suffix: Option<&str>,
727 ) -> (
728 BTreeMap<U, PublishTypedArtifact<T>>,
729 BTreeMap<U, UseTypedArtifact<T>>,
730 ) {
731 artifact_types
732 .into_iter()
733 .map(|artifact_type| {
734 let (pub_artifact, use_artifact) =
735 self.new_typed_artifact(artifact_type.name(prefix, suffix));
736 (
737 (artifact_type.clone(), pub_artifact),
738 (artifact_type, use_artifact),
739 )
740 })
741 .unzip()
742 }
743
744 /// (ADO only) Set the pipeline-level name.
745 ///
746 /// <https://learn.microsoft.com/en-us/azure/devops/pipelines/process/run-number?view=azure-devops&tabs=yaml>
747 pub fn ado_add_name(&mut self, name: String) -> &mut Self {
748 self.ado_name = Some(name);
749 self
750 }
751
752 /// (ADO only) Declare a pipeline-level, named, read-only ADO variable.
753 ///
754 /// `name` and `value` are both arbitrary strings.
755 ///
756 /// Returns an instance of [`AdoRuntimeVar`], which, if need be, can be
757 /// converted into a [`ReadVar<String>`] using
758 /// [`NodeCtx::get_ado_variable`].
759 ///
760 /// NOTE: Unless required by some particular third-party task, it's strongly
761 /// recommended to _avoid_ using this method, and to simply use
762 /// [`ReadVar::from_static`] to get a obtain a static variable.
763 ///
764 /// [`NodeCtx::get_ado_variable`]: crate::node::NodeCtx::get_ado_variable
765 pub fn ado_new_named_variable(
766 &mut self,
767 name: impl AsRef<str>,
768 value: impl AsRef<str>,
769 ) -> AdoRuntimeVar {
770 let name = name.as_ref();
771 let value = value.as_ref();
772
773 self.ado_variables.insert(name.into(), value.into());
774
775 // safe, since we'll ensure that the global exists in the ADO backend
776 AdoRuntimeVar::dangerous_from_global(name, false)
777 }
778
779 /// (ADO only) Declare multiple pipeline-level, named, read-only ADO
780 /// variables at once.
781 ///
782 /// This is a convenience method to streamline invoking
783 /// [`Self::ado_new_named_variable`] multiple times.
784 ///
785 /// NOTE: Unless required by some particular third-party task, it's strongly
786 /// recommended to _avoid_ using this method, and to simply use
787 /// [`ReadVar::from_static`] to get a obtain a static variable.
788 ///
789 /// DEVNOTE: In the future, this API may be updated to return a handle that
790 /// will allow resolving the resulting `AdoRuntimeVar`, but for
791 /// implementation expediency, this API does not currently do this. If you
792 /// need to read the value of this variable at runtime, you may need to
793 /// invoke [`AdoRuntimeVar::dangerous_from_global`] manually.
794 ///
795 /// [`NodeCtx::get_ado_variable`]: crate::node::NodeCtx::get_ado_variable
796 pub fn ado_new_named_variables<K, V>(
797 &mut self,
798 vars: impl IntoIterator<Item = (K, V)>,
799 ) -> &mut Self
800 where
801 K: AsRef<str>,
802 V: AsRef<str>,
803 {
804 self.ado_variables.extend(
805 vars.into_iter()
806 .map(|(k, v)| (k.as_ref().into(), v.as_ref().into())),
807 );
808 self
809 }
810
811 /// Declare a pipeline-level runtime parameter with type `bool`.
812 ///
813 /// To obtain a [`ReadVar<bool>`] that can be used within a node, use the
814 /// [`PipelineJobCtx::use_parameter`] method.
815 ///
816 /// `name` is the name of the parameter.
817 ///
818 /// `description` is an arbitrary string, which will be be shown to users.
819 ///
820 /// `kind` is the type of parameter and if it should be treated as a stable
821 /// external API to callers of the pipeline.
822 ///
823 /// `default` is the default value for the parameter. If none is provided,
824 /// the parameter _must_ be specified in order for the pipeline to run.
825 ///
826 /// `possible_values` can be used to limit the set of valid values the
827 /// parameter accepts.
828 pub fn new_parameter_bool(
829 &mut self,
830 name: impl AsRef<str>,
831 description: impl AsRef<str>,
832 kind: ParameterKind,
833 default: Option<bool>,
834 ) -> UseParameter<bool> {
835 let idx = self.parameters.len();
836 let name = new_parameter_name(name, kind.clone());
837 self.parameters.push(ParameterMeta {
838 parameter: Parameter::Bool {
839 name,
840 description: description.as_ref().into(),
841 kind,
842 default,
843 },
844 used_by_jobs: BTreeSet::new(),
845 });
846
847 UseParameter {
848 idx,
849 _kind: std::marker::PhantomData,
850 }
851 }
852
853 /// Declare a pipeline-level runtime parameter with type `i64`.
854 ///
855 /// To obtain a [`ReadVar<i64>`] that can be used within a node, use the
856 /// [`PipelineJobCtx::use_parameter`] method.
857 ///
858 /// `name` is the name of the parameter.
859 ///
860 /// `description` is an arbitrary string, which will be be shown to users.
861 ///
862 /// `kind` is the type of parameter and if it should be treated as a stable
863 /// external API to callers of the pipeline.
864 ///
865 /// `default` is the default value for the parameter. If none is provided,
866 /// the parameter _must_ be specified in order for the pipeline to run.
867 ///
868 /// `possible_values` can be used to limit the set of valid values the
869 /// parameter accepts.
870 pub fn new_parameter_num(
871 &mut self,
872 name: impl AsRef<str>,
873 description: impl AsRef<str>,
874 kind: ParameterKind,
875 default: Option<i64>,
876 possible_values: Option<Vec<i64>>,
877 ) -> UseParameter<i64> {
878 let idx = self.parameters.len();
879 let name = new_parameter_name(name, kind.clone());
880 self.parameters.push(ParameterMeta {
881 parameter: Parameter::Num {
882 name,
883 description: description.as_ref().into(),
884 kind,
885 default,
886 possible_values,
887 },
888 used_by_jobs: BTreeSet::new(),
889 });
890
891 UseParameter {
892 idx,
893 _kind: std::marker::PhantomData,
894 }
895 }
896
897 /// Declare a pipeline-level runtime parameter with type `String`.
898 ///
899 /// To obtain a [`ReadVar<String>`] that can be used within a node, use the
900 /// [`PipelineJobCtx::use_parameter`] method.
901 ///
902 /// `name` is the name of the parameter.
903 ///
904 /// `description` is an arbitrary string, which will be be shown to users.
905 ///
906 /// `kind` is the type of parameter and if it should be treated as a stable
907 /// external API to callers of the pipeline.
908 ///
909 /// `default` is the default value for the parameter. If none is provided,
910 /// the parameter _must_ be specified in order for the pipeline to run.
911 ///
912 /// `possible_values` allows restricting inputs to a set of possible values.
913 /// Depending on the backend, these options may be presented as a set of
914 /// radio buttons, a dropdown menu, or something in that vein. If `None`,
915 /// then any string is allowed.
916 pub fn new_parameter_string(
917 &mut self,
918 name: impl AsRef<str>,
919 description: impl AsRef<str>,
920 kind: ParameterKind,
921 default: Option<impl AsRef<str>>,
922 possible_values: Option<Vec<String>>,
923 ) -> UseParameter<String> {
924 let idx = self.parameters.len();
925 let name = new_parameter_name(name, kind.clone());
926 self.parameters.push(ParameterMeta {
927 parameter: Parameter::String {
928 name,
929 description: description.as_ref().into(),
930 kind,
931 default: default.map(|x| x.as_ref().into()),
932 possible_values,
933 },
934 used_by_jobs: BTreeSet::new(),
935 });
936
937 UseParameter {
938 idx,
939 _kind: std::marker::PhantomData,
940 }
941 }
942}
943
944pub struct PipelineJobCtx<'a> {
945 pipeline: &'a mut Pipeline,
946 job_idx: usize,
947}
948
949impl PipelineJobCtx<'_> {
950 /// Create a new `WriteVar<SideEffect>` anchored to the pipeline job.
951 pub fn new_done_handle(&mut self) -> WriteVar<crate::node::SideEffect> {
952 self.pipeline.dummy_done_idx += 1;
953 crate::node::thin_air_write_runtime_var(format!("start{}", self.pipeline.dummy_done_idx))
954 }
955
956 /// Claim that this job will use this artifact, obtaining a path to a folder
957 /// with the artifact's contents.
958 pub fn use_artifact(&mut self, artifact: &UseArtifact) -> ReadVar<PathBuf> {
959 self.pipeline.artifacts[artifact.idx]
960 .used_by_jobs
961 .insert(self.job_idx);
962
963 crate::node::thin_air_read_runtime_var(consistent_artifact_runtime_var_name(
964 &self.pipeline.artifacts[artifact.idx].name,
965 true,
966 ))
967 }
968
969 /// Claim that this job will publish this artifact, obtaining a path to a
970 /// fresh, empty folder which will be published as the specific artifact at
971 /// the end of the job.
972 pub fn publish_artifact(&mut self, artifact: PublishArtifact) -> ReadVar<PathBuf> {
973 let existing = self.pipeline.artifacts[artifact.idx]
974 .published_by_job
975 .replace(self.job_idx);
976 assert!(existing.is_none()); // PublishArtifact isn't cloneable
977
978 crate::node::thin_air_read_runtime_var(consistent_artifact_runtime_var_name(
979 &self.pipeline.artifacts[artifact.idx].name,
980 false,
981 ))
982 }
983
984 fn helper_request<R: IntoRequest>(&mut self, req: R)
985 where
986 R::Node: 'static,
987 {
988 self.pipeline.jobs[self.job_idx]
989 .root_nodes
990 .entry(NodeHandle::from_type::<R::Node>())
991 .or_default()
992 .push(serde_json::to_vec(&req.into_request()).unwrap().into());
993 }
994
995 fn new_artifact_map_vars<T: Artifact>(&mut self) -> (ReadVar<T>, WriteVar<T>) {
996 let artifact_map_idx = self.pipeline.artifact_map_idx;
997 self.pipeline.artifact_map_idx += 1;
998
999 let backing_var = format!("artifact_map{}", artifact_map_idx);
1000 let read_var = crate::node::thin_air_read_runtime_var(backing_var.clone());
1001 let write_var = crate::node::thin_air_write_runtime_var(backing_var);
1002 (read_var, write_var)
1003 }
1004
1005 /// Claim that this job will use this artifact, obtaining the resolved
1006 /// contents of the artifact.
1007 pub fn use_typed_artifact<T: Artifact>(
1008 &mut self,
1009 artifact: &UseTypedArtifact<T>,
1010 ) -> ReadVar<T> {
1011 let artifact_path = self.use_artifact(&artifact.0);
1012 let (read, write) = self.new_artifact_map_vars::<T>();
1013 self.helper_request(artifact::resolve::Request::new(artifact_path, write));
1014 read
1015 }
1016
1017 /// Claim that this job will publish this artifact, obtaining a variable to
1018 /// write the artifact's contents to. The artifact will be published at
1019 /// the end of the job.
1020 pub fn publish_typed_artifact<T: Artifact>(
1021 &mut self,
1022 artifact: PublishTypedArtifact<T>,
1023 ) -> WriteVar<T> {
1024 let artifact_path = self.publish_artifact(artifact.0);
1025 let (read, write) = self.new_artifact_map_vars::<T>();
1026 let done = self.new_done_handle();
1027 self.helper_request(artifact::publish::Request::new(read, artifact_path, done));
1028 write
1029 }
1030
1031 /// Obtain a `ReadVar<T>` corresponding to a pipeline parameter which is
1032 /// specified at runtime.
1033 pub fn use_parameter<T>(&mut self, param: UseParameter<T>) -> ReadVar<T>
1034 where
1035 T: Serialize + DeserializeOwned,
1036 {
1037 self.pipeline.parameters[param.idx]
1038 .used_by_jobs
1039 .insert(self.job_idx);
1040
1041 crate::node::thin_air_read_runtime_var(
1042 self.pipeline.parameters[param.idx]
1043 .parameter
1044 .name()
1045 .to_string(),
1046 )
1047 }
1048
1049 /// Shortcut which allows defining a bool pipeline parameter within a Job.
1050 ///
1051 /// To share a single parameter between multiple jobs, don't use this method
1052 /// - use [`Pipeline::new_parameter_bool`] + [`Self::use_parameter`] instead.
1053 pub fn new_parameter_bool(
1054 &mut self,
1055 name: impl AsRef<str>,
1056 description: impl AsRef<str>,
1057 kind: ParameterKind,
1058 default: Option<bool>,
1059 ) -> ReadVar<bool> {
1060 let param = self
1061 .pipeline
1062 .new_parameter_bool(name, description, kind, default);
1063 self.use_parameter(param)
1064 }
1065
1066 /// Shortcut which allows defining a number pipeline parameter within a Job.
1067 ///
1068 /// To share a single parameter between multiple jobs, don't use this method
1069 /// - use [`Pipeline::new_parameter_num`] + [`Self::use_parameter`] instead.
1070 pub fn new_parameter_num(
1071 &mut self,
1072 name: impl AsRef<str>,
1073 description: impl AsRef<str>,
1074 kind: ParameterKind,
1075 default: Option<i64>,
1076 possible_values: Option<Vec<i64>>,
1077 ) -> ReadVar<i64> {
1078 let param =
1079 self.pipeline
1080 .new_parameter_num(name, description, kind, default, possible_values);
1081 self.use_parameter(param)
1082 }
1083
1084 /// Shortcut which allows defining a string pipeline parameter within a Job.
1085 ///
1086 /// To share a single parameter between multiple jobs, don't use this method
1087 /// - use [`Pipeline::new_parameter_string`] + [`Self::use_parameter`] instead.
1088 pub fn new_parameter_string(
1089 &mut self,
1090 name: impl AsRef<str>,
1091 description: impl AsRef<str>,
1092 kind: ParameterKind,
1093 default: Option<String>,
1094 possible_values: Option<Vec<String>>,
1095 ) -> ReadVar<String> {
1096 let param =
1097 self.pipeline
1098 .new_parameter_string(name, description, kind, default, possible_values);
1099 self.use_parameter(param)
1100 }
1101}
1102
1103#[must_use]
1104pub struct PipelineJob<'a> {
1105 pipeline: &'a mut Pipeline,
1106 job_idx: usize,
1107}
1108
1109impl PipelineJob<'_> {
1110 /// (ADO only) specify which agent pool this job will be run on.
1111 pub fn ado_set_pool(self, pool: AdoPool) -> Self {
1112 self.pipeline.jobs[self.job_idx].ado_pool = Some(pool);
1113 self
1114 }
1115
1116 /// (ADO only) specify which agent pool this job will be run on, with
1117 /// additional special runner demands.
1118 pub fn ado_set_pool_with_demands(self, pool: impl AsRef<str>, demands: Vec<String>) -> Self {
1119 self.pipeline.jobs[self.job_idx].ado_pool = Some(AdoPool {
1120 name: pool.as_ref().into(),
1121 demands,
1122 });
1123 self
1124 }
1125
1126 /// (ADO only) Declare a job-level, named, read-only ADO variable.
1127 ///
1128 /// `name` and `value` are both arbitrary strings, which may include ADO
1129 /// template expressions.
1130 ///
1131 /// NOTE: Unless required by some particular third-party task, it's strongly
1132 /// recommended to _avoid_ using this method, and to simply use
1133 /// [`ReadVar::from_static`] to get a obtain a static variable.
1134 ///
1135 /// DEVNOTE: In the future, this API may be updated to return a handle that
1136 /// will allow resolving the resulting `AdoRuntimeVar`, but for
1137 /// implementation expediency, this API does not currently do this. If you
1138 /// need to read the value of this variable at runtime, you may need to
1139 /// invoke [`AdoRuntimeVar::dangerous_from_global`] manually.
1140 ///
1141 /// [`NodeCtx::get_ado_variable`]: crate::node::NodeCtx::get_ado_variable
1142 pub fn ado_new_named_variable(self, name: impl AsRef<str>, value: impl AsRef<str>) -> Self {
1143 let name = name.as_ref();
1144 let value = value.as_ref();
1145 self.pipeline.jobs[self.job_idx]
1146 .ado_variables
1147 .insert(name.into(), value.into());
1148 self
1149 }
1150
1151 /// (ADO only) Declare multiple job-level, named, read-only ADO variables at
1152 /// once.
1153 ///
1154 /// This is a convenience method to streamline invoking
1155 /// [`Self::ado_new_named_variable`] multiple times.
1156 ///
1157 /// NOTE: Unless required by some particular third-party task, it's strongly
1158 /// recommended to _avoid_ using this method, and to simply use
1159 /// [`ReadVar::from_static`] to get a obtain a static variable.
1160 ///
1161 /// DEVNOTE: In the future, this API may be updated to return a handle that
1162 /// will allow resolving the resulting `AdoRuntimeVar`, but for
1163 /// implementation expediency, this API does not currently do this. If you
1164 /// need to read the value of this variable at runtime, you may need to
1165 /// invoke [`AdoRuntimeVar::dangerous_from_global`] manually.
1166 ///
1167 /// [`NodeCtx::get_ado_variable`]: crate::node::NodeCtx::get_ado_variable
1168 pub fn ado_new_named_variables<K, V>(self, vars: impl IntoIterator<Item = (K, V)>) -> Self
1169 where
1170 K: AsRef<str>,
1171 V: AsRef<str>,
1172 {
1173 self.pipeline.jobs[self.job_idx].ado_variables.extend(
1174 vars.into_iter()
1175 .map(|(k, v)| (k.as_ref().into(), v.as_ref().into())),
1176 );
1177 self
1178 }
1179
1180 /// Overrides the id of the job.
1181 ///
1182 /// Flowey typically generates a reasonable job ID but some use cases that depend
1183 /// on the ID may find it useful to override it to something custom.
1184 pub fn ado_override_job_id(self, name: impl AsRef<str>) -> Self {
1185 self.pipeline
1186 .ado_job_id_overrides
1187 .insert(self.job_idx, name.as_ref().into());
1188 self
1189 }
1190
1191 /// (GitHub Actions only) specify which Github runner this job will be run on.
1192 pub fn gh_set_pool(self, pool: GhRunner) -> Self {
1193 self.pipeline.jobs[self.job_idx].gh_pool = Some(pool);
1194 self
1195 }
1196
1197 /// (GitHub Actions only) Manually override the `if:` condition for this
1198 /// particular job.
1199 ///
1200 /// **This is dangerous**, as an improperly set `if` condition may break
1201 /// downstream flowey jobs which assume flowey is in control of the job's
1202 /// scheduling logic.
1203 ///
1204 /// See
1205 /// <https://docs.github.com/en/actions/writing-workflows/workflow-syntax-for-github-actions#jobsjob_idif>
1206 /// for more info.
1207 pub fn gh_dangerous_override_if(self, condition: impl AsRef<str>) -> Self {
1208 self.pipeline.jobs[self.job_idx].gh_override_if = Some(condition.as_ref().into());
1209 self
1210 }
1211
1212 /// (GitHub Actions only) Declare a global job-level environment variable,
1213 /// visible to all downstream steps.
1214 ///
1215 /// `name` and `value` are both arbitrary strings, which may include GitHub
1216 /// Actions template expressions.
1217 ///
1218 /// **This is dangerous**, as it is easy to misuse this API in order to
1219 /// write a node which takes an implicit dependency on there being a global
1220 /// variable set on its behalf by the top-level pipeline code, making it
1221 /// difficult to "locally reason" about the behavior of a node simply by
1222 /// reading its code.
1223 ///
1224 /// Whenever possible, nodes should "late bind" environment variables:
1225 /// accepting a compile-time / runtime flowey parameter, and then setting it
1226 /// prior to executing a child command that requires it.
1227 ///
1228 /// Only use this API in exceptional cases, such as obtaining an environment
1229 /// variable whose value is determined by a job-level GitHub Actions
1230 /// expression evaluation.
1231 pub fn gh_dangerous_global_env_var(
1232 self,
1233 name: impl AsRef<str>,
1234 value: impl AsRef<str>,
1235 ) -> Self {
1236 let name = name.as_ref();
1237 let value = value.as_ref();
1238 self.pipeline.jobs[self.job_idx]
1239 .gh_global_env
1240 .insert(name.into(), value.into());
1241 self
1242 }
1243
1244 /// (GitHub Actions only) Grant permissions required by nodes in the job.
1245 ///
1246 /// For a given node handle, grant the specified permissions.
1247 /// The list provided must match the permissions specified within the node
1248 /// using `requires_permission`.
1249 ///
1250 /// NOTE: While this method is called at a node-level for auditability, the emitted
1251 /// yaml grants permissions at the job-level.
1252 ///
1253 /// This can lead to weird situations where node 1 might not specify a permission
1254 /// required according to Github Actions, but due to job-level granting of the permission
1255 /// by another node 2, the pipeline executes even though it wouldn't if node 2 was removed.
1256 ///
1257 /// For available permission scopes and their descriptions, see
1258 /// <https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#permissions>.
1259 pub fn gh_grant_permissions<N: FlowNodeBase + 'static>(
1260 self,
1261 permissions: impl IntoIterator<Item = (GhPermission, GhPermissionValue)>,
1262 ) -> Self {
1263 let node_handle = NodeHandle::from_type::<N>();
1264 for (permission, value) in permissions {
1265 self.pipeline.jobs[self.job_idx]
1266 .gh_permissions
1267 .entry(node_handle)
1268 .or_default()
1269 .insert(permission, value);
1270 }
1271 self
1272 }
1273
1274 pub fn apply_patchfn(self, patchfn: crate::patch::PatchFn) -> Self {
1275 self.pipeline.jobs[self.job_idx]
1276 .patches
1277 .apply_patchfn(patchfn);
1278 self
1279 }
1280
1281 /// Set a timeout for the job, in minutes.
1282 ///
1283 /// Not calling this will result in the platform's default timeout being used,
1284 /// which is typically 60 minutes, but may vary.
1285 pub fn with_timeout_in_minutes(self, timeout: u32) -> Self {
1286 self.pipeline.jobs[self.job_idx].timeout_minutes = Some(timeout);
1287 self
1288 }
1289
1290 /// (ADO+Local Only) Only run the job if the specified condition is true.
1291 pub fn with_condition(self, cond: UseParameter<bool>) -> Self {
1292 self.pipeline.jobs[self.job_idx].cond_param_idx = Some(cond.idx);
1293 self.pipeline.parameters[cond.idx]
1294 .used_by_jobs
1295 .insert(self.job_idx);
1296 self
1297 }
1298
1299 /// Set a [`CommandWrapperKind`] that will be applied to all shell
1300 /// commands executed in this job's steps.
1301 ///
1302 /// The wrapper is applied both when running locally (via direct run)
1303 /// and when running in CI (the kind is serialized into
1304 /// `pipeline.json` and reconstructed at runtime).
1305 ///
1306 /// [`CommandWrapperKind`]: crate::shell::CommandWrapperKind
1307 pub fn set_command_wrapper(self, wrapper: crate::shell::CommandWrapperKind) -> Self {
1308 self.pipeline.jobs[self.job_idx].command_wrapper = Some(wrapper);
1309 self
1310 }
1311
1312 /// Add a flow node which will be run as part of the job.
1313 pub fn dep_on<R: IntoRequest + 'static>(
1314 self,
1315 f: impl FnOnce(&mut PipelineJobCtx<'_>) -> R,
1316 ) -> Self {
1317 // JobToNodeCtx will ensure artifact deps are taken care of
1318 let req = f(&mut PipelineJobCtx {
1319 pipeline: self.pipeline,
1320 job_idx: self.job_idx,
1321 });
1322
1323 self.pipeline.jobs[self.job_idx]
1324 .root_nodes
1325 .entry(NodeHandle::from_type::<R::Node>())
1326 .or_default()
1327 .push(serde_json::to_vec(&req.into_request()).unwrap().into());
1328
1329 self
1330 }
1331
1332 /// Add a flow node whose request publishes a typed artifact.
1333 ///
1334 /// This is a shortcut for the common pattern of calling
1335 /// [`PipelineJobCtx::publish_typed_artifact`] inside a [`Self::dep_on`]
1336 /// closure and passing the resulting [`WriteVar`] into a request.
1337 pub fn publish<T: Artifact, R: IntoRequest + 'static>(
1338 self,
1339 artifact: PublishTypedArtifact<T>,
1340 f: impl FnOnce(WriteVar<T>) -> R,
1341 ) -> Self {
1342 self.dep_on(|ctx| f(ctx.publish_typed_artifact(artifact)))
1343 }
1344
1345 /// Add a flow node whose request is run purely for its side effect.
1346 ///
1347 /// This is a shortcut for the common pattern of calling
1348 /// [`PipelineJobCtx::new_done_handle`] inside a [`Self::dep_on`]
1349 /// closure and passing the resulting [`WriteVar`] into a request.
1350 pub fn side_effect<R: IntoRequest + 'static>(
1351 self,
1352 f: impl FnOnce(WriteVar<crate::node::SideEffect>) -> R,
1353 ) -> Self {
1354 self.dep_on(|ctx| f(ctx.new_done_handle()))
1355 }
1356
1357 /// Set config on a node for this job.
1358 ///
1359 /// This is the pipeline-level equivalent of [`NodeCtx::config`]. Config
1360 /// set here is merged with any config set by nodes within the job.
1361 ///
1362 /// [`NodeCtx::config`]: crate::node::NodeCtx::config
1363 pub fn config<C: IntoConfig + 'static>(self, config: C) -> Self {
1364 self.pipeline.jobs[self.job_idx]
1365 .root_configs
1366 .entry(NodeHandle::from_type::<C::Node>())
1367 .or_default()
1368 .push(serde_json::to_vec(&config).unwrap().into());
1369
1370 self
1371 }
1372
1373 /// Finish describing the pipeline job.
1374 pub fn finish(self) -> PipelineJobHandle {
1375 PipelineJobHandle {
1376 job_idx: self.job_idx,
1377 }
1378 }
1379
1380 /// Return the job's platform.
1381 pub fn get_platform(&self) -> FlowPlatform {
1382 self.pipeline.jobs[self.job_idx].platform
1383 }
1384
1385 /// Return the job's architecture.
1386 pub fn get_arch(&self) -> FlowArch {
1387 self.pipeline.jobs[self.job_idx].arch
1388 }
1389}
1390
1391#[derive(Clone)]
1392pub struct PipelineJobHandle {
1393 job_idx: usize,
1394}
1395
1396impl PipelineJobHandle {
1397 pub fn is_handle_for(&self, job: &PipelineJob<'_>) -> bool {
1398 self.job_idx == job.job_idx
1399 }
1400}
1401
1402#[derive(Clone, Copy)]
1403pub enum PipelineBackendHint {
1404 /// Pipeline is being run on the user's dev machine (via bash / direct run)
1405 Local,
1406 /// Pipeline is run on ADO
1407 Ado,
1408 /// Pipeline is run on GitHub Actions
1409 Github,
1410}
1411
1412/// Trait for types that can be converted into a [`Pipeline`].
1413///
1414/// This is the primary entry point for defining flowey pipelines. Implement this trait
1415/// to create a pipeline definition that can be executed locally or converted to CI YAML.
1416///
1417/// # Example
1418///
1419/// ```rust,no_run
1420/// use flowey_core::pipeline::{IntoPipeline, Pipeline, PipelineBackendHint};
1421/// use flowey_core::node::{FlowPlatform, FlowPlatformLinuxDistro, FlowArch};
1422///
1423/// struct MyPipeline;
1424///
1425/// impl IntoPipeline for MyPipeline {
1426/// fn into_pipeline(self, backend_hint: PipelineBackendHint) -> anyhow::Result<Pipeline> {
1427/// let mut pipeline = Pipeline::new();
1428///
1429/// // Define a job that runs on Linux x86_64
1430/// let _job = pipeline
1431/// .new_job(
1432/// FlowPlatform::Linux(FlowPlatformLinuxDistro::Ubuntu),
1433/// FlowArch::X86_64,
1434/// "build"
1435/// )
1436/// .finish();
1437///
1438/// Ok(pipeline)
1439/// }
1440/// }
1441/// ```
1442///
1443/// # Complex Example with Parameters and Artifacts
1444///
1445/// ```rust,ignore
1446/// use flowey_core::pipeline::{IntoPipeline, Pipeline, PipelineBackendHint, ParameterKind};
1447/// use flowey_core::node::{FlowPlatform, FlowPlatformLinuxDistro, FlowArch};
1448///
1449/// struct BuildPipeline;
1450///
1451/// impl IntoPipeline for BuildPipeline {
1452/// fn into_pipeline(self, backend_hint: PipelineBackendHint) -> anyhow::Result<Pipeline> {
1453/// let mut pipeline = Pipeline::new();
1454///
1455/// // Define a runtime parameter
1456/// let enable_tests = pipeline.new_parameter_bool(
1457/// "enable_tests",
1458/// "Whether to run tests",
1459/// ParameterKind::Stable,
1460/// Some(true) // default value
1461/// );
1462///
1463/// // Create an artifact for passing data between jobs
1464/// let (publish_build, use_build) = pipeline.new_artifact("build-output");
1465///
1466/// // Job 1: Build
1467/// let build_job = pipeline
1468/// .new_job(
1469/// FlowPlatform::Linux(FlowPlatformLinuxDistro::Ubuntu),
1470/// FlowArch::X86_64,
1471/// "build"
1472/// )
1473/// .with_timeout_in_minutes(30)
1474/// .dep_on(|ctx| flowey_lib_hvlite::_jobs::example_node::Request {
1475/// output_dir: ctx.publish_artifact(publish_build),
1476/// })
1477/// .finish();
1478///
1479/// // Job 2: Test (conditionally run based on parameter)
1480/// let _test_job = pipeline
1481/// .new_job(
1482/// FlowPlatform::Linux(FlowPlatformLinuxDistro::Ubuntu),
1483/// FlowArch::X86_64,
1484/// "test"
1485/// )
1486/// .with_condition(enable_tests)
1487/// .dep_on(|ctx| flowey_lib_hvlite::_jobs::example_node2::Request {
1488/// input_dir: ctx.use_artifact(&use_build),
1489/// })
1490/// .finish();
1491///
1492/// Ok(pipeline)
1493/// }
1494/// }
1495/// ```
1496pub trait IntoPipeline {
1497 fn into_pipeline(self, backend_hint: PipelineBackendHint) -> anyhow::Result<Pipeline>;
1498}
1499
1500fn new_parameter_name(name: impl AsRef<str>, kind: ParameterKind) -> String {
1501 match kind {
1502 ParameterKind::Unstable => format!("__unstable_{}", name.as_ref()),
1503 ParameterKind::Stable => name.as_ref().into(),
1504 }
1505}
1506
1507/// Structs which should only be used by top-level flowey emitters. If you're a
1508/// pipeline author, these are not types you need to care about!
1509pub mod internal {
1510 use super::*;
1511 use std::collections::BTreeMap;
1512
1513 pub fn consistent_artifact_runtime_var_name(artifact: impl AsRef<str>, is_use: bool) -> String {
1514 format!(
1515 "artifact_{}_{}",
1516 if is_use { "use_from" } else { "publish_from" },
1517 artifact.as_ref()
1518 )
1519 }
1520
1521 #[derive(Debug)]
1522 pub struct InternalAdoResourcesRepository {
1523 /// flowey-generated unique repo identifier
1524 pub repo_id: String,
1525 /// Type of repo that is being connected to.
1526 pub repo_type: AdoResourcesRepositoryType,
1527 /// Repository name. Format depends on `repo_type`.
1528 pub name: String,
1529 /// git ref to checkout.
1530 pub git_ref: AdoResourcesRepositoryRef<usize>,
1531 /// (optional) ID of the service endpoint connecting to this repository.
1532 pub endpoint: Option<String>,
1533 }
1534
1535 pub struct PipelineJobMetadata {
1536 pub root_nodes: BTreeMap<NodeHandle, Vec<Box<[u8]>>>,
1537 pub root_configs: BTreeMap<NodeHandle, Vec<Box<[u8]>>>,
1538 pub patches: PatchResolver,
1539 pub label: String,
1540 pub platform: FlowPlatform,
1541 pub arch: FlowArch,
1542 pub cond_param_idx: Option<usize>,
1543 pub timeout_minutes: Option<u32>,
1544 pub command_wrapper: Option<crate::shell::CommandWrapperKind>,
1545 // backend specific
1546 pub ado_pool: Option<AdoPool>,
1547 pub ado_variables: BTreeMap<String, String>,
1548 pub gh_override_if: Option<String>,
1549 pub gh_pool: Option<GhRunner>,
1550 pub gh_global_env: BTreeMap<String, String>,
1551 pub gh_permissions: BTreeMap<NodeHandle, BTreeMap<GhPermission, GhPermissionValue>>,
1552 }
1553
1554 #[derive(Debug)]
1555 pub struct ArtifactMeta {
1556 pub name: String,
1557 pub published_by_job: Option<usize>,
1558 pub used_by_jobs: BTreeSet<usize>,
1559 }
1560
1561 #[derive(Debug)]
1562 pub struct ParameterMeta {
1563 pub parameter: Parameter,
1564 pub used_by_jobs: BTreeSet<usize>,
1565 }
1566
1567 /// Mirror of [`Pipeline`], except with all field marked as `pub`.
1568 pub struct PipelineFinalized {
1569 pub jobs: Vec<PipelineJobMetadata>,
1570 pub artifacts: Vec<ArtifactMeta>,
1571 pub parameters: Vec<ParameterMeta>,
1572 pub extra_deps: BTreeSet<(usize, usize)>,
1573 // backend specific
1574 pub ado_name: Option<String>,
1575 pub ado_schedule_triggers: Vec<AdoScheduleTriggers>,
1576 pub ado_ci_triggers: Option<AdoCiTriggers>,
1577 pub ado_pr_triggers: Option<AdoPrTriggers>,
1578 pub ado_bootstrap_template: String,
1579 pub ado_resources_repository: Vec<InternalAdoResourcesRepository>,
1580 pub ado_post_process_yaml_cb:
1581 Option<Box<dyn FnOnce(serde_yaml::Value) -> serde_yaml::Value>>,
1582 pub ado_variables: BTreeMap<String, String>,
1583 pub ado_job_id_overrides: BTreeMap<usize, String>,
1584 pub gh_name: Option<String>,
1585 pub gh_schedule_triggers: Vec<GhScheduleTriggers>,
1586 pub gh_ci_triggers: Option<GhCiTriggers>,
1587 pub gh_pr_triggers: Option<GhPrTriggers>,
1588 pub gh_bootstrap_template: String,
1589 }
1590
1591 impl PipelineFinalized {
1592 pub fn from_pipeline(mut pipeline: Pipeline) -> Self {
1593 if let Some(cb) = pipeline.inject_all_jobs_with.take() {
1594 for job_idx in 0..pipeline.jobs.len() {
1595 let _ = cb(PipelineJob {
1596 pipeline: &mut pipeline,
1597 job_idx,
1598 });
1599 }
1600 }
1601
1602 let Pipeline {
1603 mut jobs,
1604 artifacts,
1605 parameters,
1606 extra_deps,
1607 ado_name,
1608 ado_bootstrap_template,
1609 ado_schedule_triggers,
1610 ado_ci_triggers,
1611 ado_pr_triggers,
1612 ado_resources_repository,
1613 ado_post_process_yaml_cb,
1614 ado_variables,
1615 ado_job_id_overrides,
1616 gh_name,
1617 gh_schedule_triggers,
1618 gh_ci_triggers,
1619 gh_pr_triggers,
1620 gh_bootstrap_template,
1621 // not relevant to consumer code
1622 dummy_done_idx: _,
1623 artifact_map_idx: _,
1624 artifact_names: _,
1625 global_patchfns,
1626 inject_all_jobs_with: _, // processed above
1627 } = pipeline;
1628
1629 for patchfn in global_patchfns {
1630 for job in &mut jobs {
1631 job.patches.apply_patchfn(patchfn)
1632 }
1633 }
1634
1635 Self {
1636 jobs,
1637 artifacts,
1638 parameters,
1639 extra_deps,
1640 ado_name,
1641 ado_schedule_triggers,
1642 ado_ci_triggers,
1643 ado_pr_triggers,
1644 ado_bootstrap_template,
1645 ado_resources_repository,
1646 ado_post_process_yaml_cb,
1647 ado_variables,
1648 ado_job_id_overrides,
1649 gh_name,
1650 gh_schedule_triggers,
1651 gh_ci_triggers,
1652 gh_pr_triggers,
1653 gh_bootstrap_template,
1654 }
1655 }
1656 }
1657
1658 #[derive(Debug, Clone)]
1659 pub enum Parameter {
1660 Bool {
1661 name: String,
1662 description: String,
1663 kind: ParameterKind,
1664 default: Option<bool>,
1665 },
1666 String {
1667 name: String,
1668 description: String,
1669 default: Option<String>,
1670 kind: ParameterKind,
1671 possible_values: Option<Vec<String>>,
1672 },
1673 Num {
1674 name: String,
1675 description: String,
1676 default: Option<i64>,
1677 kind: ParameterKind,
1678 possible_values: Option<Vec<i64>>,
1679 },
1680 }
1681
1682 impl Parameter {
1683 pub fn name(&self) -> &str {
1684 match self {
1685 Parameter::Bool { name, .. } => name,
1686 Parameter::String { name, .. } => name,
1687 Parameter::Num { name, .. } => name,
1688 }
1689 }
1690 }
1691}