1use flowey::node::prelude::*;
7use std::collections::BTreeMap;
8
9#[derive(Serialize, Deserialize)]
11pub enum RepoSource<C = VarNotClaimed> {
12 AdoResource(AdoResourcesRepositoryId),
18 GithubRepo { owner: String, name: String },
20 GithubSelf,
22 ExistingClone(ReadVar<PathBuf, C>),
24 LocalOnlyNewClone {
26 url: String,
27 path: PathBuf,
28 ignore_existing_clone: bool,
29 },
30}
31
32impl<C> Clone for RepoSource<C> {
33 fn clone(&self) -> Self {
34 match self {
35 Self::AdoResource(arg0) => Self::AdoResource(arg0.clone()),
36 Self::GithubRepo { owner, name } => Self::GithubRepo {
37 owner: owner.clone(),
38 name: name.clone(),
39 },
40 Self::GithubSelf => Self::GithubSelf,
41 Self::ExistingClone(arg0) => Self::ExistingClone(arg0.clone()),
42 Self::LocalOnlyNewClone {
43 url,
44 path,
45 ignore_existing_clone,
46 } => Self::LocalOnlyNewClone {
47 url: url.clone(),
48 path: path.clone(),
49 ignore_existing_clone: *ignore_existing_clone,
50 },
51 }
52 }
53}
54
55impl ClaimVar for RepoSource {
57 type Claimed = RepoSource<VarClaimed>;
58
59 fn claim(self, ctx: &mut StepCtx<'_>) -> Self::Claimed {
60 match self {
61 RepoSource::AdoResource(x) => RepoSource::AdoResource(x),
62 RepoSource::GithubRepo { owner, name } => RepoSource::GithubRepo { owner, name },
63 RepoSource::GithubSelf => RepoSource::GithubSelf,
64 RepoSource::ExistingClone(v) => RepoSource::ExistingClone(v.claim(ctx)),
65 RepoSource::LocalOnlyNewClone {
66 url,
67 path,
68 ignore_existing_clone,
69 } => RepoSource::LocalOnlyNewClone {
70 url,
71 path,
72 ignore_existing_clone,
73 },
74 }
75 }
76}
77
78flowey_config! {
79 pub struct Config {
81 pub require_local_clones: Option<bool>,
84 }
85}
86
87flowey_request! {
88 pub enum Request {
89 CheckoutRepo {
104 repo_id: ReadVar<String>,
107 repo_path: WriteVar<PathBuf>,
109 persist_credentials: bool,
112 },
115 RegisterRepo {
117 repo_id: String,
120 repo_src: RepoSource,
122 allow_persist_credentials: bool,
130 depth: Option<usize>,
136 pre_run_deps: Vec<ReadVar<SideEffect>>,
137 },
138 }
139}
140
141new_flow_node_with_config!(struct Node);
142
143pub mod process_reqs {
145 use super::*;
146
147 pub struct RequestCheckoutRepo {
148 pub repo_id: ReadVar<String>,
149 pub repo_path: WriteVar<PathBuf>,
150 pub persist_credentials: bool,
151 }
152
153 pub struct RequestRegisterRepo {
154 pub repo_id: String,
155 pub repo_src: RepoSource,
156 pub allow_persist_credentials: bool,
157 pub depth: Option<usize>,
158 pub pre_run_deps: Vec<ReadVar<SideEffect>>,
159 }
160
161 pub struct ResolvedRequestsAdo {
162 pub checkout_repo: Vec<RequestCheckoutRepo>,
163 pub register_repo: Vec<RequestRegisterRepo>,
164 }
165
166 impl ResolvedRequestsAdo {
167 pub fn from_reqs(requests: Vec<Request>) -> anyhow::Result<Self> {
168 let ResolvedRequests::Ado(v) = process_reqs(requests, None)? else {
169 panic!()
170 };
171 Ok(v)
172 }
173 }
174
175 pub struct ResolvedRequestsLocal {
176 pub checkout_repo: Vec<RequestCheckoutRepo>,
177 pub register_repo: Vec<RequestRegisterRepo>,
178 pub require_local_clones: bool,
179 }
180
181 impl ResolvedRequestsLocal {
182 pub fn from_reqs(
183 requests: Vec<Request>,
184 require_local_clones: Option<bool>,
185 ) -> anyhow::Result<Self> {
186 let ResolvedRequests::Local(v) = process_reqs(requests, require_local_clones)? else {
187 panic!()
188 };
189 Ok(v)
190 }
191 }
192
193 enum ResolvedRequests {
194 Ado(ResolvedRequestsAdo),
195 Local(ResolvedRequestsLocal),
196 }
197
198 fn process_reqs(
199 requests: Vec<Request>,
200 require_local_clones: Option<bool>,
201 ) -> anyhow::Result<ResolvedRequests> {
202 let mut checkout_repo = Vec::new();
203 let mut register_repo = Vec::new();
204
205 for req in requests {
206 match req {
207 Request::CheckoutRepo {
208 repo_id,
209 repo_path,
210 persist_credentials,
211 } => checkout_repo.push(RequestCheckoutRepo {
212 repo_id,
213 repo_path,
214 persist_credentials,
215 }),
216 Request::RegisterRepo {
217 repo_id,
218 repo_src,
219 allow_persist_credentials,
220 depth,
221 pre_run_deps,
222 } => register_repo.push(RequestRegisterRepo {
223 repo_id,
224 repo_src,
225 allow_persist_credentials,
226 depth,
227 pre_run_deps,
228 }),
229 }
230 }
231
232 Ok(if let Some(require_local_clones) = require_local_clones {
233 ResolvedRequests::Local(ResolvedRequestsLocal {
234 checkout_repo,
235 register_repo,
236 require_local_clones,
237 })
238 } else {
239 ResolvedRequests::Ado(ResolvedRequestsAdo {
240 checkout_repo,
241 register_repo,
242 })
243 })
244 }
245}
246
247impl FlowNodeWithConfig for Node {
248 type Request = Request;
249 type Config = Config;
250
251 fn imports(dep: &mut ImportCtx<'_>) {
252 dep.import::<crate::install_git::Node>();
253 }
254
255 fn emit(
256 config: Config,
257 requests: Vec<Self::Request>,
258 ctx: &mut NodeCtx<'_>,
259 ) -> anyhow::Result<()> {
260 match ctx.backend() {
261 FlowBackend::Local => {
262 let require_local_clones = config
263 .require_local_clones
264 .ok_or(anyhow::anyhow!("missing config: require_local_clones"))?;
265 Self::emit_local(requests, require_local_clones, ctx)
266 }
267 FlowBackend::Ado => {
268 if config.require_local_clones.is_some() {
269 anyhow::bail!(
270 "can only set `require_local_clones` when using the Local backend"
271 );
272 }
273 Self::emit_ado(requests, ctx)
274 }
275 FlowBackend::Github => {
276 if config.require_local_clones.is_some() {
277 anyhow::bail!(
278 "can only set `require_local_clones` when using the Local backend"
279 );
280 }
281 Self::emit_gh(requests, ctx)
282 }
283 }
284 }
285}
286
287impl Node {
288 fn emit_ado(requests: Vec<Request>, ctx: &mut NodeCtx<'_>) -> anyhow::Result<()> {
289 let process_reqs::ResolvedRequestsAdo {
290 checkout_repo,
291 register_repo,
292 } = process_reqs::ResolvedRequestsAdo::from_reqs(requests)?;
293
294 if checkout_repo.is_empty() {
295 return Ok(());
296 }
297
298 let mut did_checkouts = Vec::new();
299 let mut registered_repos = BTreeMap::<(String, bool), (usize, RepoSource)>::new();
300 for (
301 idx,
302 process_reqs::RequestRegisterRepo {
303 repo_id,
304 repo_src,
305 allow_persist_credentials,
306 depth,
307 pre_run_deps,
308 },
309 ) in register_repo.into_iter().enumerate()
310 {
311 let existing = registered_repos.insert(
312 (repo_id.clone(), allow_persist_credentials),
313 (idx, repo_src.clone()),
314 );
315 if existing.is_some() {
316 anyhow::bail!("got a duplicate RegisterRepo request for {repo_id}")
317 }
318
319 let (persist_credentials_str, write_persist_credentials_str) = ctx.new_var();
320 let (active, write_active) = ctx.new_var();
321
322 ctx.emit_rust_step(format!("check if {repo_id} needs to be cloned"), |ctx| {
323 pre_run_deps.claim(ctx);
324 let write_active = write_active.claim(ctx);
325 let write_persist_credentials_str = write_persist_credentials_str.claim(ctx);
326 let repo_ids = checkout_repo
327 .iter()
328 .map(|process_reqs::RequestCheckoutRepo { repo_id, persist_credentials, .. }| {
329 ( repo_id.clone().claim(ctx), *persist_credentials)
330 })
331 .collect::<Vec<_>>();
332 let repo_id = repo_id.clone();
333 move |rt| {
334 for (requested_checkout_repo_id, persist_credentials) in repo_ids {
335 if rt.read(requested_checkout_repo_id) == repo_id {
336 if persist_credentials {
337 if allow_persist_credentials != persist_credentials {
338 anyhow::bail!("pipeline implementation bug: attempted to checkout repo with `persist_credentials`, whose registration didn't include `allow_persist_credentials: true`")
339 }
340 }
341
342 rt.write(write_persist_credentials_str, &persist_credentials.to_string());
343 rt.write(write_active, &true);
344 return Ok(());
345 }
346 }
347
348 rt.write(write_active, &false);
349 Ok(())
350 }
351 });
352
353 let (did_checkout, claim_did_checkout) = ctx.new_var();
354 if let RepoSource::AdoResource(checkout_str) = repo_src {
355 ctx.emit_ado_step_with_condition(
356 format!("checkout repo {repo_id}"),
357 active.clone(),
358 |ctx| {
359 claim_did_checkout.claim(ctx);
360 let persist_credentials_str = persist_credentials_str.claim(ctx);
361 move |rt| {
362 let checkout_str = rt.resolve_repository_id(checkout_str);
363 let persist_credentials =
364 rt.get_var(persist_credentials_str).as_raw_var_name();
365 let depth = match depth {
366 Some(x) => x.to_string(),
367 None => "0".into(),
368 };
369
370 format!(
377 r#"
378 - checkout: {checkout_str}
379 path: repo{idx}
380 fetchTags: false
381 fetchDepth: {depth}
382 persistCredentials: $({persist_credentials})
383 submodules: recursive
384 "#
385 )
386 }
387 },
388 );
389 } else {
390 ctx.emit_side_effect_step(
391 [
392 active.into_side_effect(),
393 persist_credentials_str.into_side_effect(),
394 ],
395 [claim_did_checkout],
396 )
397 }
398
399 did_checkouts.push(did_checkout);
400 }
401
402 let workspace = ctx.get_ado_variable(AdoRuntimeVar::PIPELINE_WORKSPACE);
403
404 ctx.emit_rust_step("report cloned repo directories", move |ctx| {
405 did_checkouts.claim(ctx);
406 let workspace = workspace.claim(ctx);
407 let mut registered_repos = registered_repos.into_iter().map(|(k, (a, b))| (k, (a, b.claim(ctx)))).collect::<BTreeMap<_, _>>();
408 let checkout_repo = checkout_repo
409 .into_iter()
410 .map(|process_reqs::RequestCheckoutRepo { repo_id, repo_path, persist_credentials }| {
411 (repo_id.claim(ctx), repo_path.claim(ctx), persist_credentials)
412 })
413 .collect::<Vec<_>>();
414
415 move |rt| {
416 let workspace = PathBuf::from(rt.read(workspace));
417 let mut checkout_reqs = BTreeMap::<(String, bool), Vec<ClaimedWriteVar<PathBuf>>>::new();
418 for (repo_id, repo_path, persist_credentials) in checkout_repo {
419 checkout_reqs
420 .entry((rt.read(repo_id), persist_credentials))
421 .or_default()
422 .push(repo_path);
423 }
424
425
426 for ((repo_id, persist_credentials), repo_paths) in checkout_reqs {
427 let (idx, repo_src) = registered_repos
428 .remove(&(repo_id.clone(), persist_credentials))
429 .with_context(|| format!("pipeline implementation bug: did not specify a RegisterRepo request for repo {repo_id}"))?;
430
431 let path = match repo_src {
432 RepoSource::AdoResource(_) => {
433 workspace.join(format!("repo{idx}"))
434 },
435 RepoSource::GithubRepo{ .. } | RepoSource::GithubSelf => anyhow::bail!("repo source for ADO backend must be an `AdoResource` or `ExistingClone`"),
436 RepoSource::ExistingClone(path) => {
437 let path = rt.read(path);
438 path.absolute().context(format!("Failed to make {} absolute", path.display()))?
439 },
440 RepoSource::LocalOnlyNewClone { .. } => unreachable!(),
441 };
442
443 log::info!("reporting repo is cloned at {}", path.display());
444 for var in repo_paths {
445 rt.write(var, &path);
446 }
447 }
448
449 Ok(())
450 }
451 });
452
453 Ok(())
454 }
455
456 fn emit_gh(requests: Vec<Request>, ctx: &mut NodeCtx<'_>) -> anyhow::Result<()> {
457 let process_reqs::ResolvedRequestsAdo {
458 checkout_repo,
459 register_repo,
460 } = process_reqs::ResolvedRequestsAdo::from_reqs(requests)?;
461
462 if checkout_repo.is_empty() {
463 return Ok(());
464 }
465
466 let mut did_checkouts = Vec::new();
467 let mut registered_repos = BTreeMap::<(String, bool), (usize, RepoSource)>::new();
468 for (
469 idx,
470 process_reqs::RequestRegisterRepo {
471 repo_id,
472 repo_src,
473 allow_persist_credentials,
474 depth,
475 pre_run_deps,
476 },
477 ) in register_repo.into_iter().enumerate()
478 {
479 let existing = registered_repos.insert(
480 (repo_id.clone(), allow_persist_credentials),
481 (idx, repo_src.clone()),
482 );
483 if existing.is_some() {
484 anyhow::bail!("got a duplicate RegisterRepo request for {repo_id}")
485 }
486
487 let (persist_credentials_str, write_persist_credentials_str) = ctx.new_var();
488 let (active, write_active) = ctx.new_var();
489 ctx.emit_rust_step(format!("check if {repo_id} needs to be cloned"), |ctx| {
490 pre_run_deps.claim(ctx);
491 let write_active = write_active.claim(ctx);
492 let write_persist_credentials_str = write_persist_credentials_str.claim(ctx);
493 let repo_ids = checkout_repo
494 .iter()
495 .map(|process_reqs::RequestCheckoutRepo { repo_id, persist_credentials, .. }| {
496 (repo_id.clone().claim(ctx), *persist_credentials)
497 })
498 .collect::<Vec<_>>();
499 let repo_id = repo_id.clone();
500 move |rt| {
501 for (requested_checkout_repo_id, persist_credentials) in repo_ids {
502 if rt.read(requested_checkout_repo_id) == repo_id {
503 if persist_credentials {
504 if allow_persist_credentials != persist_credentials {
505 anyhow::bail!("pipeline implementation bug: attempted to checkout repo with `persist_credentials`, whose registration didn't include `allow_persist_credentials: true`")
506 }
507 }
508
509 rt.write(write_persist_credentials_str, &persist_credentials.to_string());
510 rt.write(write_active, &true);
511 return Ok(());
512 }
513 }
514
515 rt.write(write_active, &false);
516 Ok(())
517 }
518 });
519
520 if matches!(
521 repo_src,
522 RepoSource::GithubSelf | RepoSource::GithubRepo { .. }
523 ) {
524 let mut step = ctx
526 .emit_gh_step(
527 format!("checkout repo {repo_id}"),
528 "actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803",
529 )
530 .condition(active.clone())
531 .with("path", format!("repo{idx}"))
532 .with("fetch-depth", depth.unwrap_or(0).to_string())
533 .with("persist-credentials", persist_credentials_str)
534 .requires_permission(GhPermission::Contents, GhPermissionValue::Read);
535 if let RepoSource::GithubRepo { owner, name } = repo_src {
536 step = step.with("repository", format!("{owner}/{name}"))
537 }
538 did_checkouts.push(step.finish(ctx));
539 } else if !matches!(repo_src, RepoSource::ExistingClone(_)) {
540 anyhow::bail!(
541 "repo source must be a `GithubRepo`, `GithubSelf`, or `ExistingClone` for GitHub backend"
542 );
543 }
544 }
545
546 let parent_path = ctx.get_gh_context_var().global().workspace();
547 ctx.emit_rust_step("report cloned repo directories", move |ctx| {
548 did_checkouts.claim(ctx);
549 let mut registered_repos = registered_repos.into_iter().map(|(k, (a, b))| (k, (a, b.claim(ctx)))).collect::<BTreeMap<_, _>>();
550 let checkout_repo = checkout_repo
551 .into_iter()
552 .map(|process_reqs::RequestCheckoutRepo { repo_id, repo_path, persist_credentials }| {
553 (repo_id.claim(ctx), repo_path.claim(ctx), persist_credentials)
554 })
555 .collect::<Vec<_>>();
556 let parent_path = parent_path.claim(ctx);
557
558 move |rt| {
559 let mut checkout_reqs = BTreeMap::<(String, bool), Vec<ClaimedWriteVar<PathBuf>>>::new();
560 for (repo_id, repo_path, persist_credentials) in checkout_repo {
561 checkout_reqs
562 .entry((rt.read(repo_id), persist_credentials))
563 .or_default()
564 .push(repo_path);
565 }
566
567 let parent_path = rt.read(parent_path);
568 for ((repo_id, persist_credentials), repo_paths) in checkout_reqs {
569 let (idx, repo_src) = registered_repos
570 .remove(&(repo_id.clone(), persist_credentials))
571 .with_context(|| format!("pipeline implementation bug: did not specify a RegisterRepo request for repo {repo_id}"))?;
572
573 let path = match repo_src {
574 RepoSource::AdoResource(_) => unreachable!(),
575 RepoSource::GithubRepo{ .. } => {
576 PathBuf::from(parent_path.clone()).join(format!("repo{idx}"))
577 },
578 RepoSource::GithubSelf => {
579 PathBuf::from(parent_path.clone()).join(format!("repo{idx}"))
580 },
581 RepoSource::ExistingClone(path) => {
582 let path = rt.read(path);
583 path.absolute().context(format!("Failed to make {} absolute", path.display()))?
584 },
585 RepoSource::LocalOnlyNewClone { .. } => unreachable!(),
586 };
587
588 log::info!("reporting repo is cloned at {}", path.display());
589
590 for var in repo_paths {
591 rt.write(var, &path);
592 }
593 }
594
595 Ok(())
596 }
597 });
598
599 Ok(())
600 }
601
602 fn emit_local(
603 requests: Vec<Request>,
604 require_local_clones: bool,
605 ctx: &mut NodeCtx<'_>,
606 ) -> anyhow::Result<()> {
607 let process_reqs::ResolvedRequestsLocal {
608 checkout_repo,
609 register_repo,
610 require_local_clones,
611 } = process_reqs::ResolvedRequestsLocal::from_reqs(requests, Some(require_local_clones))?;
612
613 if checkout_repo.is_empty() {
614 return Ok(());
615 }
616
617 let git_ensure_installed = ctx.reqv(crate::install_git::Request::EnsureInstalled);
618
619 ctx.emit_rust_step("report repo directory", move |ctx| {
620 git_ensure_installed.claim(ctx);
621 let register_repo = register_repo
622 .into_iter()
623 .map(|process_reqs::RequestRegisterRepo { repo_id, repo_src, allow_persist_credentials: _, depth, pre_run_deps }|
624 (repo_id, repo_src.claim(ctx), depth, pre_run_deps.claim(ctx)
625 )).collect::<Vec<_>>();
626 let checkout_repo = checkout_repo
627 .into_iter()
628 .map(|process_reqs::RequestCheckoutRepo { repo_id, repo_path, persist_credentials }| {
629 (repo_id.claim(ctx), repo_path.claim(ctx), persist_credentials)
630 })
631 .collect::<Vec<_>>();
632
633 move |rt| {
634 for (checkout_repo_id, repo_path, _persist_credentials) in checkout_repo {
635 let checkout_repo_id = rt.read(checkout_repo_id);
636
637 log::info!("reporting checkout info for {checkout_repo_id}");
638
639 let mut found_path = None;
640 for (repo_id, repo_src, depth, _) in ®ister_repo {
641 if &checkout_repo_id != repo_id {
642 continue;
643 }
644
645 match repo_src {
646 RepoSource::ExistingClone(path) => {
647 let path = rt.read(path.clone());
648 let path = path.absolute().context(format!("Failed to make {} absolute", path.display()))?;
649 found_path = Some(path);
650 break;
651 }
652 RepoSource::LocalOnlyNewClone { .. } if require_local_clones => {
653 anyhow::bail!("`LocalOnlyRequireExistingClones` is active, all repos must be registered using `RepoKind::ExistingClone`");
654 }
655 RepoSource::LocalOnlyNewClone { url, path, ignore_existing_clone } => {
656 if rt.sh.path_exists(path) {
657 rt.sh.change_dir(path);
658 if flowey::shell_cmd!(rt, "git status").run().is_ok()
659 && *ignore_existing_clone
660 {
661 rt.write(repo_path, path);
662 return Ok(());
663 }
664 }
665 if let Some(depth_arg) = depth {
666 let depth_arg_string = depth_arg.to_string();
667 flowey::shell_cmd!(rt, "git clone --depth {depth_arg_string} {url} {path}").run()?;
668 } else {
669 flowey::shell_cmd!(rt, "git clone {url} {path}").run()?;
670 }
671 found_path = Some(path.clone());
672 break;
673 }
674 RepoSource::AdoResource( .. ) => {
675 anyhow::bail!("ADO resources are not supported on local backend");
676 }
677 RepoSource::GithubRepo{ .. } | RepoSource::GithubSelf => {
678 anyhow::bail!("Github repos for GH Actions are not supported on local backend");
679 }
680 }
681 }
682
683 if let Some(path) = found_path {
684 rt.write(repo_path, &path);
685 } else {
686 anyhow::bail!("missing registration for id {checkout_repo_id}")
687 }
688 }
689
690 Ok(())
691 }
692 });
693
694 Ok(())
695 }
696}