feat: add --{include,only}-output-dependencies flag. (#2033)

## What this PR does / why we need it:

Add:
- `terramate run --include-output-dependencies`
- `terramate run --only-output-dependencies`
- `terramate script run --include-output-dependencies`
- `terramate script run --only-output-dependencies`

## Which issue(s) this PR fixes:
none

## Special notes for your reviewer:

## Does this PR introduce a user-facing change?
```
yes, add new flags. Requires documentation
```

CHANGELOG.md

@@ -22,6 +22,13 @@ Given a version number `MAJOR.MINOR.PATCH`, we increment the:
 
 ## Unreleased
 
+### Added
+
+- Add `terramate [run, script run] --{include,only}-output-dependencies` flags.
+  - These flags are needed for initializing the output dependencies of stacks which had its dependencies not changed in the same run.
+  - The `--include-output-dependencies` flag includes the output dependencies in the execution order.
+  - The `--only-output-dependencies` flag only includes the output dependencies in the execution order.
+
 ### Fixed
 
 - Fix the sync of `base_branch` information in the Terramate Cloud deployment.

cmd/terramate/cli/cli.go

@@ -152,6 +152,7 @@ type cliSpec struct {
 	Run struct {
 		runCommandFlags `envprefix:"TM_ARG_RUN_"`
 		runSafeguardsCliSpec
+		outputsSharingFlags
 	} `cmd:"" help:"Run command in the stacks"`
 
 	Generate struct {
@@ -168,6 +169,7 @@ type cliSpec struct {
 		Run struct {
 			runScriptFlags `envprefix:"TM_ARG_RUN_"`
 			runSafeguardsCliSpec
+			outputsSharingFlags
 		} `cmd:"" help:"Run a Terramate Script in stacks."`
 	} `cmd:"" help:"Use Terramate Scripts"`
 
@@ -304,6 +306,11 @@ type changeDetectionFlags struct {
 	DisableChangeDetection []string `help:"Disable specific change detection modes" enum:"git-untracked,git-uncommitted"`
 }
 
+type outputsSharingFlags struct {
+	IncludeOutputDependencies bool `help:"Include stacks that are dependencies of the selected stacks. (requires outputs-sharing experiment enabled)"`
+	OnlyOutputDependencies    bool `help:"Only include stacks that are dependencies of the selected stacks. (requires outputs-sharing experiment enabled)"`
+}
+
 type cloudTargetFlags struct {
 	Target     string `env:"TARGET" help:"Set the deployment target for stacks synchronized to Terramate Cloud."`
 	FromTarget string `env:"FROM_TARGET" help:"Migrate stacks from given deployment target."`
@@ -1151,7 +1158,7 @@ func (c *cli) triggerStackByFilter() {
 	stackFilter := cloud.StatusFilters{
 		StackStatus: statusFilter,
 	}
-	stacksReport, err := c.listStacks(false, "", stackFilter, false)
+	stacksReport, err := c.listStacks(false, cloudstack.AnyTarget, stackFilter, false)
 	if err != nil {
 		fatalWithDetailf(err, "unable to list stacks")
 	}
@@ -2189,19 +2196,19 @@ func generateDot(
 }
 
 func (c *cli) generateDebug() {
-	// TODO(KATCIPIS): When we introduce config defined on root context
-	// we need to know blocks that have root context, since they should
-	// not be filtered by stack selection.
-	stacks, err := c.computeSelectedStacks(false, cloudstack.AnyTarget, cloud.NoStatusFilters())
+	report, err := c.listStacks(c.parsedArgs.Changed, cloudstack.AnyTarget, cloud.NoStatusFilters(), false)
 	if err != nil {
 		fatalWithDetailf(err, "generate debug: selecting stacks")
 	}
 
 	selectedStacks := map[prj.Path]struct{}{}
-	for _, stack := range stacks {
-		log.Debug().Msgf("selected stack: %s", stack.Dir())
+	for _, entry := range report.Stacks {
+		stackdir := entry.Stack.HostDir(c.cfg())
+		if stackdir == c.wd() || strings.HasPrefix(stackdir, c.wd()+string(filepath.Separator)) {
+			log.Debug().Msgf("selected stack: %s", entry.Stack.Dir)
 
-		selectedStacks[stack.Dir()] = struct{}{}
+			selectedStacks[entry.Stack.Dir] = struct{}{}
+		}
 	}
 
 	results, err := generate.Load(c.cfg(), c.vendorDir())
@@ -2584,7 +2591,7 @@ func (c *cli) friendlyFmtDir(dir string) (string, bool) {
 	return prj.FriendlyFmtDir(c.rootdir(), c.wd(), dir)
 }
 
-func (c *cli) computeSelectedStacks(ensureCleanRepo bool, target string, stackFilters cloud.StatusFilters) (config.List[*config.SortableStack], error) {
+func (c *cli) computeSelectedStacks(ensureCleanRepo bool, outputFlags outputsSharingFlags, target string, stackFilters cloud.StatusFilters) (config.List[*config.SortableStack], error) {
 	report, err := c.listStacks(c.parsedArgs.Changed, target, stackFilters, true)
 	if err != nil {
 		return nil, err
@@ -2602,7 +2609,96 @@ func (c *cli) computeSelectedStacks(ensureCleanRepo bool, target string, stackFi
 	if err != nil {
 		return nil, errors.E(err, "adding wanted stacks")
 	}
-	return stacks, nil
+	return c.addOutputDependencies(outputFlags, stacks), nil
+}
+
+func (c *cli) addOutputDependencies(outputFlags outputsSharingFlags, stacks config.List[*config.SortableStack]) config.List[*config.SortableStack] {
+	logger := log.With().
+		Str("action", "cli.addOutputDependencies()").
+		Logger()
+
+	if !outputFlags.IncludeOutputDependencies && !outputFlags.OnlyOutputDependencies {
+		logger.Debug().Msg("output dependencies not requested")
+		return stacks
+	}
+
+	if outputFlags.IncludeOutputDependencies && outputFlags.OnlyOutputDependencies {
+		fatal(errors.E("--include-output-dependencies and --only-output-dependencies cannot be used together"))
+	}
+	if (outputFlags.IncludeOutputDependencies || outputFlags.OnlyOutputDependencies) && !c.cfg().HasExperiment(hcl.SharingIsCaringExperimentName) {
+		fatal(errors.E("--include-output-dependencies requires the '%s' experiment enabled", hcl.SharingIsCaringExperimentName))
+	}
+
+	stacksMap := map[string]*config.SortableStack{}
+	for _, stack := range stacks {
+		stacksMap[stack.Stack.Dir.String()] = stack
+	}
+
+	rootcfg := c.cfg()
+	depIDs := map[string]struct{}{}
+	depOrigins := map[string][]string{} // id -> stack paths
+	for _, st := range stacks {
+		evalctx := c.setupEvalContext(st.Stack, map[string]string{})
+		cfg, _ := rootcfg.Lookup(st.Stack.Dir)
+		for _, inputcfg := range cfg.Node.Inputs {
+			fromStackID, err := config.EvalInputFromStackID(evalctx, inputcfg)
+			if err != nil {
+				fatalWithDetailf(err, "evaluating `input.%s.from_stack_id`", inputcfg.Name)
+			}
+			depIDs[fromStackID] = struct{}{}
+			depOrigins[fromStackID] = append(depOrigins[fromStackID], st.Stack.Dir.String())
+
+			logger.Debug().
+				Str("stack", st.Stack.Dir.String()).
+				Str("dependency", fromStackID).
+				Msg("stack has output dependency")
+		}
+	}
+
+	mgr := c.stackManager()
+	outputsMap := map[string]*config.SortableStack{}
+	for depID := range depIDs {
+		st, found, err := mgr.StackByID(depID)
+		if err != nil {
+			fatalWithDetailf(err, "loading output dependencies of selected stacks")
+		}
+		if !found {
+			fatalWithDetailf(errors.E("dependency stack %s not found", depID), "loading output dependencies of selected stacks")
+		}
+
+		var reason string
+		depsOf := depOrigins[depID]
+		if len(depsOf) == 1 {
+			reason = stdfmt.Sprintf("Output dependency of stack %s", depsOf[0])
+		} else {
+			reason = stdfmt.Sprintf("Output dependency of stacks %s", strings.Join(depsOf, ", "))
+		}
+
+		logger.Debug().
+			Str("stack", st.Dir.String()).
+			Str("reason", reason).
+			Msg("adding output dependency")
+
+		outputsMap[st.Dir.String()] = &config.SortableStack{
+			Stack: st,
+		}
+	}
+
+	if outputFlags.IncludeOutputDependencies {
+		for _, dep := range outputsMap {
+			if _, found := stacksMap[dep.Stack.Dir.String()]; !found {
+				stacks = append(stacks, dep)
+			}
+		}
+		return stacks
+	}
+
+	// only output dependencies
+	stacks = config.List[*config.SortableStack]{}
+	for _, dep := range outputsMap {
+		stacks = append(stacks, dep)
+	}
+	return stacks
 }
 
 func (c *cli) filterStacks(stacks []stack.Entry) []stack.Entry {

cmd/terramate/cli/run.go

@@ -144,7 +144,7 @@ func (c *cli) runOnStacks() {
 		stackFilter := parseStatusFilter(c.parsedArgs.Run.Status)
 		deploymentFilter := parseDeploymentStatusFilter(c.parsedArgs.Run.DeploymentStatus)
 		driftFilter := parseDriftStatusFilter(c.parsedArgs.Run.DriftStatus)
-		stacks, err = c.computeSelectedStacks(true, c.parsedArgs.Run.Target, cloud.StatusFilters{
+		stacks, err = c.computeSelectedStacks(true, c.parsedArgs.Run.outputsSharingFlags, c.parsedArgs.Run.Target, cloud.StatusFilters{
 			StackStatus:      stackFilter,
 			DeploymentStatus: deploymentFilter,
 			DriftStatus:      driftFilter,

cmd/terramate/cli/script_info.go

@@ -25,7 +25,7 @@ import (
 func (c *cli) printScriptInfo() {
 	labels := c.parsedArgs.Script.Info.Cmds
 
-	stacks, err := c.computeSelectedStacks(false, cloudstack.AnyTarget, cloud.NoStatusFilters())
+	stacks, err := c.computeSelectedStacks(false, outputsSharingFlags{}, cloudstack.AnyTarget, cloud.NoStatusFilters())
 	if err != nil {
 		fatalWithDetailf(err, "computing selected stacks")
 	}

cmd/terramate/cli/script_run.go

@@ -58,7 +58,7 @@ func (c *cli) runScript() {
 		statusFilter := parseStatusFilter(c.parsedArgs.Script.Run.Status)
 		deploymentFilter := parseDeploymentStatusFilter(c.parsedArgs.Script.Run.DeploymentStatus)
 		driftFilter := parseDriftStatusFilter(c.parsedArgs.Script.Run.DriftStatus)
-		stacks, err = c.computeSelectedStacks(true, c.parsedArgs.Script.Run.Target, cloud.StatusFilters{
+		stacks, err = c.computeSelectedStacks(true, c.parsedArgs.Script.Run.outputsSharingFlags, c.parsedArgs.Script.Run.Target, cloud.StatusFilters{
 			StackStatus:      statusFilter,
 			DeploymentStatus: deploymentFilter,
 			DriftStatus:      driftFilter,

config/config.go

@@ -220,6 +220,24 @@ func (root *Root) StacksByTagsFilters(filters []string) (project.Paths, error) {
 	}).Paths(), nil
 }
 
+// StackByID returns the stack with the given ID.
+func (root *Root) StackByID(id string) (*Stack, bool, error) {
+	stacks := root.tree.stacks(func(tree *Tree) bool {
+		return tree.IsStack() && tree.Node.Stack.ID == id
+	})
+	if len(stacks) == 0 {
+		return nil, false, nil
+	}
+	if len(stacks) > 1 {
+		printer.Stderr.Warnf("multiple stacks with the same ID %q found.", id)
+	}
+	stack, err := stacks[0].Stack()
+	if err != nil {
+		return nil, true, err
+	}
+	return stack, true, nil
+}
+
 // LoadSubTree loads a subtree located at cfgdir into the current tree.
 func (root *Root) LoadSubTree(cfgdir project.Path) error {
 	var parent project.Path

config/sharing_backend.go

@@ -41,6 +41,11 @@ type (
 	Outputs []Output
 )
 
+// EvalInputFromStackID the `from_stack_id` field of an input block using the provided evaluation context.
+func EvalInputFromStackID(evalctx *eval.Context, input hcl.Input) (string, error) {
+	return evalString(evalctx, input.FromStackID, "input.from_stack_id")
+}
+
 // EvalInput evaluates an input block using the provided evaluation context.
 func EvalInput(evalctx *eval.Context, input hcl.Input) (Input, error) {
 	evaluatedInput := Input{