From 9e449be38be48b614cee9e1ab76bc469083a7bec Mon Sep 17 00:00:00 2001 From: Sebastian Date: Tue, 5 Mar 2024 11:21:06 +0100 Subject: [PATCH] chore: support -j N syntax for parallel flag and remove default --- CHANGELOG.md | 3 +- cmd/terramate/cli/cli.go | 42 +------------------ cmd/terramate/cli/run.go | 2 +- cmd/terramate/cli/script_run.go | 2 +- .../cloud/run_cloud_deployment_test.go | 2 +- .../e2etests/cloud/run_cloud_drift_test.go | 2 +- .../e2etests/cloud/run_cloud_signal_test.go | 4 +- .../cloud/run_script_cloud_deployment_test.go | 2 +- .../cloud/run_script_cloud_drift_test.go | 2 +- .../e2etests/core/run_parallel_test.go | 2 +- 10 files changed, 12 insertions(+), 51 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a48e7610e..ba1164a23 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -41,8 +41,7 @@ Given a version number `MAJOR.MINOR.PATCH`, we increment the: - Improve user experience when using Terramate with existing Terragrunt projects. - Add `terramate create --all-terragrunt` option, which will automatically create Terramate stacks for each Terraform module. - Allow to run independent stacks in parallel for faster deployments and better utilization of system resources in general. - - Add `--parallel` (short `-j`) option to `terramate run` and `terramate script run`. - - `--parallel=N` limits the number of concurrent runs to `N`, otherwise a sensible default limit is chosen. + - Add `--parallel=N` (short `-j N`) option to `terramate run` and `terramate script run` to allow running up to `N` stacks in parallel. - Ordering constraints between stacks are still respected, i.e. `before`/`after`, parent before sub-folders. - Add `cloud_sync_drift_status` option to `script` block commands. It allows for synchronizing the stack drift details from script jobs. diff --git a/cmd/terramate/cli/cli.go b/cmd/terramate/cli/cli.go index 19254b7a2..c3867cdd0 100644 --- a/cmd/terramate/cli/cli.go +++ b/cmd/terramate/cli/cli.go @@ -11,8 +11,6 @@ import ( "os" "path" "path/filepath" - "reflect" - "strconv" "strings" "time" @@ -102,12 +100,6 @@ const ( // UIMode defines different modes of operation for the cli. type UIMode int -type kongParallelFlag struct { - Value int -} - -const defaultParallelRunCount = 5 - type cliSpec struct { VersionFlag bool `hidden:"true" name:"version" help:"Show Terramate version."` Chdir string `short:"C" optional:"true" predictor:"file" help:"Set working directory."` @@ -171,7 +163,7 @@ type cliSpec struct { // Note: 0 is not the real default value here, this is just a workaround. // Kong doesn't support having 0 as the default value in case the flag isn't set, but K in case it's set without a value. // The K case is handled in the custom decoder. - Parallel kongParallelFlag `short:"j" optional:"true" default:"0" help:"Run independent stacks in parallel."` + Parallel int `short:"j" optional:"true" help:"Run independent stacks in parallel."` runSafeguardsCliSpec @@ -196,7 +188,7 @@ type cliSpec struct { Cmds []string `arg:"" optional:"true" passthrough:"" help:"Script to execute."` // See above comment regarding for run --parallel. - Parallel kongParallelFlag `short:"j" optional:"true" default:"0" help:"Run independent stacks in parallel"` + Parallel int `short:"j" optional:"true" help:"Run independent stacks in parallel."` runSafeguardsCliSpec } `cmd:"" help:"Run a Terramate Script in stacks."` @@ -709,36 +701,6 @@ func (c *cli) run() { } } -func (s *kongParallelFlag) Decode(ctx *kong.DecodeContext) error { - if ctx.Scan.Peek().Type == kong.FlagValueToken { - t, err := ctx.Scan.PopValue("counter") - if err != nil { - return err - } - - switch v := t.Value.(type) { - case string: - n, err := strconv.ParseInt(v, 10, 64) - if err != nil { - return stdfmt.Errorf("expected a counter but got %q (%T)", t, t.Value) - } - s.Value = int(n) - - case int, int8, int16, int32, int64: - t := reflect.ValueOf(v) - s.Value = int(t.Int()) - - default: - return stdfmt.Errorf("expected a counter but got %q (%T)", t, t.Value) - } - return nil - } - - s.Value = defaultParallelRunCount - - return nil -} - func (c *cli) setupSafeguards(run runSafeguardsCliSpec) { global := c.parsedArgs.deprecatedGlobalSafeguardsCliSpec diff --git a/cmd/terramate/cli/run.go b/cmd/terramate/cli/run.go index ec42a8293..402c3c761 100644 --- a/cmd/terramate/cli/run.go +++ b/cmd/terramate/cli/run.go @@ -187,7 +187,7 @@ func (c *cli) runOnStacks() { Reverse: c.parsedArgs.Run.Reverse, ScriptRun: false, ContinueOnError: c.parsedArgs.Run.ContinueOnError, - Parallel: c.parsedArgs.Run.Parallel.Value, + Parallel: c.parsedArgs.Run.Parallel, }) if err != nil { fatal("one or more commands failed", err) diff --git a/cmd/terramate/cli/script_run.go b/cmd/terramate/cli/script_run.go index 625fdaf6c..65bc5ae3c 100644 --- a/cmd/terramate/cli/script_run.go +++ b/cmd/terramate/cli/script_run.go @@ -118,7 +118,7 @@ func (c *cli) runScript() { DryRun: c.parsedArgs.Script.Run.DryRun, ScriptRun: true, ContinueOnError: false, - Parallel: c.parsedArgs.Script.Run.Parallel.Value, + Parallel: c.parsedArgs.Script.Run.Parallel, }) if err != nil { fatal("one or more commands failed", err) diff --git a/cmd/terramate/e2etests/cloud/run_cloud_deployment_test.go b/cmd/terramate/e2etests/cloud/run_cloud_deployment_test.go index c6e02e444..6f0518cd6 100644 --- a/cmd/terramate/e2etests/cloud/run_cloud_deployment_test.go +++ b/cmd/terramate/e2etests/cloud/run_cloud_deployment_test.go @@ -407,7 +407,7 @@ func TestCLIRunWithCloudSyncDeployment(t *testing.T) { "--cloud-sync-deployment", } if isParallel { - runflags = append(runflags, "--parallel") + runflags = append(runflags, "--parallel", "5") tc.want.run.IgnoreStdout = true tc.want.run.IgnoreStderr = true } diff --git a/cmd/terramate/e2etests/cloud/run_cloud_drift_test.go b/cmd/terramate/e2etests/cloud/run_cloud_drift_test.go index b29e27df1..99a8392f2 100644 --- a/cmd/terramate/e2etests/cloud/run_cloud_drift_test.go +++ b/cmd/terramate/e2etests/cloud/run_cloud_drift_test.go @@ -531,7 +531,7 @@ func TestCLIRunWithCloudSyncDriftStatus(t *testing.T) { "--cloud-sync-drift-status", } if isParallel { - runflags = append(runflags, "--parallel") + runflags = append(runflags, "-j", "5") tc.want.run.IgnoreStdout = true tc.want.run.IgnoreStderr = true } diff --git a/cmd/terramate/e2etests/cloud/run_cloud_signal_test.go b/cmd/terramate/e2etests/cloud/run_cloud_signal_test.go index 35fbef20e..36076e1d0 100644 --- a/cmd/terramate/e2etests/cloud/run_cloud_signal_test.go +++ b/cmd/terramate/e2etests/cloud/run_cloud_signal_test.go @@ -120,7 +120,7 @@ func TestCLIRunWithCloudSyncDeploymentWithSignals(t *testing.T) { "--cloud-sync-deployment", } if isParallel { - runflags = append(runflags, "--parallel") + runflags = append(runflags, "--parallel", "5") tc.want.run.IgnoreStdout = true tc.want.run.IgnoreStderr = true } @@ -220,7 +220,7 @@ func TestCLIRunWithCloudSyncDriftStatusWithSignals(t *testing.T) { "--cloud-sync-drift-status", } if isParallel { - runflags = append(runflags, "--parallel") + runflags = append(runflags, "--parallel=5") tc.want.run.IgnoreStdout = true tc.want.run.IgnoreStderr = true } diff --git a/cmd/terramate/e2etests/cloud/run_script_cloud_deployment_test.go b/cmd/terramate/e2etests/cloud/run_script_cloud_deployment_test.go index e57e5a86c..d59837082 100644 --- a/cmd/terramate/e2etests/cloud/run_script_cloud_deployment_test.go +++ b/cmd/terramate/e2etests/cloud/run_script_cloud_deployment_test.go @@ -236,7 +236,7 @@ func TestCLIScriptRunWithCloudSyncDeployment(t *testing.T) { scriptArgs := []string{"--quiet", "--disable-safeguards=git-out-of-sync"} if isParallel { - scriptArgs = append(scriptArgs, "--parallel") + scriptArgs = append(scriptArgs, "--parallel=5") // For the parallel test, we ignore output validation, since the print order is non-deterministic. tc.want.run.IgnoreStderr = true tc.want.run.IgnoreStdout = true diff --git a/cmd/terramate/e2etests/cloud/run_script_cloud_drift_test.go b/cmd/terramate/e2etests/cloud/run_script_cloud_drift_test.go index 422c8a1df..4c6b88072 100644 --- a/cmd/terramate/e2etests/cloud/run_script_cloud_drift_test.go +++ b/cmd/terramate/e2etests/cloud/run_script_cloud_drift_test.go @@ -512,7 +512,7 @@ func TestScriptRunDriftStatus(t *testing.T) { "--quiet", } if isParallel { - runflags = append(runflags, "--parallel") + runflags = append(runflags, "--parallel", "5") tc.want.run.IgnoreStdout = true tc.want.run.IgnoreStderr = true } diff --git a/cmd/terramate/e2etests/core/run_parallel_test.go b/cmd/terramate/e2etests/core/run_parallel_test.go index 038493814..2d7149167 100644 --- a/cmd/terramate/e2etests/core/run_parallel_test.go +++ b/cmd/terramate/e2etests/core/run_parallel_test.go @@ -39,7 +39,7 @@ func TestParallelFibonacci(t *testing.T) { tmcli := NewCLI(t, s.RootDir()) - res := tmcli.Run("run", "--quiet", "--parallel", "--", HelperPath, "fibonacci") + res := tmcli.Run("run", "--quiet", "--parallel=5", "--", HelperPath, "fibonacci") AssertRunResult(t, res, RunExpected{}) b, err := os.ReadFile(s.RootDir() + fmt.Sprintf("/fib.%v/fib.txt", tc.FibN))