Added exclude_from_copy to config (#3543)

* Added exclude_from_copy to config

* Changed README info for more clear behaviour explanation on both include and exclude

* Linting error fixes

* Fixed broken test

---------

Co-authored-by: Roman Kabaev <[email protected]>

cli/commands/terraform/download_source.go

@@ -50,13 +50,28 @@ func downloadTerraformSource(ctx context.Context, source string, opts *options.T
 
 	opts.Logger.Debugf("Copying files from %s into %s", opts.WorkingDir, terraformSource.WorkingDir)
 
-	var includeInCopy []string
+	var includeInCopy, excludeFromCopy []string
+
 	if terragruntConfig.Terraform != nil && terragruntConfig.Terraform.IncludeInCopy != nil {
 		includeInCopy = *terragruntConfig.Terraform.IncludeInCopy
 	}
+
+	if terragruntConfig.Terraform != nil && terragruntConfig.Terraform.ExcludeFromCopy != nil {
+		excludeFromCopy = *terragruntConfig.Terraform.ExcludeFromCopy
+	}
+
 	// Always include the .tflint.hcl file, if it exists
 	includeInCopy = append(includeInCopy, tfLintConfig)
-	if err := util.CopyFolderContents(opts.Logger, opts.WorkingDir, terraformSource.WorkingDir, ModuleManifestName, includeInCopy); err != nil {
+
+	err = util.CopyFolderContents(
+		opts.Logger,
+		opts.WorkingDir,
+		terraformSource.WorkingDir,
+		ModuleManifestName,
+		includeInCopy,
+		excludeFromCopy,
+	)
+	if err != nil {
 		return nil, err
 	}
 
@@ -213,12 +228,21 @@ func updateGetters(terragruntOptions *options.TerragruntOptions, terragruntConfi
 
 		for getterName, getterValue := range getter.Getters {
 			if getterName == "file" {
-				var includeInCopy []string
+				var includeInCopy, excludeFromCopy []string
+
 				if terragruntConfig.Terraform != nil && terragruntConfig.Terraform.IncludeInCopy != nil {
 					includeInCopy = *terragruntConfig.Terraform.IncludeInCopy
 				}
 
-				client.Getters[getterName] = &FileCopyGetter{IncludeInCopy: includeInCopy, Logger: terragruntOptions.Logger}
+				if terragruntConfig.Terraform != nil && terragruntConfig.Terraform.ExcludeFromCopy != nil {
+					includeInCopy = *terragruntConfig.Terraform.ExcludeFromCopy
+				}
+
+				client.Getters[getterName] = &FileCopyGetter{
+					IncludeInCopy:   includeInCopy,
+					Logger:          terragruntOptions.Logger,
+					ExcludeFromCopy: excludeFromCopy,
+				}
 			} else {
 				client.Getters[getterName] = getterValue
 			}

cli/commands/terraform/download_source_test.go

@@ -494,6 +494,6 @@ func copyFolder(t *testing.T, src string, dest string) {
 	logger := log.New()
 	logger.SetOptions(log.WithOutput(io.Discard))
 
-	err := util.CopyFolderContents(logger, filepath.FromSlash(src), filepath.FromSlash(dest), ".terragrunt-test", nil)
+	err := util.CopyFolderContents(logger, filepath.FromSlash(src), filepath.FromSlash(dest), ".terragrunt-test", nil, nil)
 	require.NoError(t, err)
 }

cli/commands/terraform/file_copy_getter.go

@@ -21,7 +21,8 @@ type FileCopyGetter struct {
 
 	// List of glob paths that should be included in the copy. This can be used to override the default behavior of
 	// Terragrunt, which will skip hidden folders.
-	IncludeInCopy []string
+	IncludeInCopy   []string
+	ExcludeFromCopy []string
 
 	Logger log.Logger
 }
@@ -42,7 +43,7 @@ func (g *FileCopyGetter) Get(dst string, u *url.URL) error {
 		return errors.Errorf("source path must be a directory")
 	}
 
-	return util.CopyFolderContents(g.Logger, path, dst, SourceManifestName, g.IncludeInCopy)
+	return util.CopyFolderContents(g.Logger, path, dst, SourceManifestName, g.IncludeInCopy, g.ExcludeFromCopy)
 }
 
 // GetFile The original FileGetter already knows how to do file copying so long as we set the Copy flag to true, so just

config/config.go

@@ -440,6 +440,7 @@ type ErrorHook struct {
 func (conf *Hook) String() string {
 	return fmt.Sprintf("Hook{Name = %s, Commands = %v}", conf.Name, len(conf.Commands))
 }
+
 func (conf *ErrorHook) String() string {
 	return fmt.Sprintf("Hook{Name = %s, Commands = %v}", conf.Name, len(conf.Commands))
 }
@@ -456,7 +457,8 @@ type TerraformConfig struct {
 
 	// Ideally we can avoid the pointer to list slice, but if it is not a pointer, Terraform requires the attribute to
 	// be defined and we want to make this optional.
-	IncludeInCopy *[]string `hcl:"include_in_copy,attr"`
+	IncludeInCopy   *[]string `hcl:"include_in_copy,attr"`
+	ExcludeFromCopy *[]string `hcl:"exclude_from_copy,attr"`
 
 	CopyTerraformLockFile *bool `hcl:"copy_terraform_lock_file,attr"`
 }
@@ -1408,7 +1410,7 @@ func (cfg *TerragruntConfig) GetMapFieldMetadata(fieldType, fieldName string) (m
 		return nil, false
 	}
 
-	var result = make(map[string]string)
+	result := make(map[string]string)
 	for key, value := range value {
 		result[key] = fmt.Sprintf("%v", value)
 	}
@@ -1422,7 +1424,7 @@ func (cfg *TerragruntConfig) EngineOptions() (*options.EngineOptions, error) {
 		return nil, nil
 	}
 	// in case of Meta is null, set empty meta
-	var meta = map[string]interface{}{}
+	meta := map[string]interface{}{}
 
 	if cfg.Engine.Meta != nil {
 		parsedMeta, err := ParseCtyValueToMap(*cfg.Engine.Meta)

config/config_as_cty.go

@@ -555,6 +555,7 @@ type CtyTerraformConfig struct {
 	ExtraArgs             map[string]TerraformExtraArguments `cty:"extra_arguments"`
 	Source                *string                            `cty:"source"`
 	IncludeInCopy         *[]string                          `cty:"include_in_copy"`
+	ExcludeFromCopy       *[]string                          `cty:"exclude_from_copy"`
 	CopyTerraformLockFile *bool                              `cty:"copy_terraform_lock_file"`
 	BeforeHooks           map[string]Hook                    `cty:"before_hook"`
 	AfterHooks            map[string]Hook                    `cty:"after_hook"`
@@ -570,6 +571,7 @@ func terraformConfigAsCty(config *TerraformConfig) (cty.Value, error) {
 	configCty := CtyTerraformConfig{
 		Source:                config.Source,
 		IncludeInCopy:         config.IncludeInCopy,
+		ExcludeFromCopy:       config.ExcludeFromCopy,
 		CopyTerraformLockFile: config.CopyTerraformLockFile,
 		ExtraArgs:             map[string]TerraformExtraArguments{},
 		BeforeHooks:           map[string]Hook{},

config/include.go

@@ -503,6 +503,18 @@ func (cfg *TerragruntConfig) DeepMerge(sourceConfig *TerragruntConfig, terragrun
 				}
 			}
 
+			if sourceConfig.Terraform.ExcludeFromCopy != nil {
+				srcList := *sourceConfig.Terraform.ExcludeFromCopy
+
+				if cfg.Terraform.ExcludeFromCopy != nil {
+					targetList := *cfg.Terraform.ExcludeFromCopy
+					combinedList := append(srcList, targetList...)
+					cfg.Terraform.ExcludeFromCopy = &combinedList
+				} else {
+					cfg.Terraform.ExcludeFromCopy = &srcList
+				}
+			}
+
 			mergeExtraArgs(terragruntOptions, sourceConfig.Terraform.ExtraArgs, &cfg.Terraform.ExtraArgs)
 
 			mergeHooks(terragruntOptions, sourceConfig.Terraform.BeforeHooks, &cfg.Terraform.BeforeHooks)

config/include_test.go

@@ -152,6 +152,11 @@ func TestMergeConfigIntoIncludedConfig(t *testing.T) {
 			&config.TerragruntConfig{Terraform: &config.TerraformConfig{IncludeInCopy: &[]string{"abc"}}},
 			&config.TerragruntConfig{Terraform: &config.TerraformConfig{CopyTerraformLockFile: &[]bool{false}[0], IncludeInCopy: &[]string{"abc"}}},
 		},
+		{
+			&config.TerragruntConfig{Terraform: &config.TerraformConfig{CopyTerraformLockFile: &[]bool{false}[0]}},
+			&config.TerragruntConfig{Terraform: &config.TerraformConfig{ExcludeFromCopy: &[]string{"abc"}}},
+			&config.TerragruntConfig{Terraform: &config.TerraformConfig{CopyTerraformLockFile: &[]bool{false}[0], ExcludeFromCopy: &[]string{"abc"}}},
+		},
 	}
 
 	for _, testCase := range testCases {
@@ -324,6 +329,12 @@ func TestDeepMergeConfigIntoIncludedConfig(t *testing.T) {
 			&config.TerragruntConfig{Terraform: &config.TerraformConfig{IncludeInCopy: &[]string{"abc"}}},
 			&config.TerragruntConfig{Terraform: &config.TerraformConfig{CopyTerraformLockFile: &[]bool{false}[0], IncludeInCopy: &[]string{"abc"}}},
 		},
+		{
+			"terraform copy_terraform_lock_file",
+			&config.TerragruntConfig{Terraform: &config.TerraformConfig{CopyTerraformLockFile: &[]bool{false}[0]}},
+			&config.TerragruntConfig{Terraform: &config.TerraformConfig{ExcludeFromCopy: &[]string{"abc"}}},
+			&config.TerragruntConfig{Terraform: &config.TerraformConfig{CopyTerraformLockFile: &[]bool{false}[0], ExcludeFromCopy: &[]string{"abc"}}},
+		},
 	}
 
 	for _, tt := range tc {

docs/_docs/04_reference/config-blocks-and-attributes.md

@@ -110,6 +110,12 @@ The `terraform` block supports the following arguments:
     can specify that in this list to ensure it gets copied over to the scratch copy
     (e.g., `include_in_copy = [".python-version"]`).
 
+- `exclude_from_copy` (attribute): A list of glob patterns (e.g., `["*.txt"]`) that should always be skipped when copying into the
+  OpenTofu/Terraform working directory. All examples valid for `include_in_copy` can be used here.
+
+  *Note that using `include_in_copy` and `exclude_from_copy` are not mutually exclusive.*
+  If a file matches a pattern in both `include_in_copy` and `exclude_from_copy`, it will not be included. If you would like to ensure that the file *is* included, make sure the patterns you use for `include_in_copy` do not match the patterns in `exclude_from_copy`.
+
 - `copy_terraform_lock_file` (attribute): In certain use cases, you don't want to check the terraform provider lock
   file into your source repository from your working directory as described in
   [Lock File Handling]({{site.baseurl}}/docs/features/lock-file-handling/). This attribute allows you to disable the copy
@@ -432,7 +438,7 @@ For the `s3` backend, the following additional properties are supported in the `
 - `dynamodb_table` - (Optional) The name of a DynamoDB table to use for state locking and consistency. The table must have a primary key named LockID. If not present, locking will be disabled.
 - `skip_bucket_versioning`: When `true`, the S3 bucket that is created to store the state will not be versioned.
 - `skip_bucket_ssencryption`: When `true`, the S3 bucket that is created to store the state will not be configured with server-side encryption.
-- `skip_bucket_accesslogging`: _DEPRECATED_ If provided, will be ignored. A log warning will be issued in the console output to notify the user.
+- `skip_bucket_accesslogging`: *DEPRECATED* If provided, will be ignored. A log warning will be issued in the console output to notify the user.
 - `skip_bucket_root_access`: When `true`, the S3 bucket that is created will not be configured with bucket policies that allow access to the root AWS user.
 - `skip_bucket_enforced_tls`: When `true`, the S3 bucket that is created will not be configured with a bucket policy that enforces access to the bucket via a TLS connection.
 - `skip_bucket_public_access_blocking`: When `true`, the S3 bucket that is created will not have public access blocking enabled.

test/fixtures/read-config/full/source.hcl

@@ -26,6 +26,7 @@ remote_state {
 terraform {
   source                   = "./delorean"
   include_in_copy          = ["time_machine.*"]
+  exclude_from_copy        = ["excluded_time_machine.*"]
   copy_terraform_lock_file = true
 
   extra_arguments "var-files" {

test/helpers/package.go

@@ -88,7 +88,10 @@ func CopyEnvironment(t *testing.T, environmentPath string, includeInCopy ...stri
 
 	t.Logf("Copying %s to %s", environmentPath, tmpDir)
 
-	require.NoError(t, util.CopyFolderContents(createLogger(), environmentPath, util.JoinPath(tmpDir, environmentPath), ".terragrunt-test", includeInCopy))
+	require.NoError(
+		t,
+		util.CopyFolderContents(createLogger(), environmentPath, util.JoinPath(tmpDir, environmentPath), ".terragrunt-test", includeInCopy, nil),
+	)
 
 	return tmpDir
 }

test/integration_aws_test.go

@@ -1012,7 +1012,7 @@ func TestAwsParallelStateInit(t *testing.T) {
 		require.NoError(t, err)
 	}
 	for i := 0; i < 20; i++ {
-		err := util.CopyFolderContents(createLogger(), testFixtureParallelStateInit, tmpEnvPath, ".terragrunt-test", nil)
+		err := util.CopyFolderContents(createLogger(), testFixtureParallelStateInit, tmpEnvPath, ".terragrunt-test", nil, nil)
 		require.NoError(t, err)
 		err = os.Rename(
 			path.Join(tmpEnvPath, "template"),

test/integration_gcp_test.go

@@ -113,7 +113,7 @@ func TestGcpParallelStateInit(t *testing.T) {
 		require.NoError(t, err)
 	}
 	for i := 0; i < 20; i++ {
-		err := util.CopyFolderContents(createLogger(), testFixtureGcsParallelStateInit, tmpEnvPath, ".terragrunt-test", nil)
+		err := util.CopyFolderContents(createLogger(), testFixtureGcsParallelStateInit, tmpEnvPath, ".terragrunt-test", nil, nil)
 		require.NoError(t, err)
 		err = os.Rename(
 			path.Join(tmpEnvPath, "template"),

test/integration_json_test.go

@@ -435,6 +435,7 @@ func TestRenderJsonMetadataTerraform(t *testing.T) {
 			"error_hook":               map[string]interface{}{},
 			"extra_arguments":          map[string]interface{}{},
 			"include_in_copy":          nil,
+			"exclude_from_copy":        nil,
 			"source":                   "../terraform",
 			"copy_terraform_lock_file": nil,
 		},

test/integration_serial_aws_test.go

@@ -104,7 +104,7 @@ func testRemoteFixtureParallelism(t *testing.T, parallelism int, numberOfModules
 		t.Fatalf("Failed to create temp dir due to error: %v", err)
 	}
 	for i := 0; i < numberOfModules; i++ {
-		err := util.CopyFolderContents(createLogger(), testFixtureParallelism, tmpEnvPath, ".terragrunt-test", nil)
+		err := util.CopyFolderContents(createLogger(), testFixtureParallelism, tmpEnvPath, ".terragrunt-test", nil, nil)
 		if err != nil {
 			return "", 0, err
 		}

test/integration_test.go

@@ -2395,6 +2395,7 @@ func TestReadTerragruntConfigFull(t *testing.T) {
 		map[string]interface{}{
 			"source":                   "./delorean",
 			"include_in_copy":          []interface{}{"time_machine.*"},
+			"exclude_from_copy":        []interface{}{"excluded_time_machine.*"},
 			"copy_terraform_lock_file": true,
 			"extra_arguments": map[string]interface{}{
 				"var-files": map[string]interface{}{

test/integration_tflint_test.go

@@ -98,7 +98,7 @@ func TestTflintInitSameModule(t *testing.T) {
 	// generate multiple "app" modules that will be initialized in parallel
 	for i := 0; i < 50; i++ {
 		appPath := util.JoinPath(modulePath, "dev", fmt.Sprintf("app-%d", i))
-		err := util.CopyFolderContents(createLogger(), appTemplate, appPath, ".terragrunt-test", []string{})
+		err := util.CopyFolderContents(createLogger(), appTemplate, appPath, ".terragrunt-test", []string{}, []string{})
 		require.NoError(t, err)
 	}
 	helpers.RunTerragrunt(t, "terragrunt run-all init --terragrunt-log-level trace --terragrunt-non-interactive --terragrunt-working-dir "+runPath)
@@ -203,7 +203,7 @@ func CopyEnvironmentWithTflint(t *testing.T, environmentPath string) string {
 
 	t.Logf("Copying %s to %s", environmentPath, tmpDir)
 
-	require.NoError(t, util.CopyFolderContents(createLogger(), environmentPath, util.JoinPath(tmpDir, environmentPath), ".terragrunt-test", []string{".tflint.hcl"}))
+	require.NoError(t, util.CopyFolderContents(createLogger(), environmentPath, util.JoinPath(tmpDir, environmentPath), ".terragrunt-test", []string{".tflint.hcl"}, []string{}))
 
 	return tmpDir
 }

test/integration_windows_test.go

@@ -179,7 +179,7 @@ func CopyEnvironmentToPath(t *testing.T, environmentPath, targetPath string) {
 		t.Fatalf("Failed to create temp dir %s due to error %v", targetPath, err)
 	}
 
-	copyErr := util.CopyFolderContents(createLogger(), environmentPath, util.JoinPath(targetPath, environmentPath), ".terragrunt-test", nil)
+	copyErr := util.CopyFolderContents(createLogger(), environmentPath, util.JoinPath(targetPath, environmentPath), ".terragrunt-test", nil, nil)
 	require.NoError(t, copyErr)
 }
 
@@ -191,7 +191,7 @@ func CopyEnvironmentWithTflint(t *testing.T, environmentPath string) string {
 
 	t.Logf("Copying %s to %s", environmentPath, tmpDir)
 
-	require.NoError(t, util.CopyFolderContents(createLogger(), environmentPath, util.JoinPath(tmpDir, environmentPath), ".terragrunt-test", []string{".tflint.hcl"}))
+	require.NoError(t, util.CopyFolderContents(createLogger(), environmentPath, util.JoinPath(tmpDir, environmentPath), ".terragrunt-test", []string{".tflint.hcl"}, []string{}))
 
 	return tmpDir
 }

util/file.go

@@ -239,6 +239,16 @@ func listContainsElementWithPrefix(list []string, elementPrefix string) bool {
 	return false
 }
 
+func pathContainsPrefix(path string, prefixes []string) bool {
+	for _, element := range prefixes {
+		if strings.HasPrefix(path, element) {
+			return true
+		}
+	}
+
+	return false
+}
+
 // Takes apbsolute glob path and returns an array of expanded relative paths
 func expandGlobPath(source, absoluteGlobPath string) ([]string, error) {
 	includeExpandedGlobs := []string{}
@@ -276,7 +286,14 @@ func expandGlobPath(source, absoluteGlobPath string) ([]string, error) {
 
 // CopyFolderContents copies the files and folders within the source folder into the destination folder. Note that hidden files and folders
 // (those starting with a dot) will be skipped. Will create a specified manifest file that contains paths of all copied files.
-func CopyFolderContents(logger log.Logger, source, destination, manifestFile string, includeInCopy []string) error {
+func CopyFolderContents(
+	logger log.Logger,
+	source,
+	destination,
+	manifestFile string,
+	includeInCopy []string,
+	excludeFromCopy []string,
+) error {
 	// Expand all the includeInCopy glob paths, converting the globbed results to relative paths so that they work in
 	// the copy filter.
 	includeExpandedGlobs := []string{}
@@ -292,12 +309,32 @@ func CopyFolderContents(logger log.Logger, source, destination, manifestFile str
 		includeExpandedGlobs = append(includeExpandedGlobs, expandGlob...)
 	}
 
+	excludeExpandedGlobs := []string{}
+
+	for _, excludeGlob := range excludeFromCopy {
+		globPath := filepath.Join(source, excludeGlob)
+
+		expandGlob, err := expandGlobPath(source, globPath)
+		if err != nil {
+			return errors.New(err)
+		}
+
+		excludeExpandedGlobs = append(excludeExpandedGlobs, expandGlob...)
+	}
+
 	return CopyFolderContentsWithFilter(logger, source, destination, manifestFile, func(absolutePath string) bool {
 		relativePath, err := GetPathRelativeTo(absolutePath, source)
-		if err == nil && listContainsElementWithPrefix(includeExpandedGlobs, relativePath) {
+		pathHasPrefix := pathContainsPrefix(relativePath, excludeExpandedGlobs)
+
+		listHasElementWithPrefix := listContainsElementWithPrefix(includeExpandedGlobs, relativePath)
+		if err == nil && listHasElementWithPrefix && !pathHasPrefix {
 			return true
 		}
 
+		if err == nil && pathContainsPrefix(relativePath, excludeExpandedGlobs) {
+			return false
+		}
+
 		return !TerragruntExcludes(filepath.FromSlash(relativePath))
 	})
 }