feat: add of multiline replacer

go.mod

@@ -1,3 +1,5 @@
 module github.com/cloudposse/tfmask
 
 go 1.13
+
+require gopkg.in/yaml.v3 v3.0.1

go.sum

@@ -0,0 +1,4 @@
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
+gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

main.go

@@ -2,13 +2,18 @@ package main
 
 import (
 	"bufio"
+	"bytes"
 	"fmt"
+	"io"
+	"io/ioutil"
 	"log"
 	"os"
 	"regexp"
 	"runtime"
 	"strings"
 	"unicode/utf8"
+
+	"gopkg.in/yaml.v3"
 )
 
 type match struct {
@@ -93,39 +98,69 @@ var versionedExpressions = map[string]expression{
 	},
 }
 
-func main() {
-	log.SetFlags(0) // no timestamps on our logs
+type multiLineRule struct {
+	BeginLineReg string `yaml:"begin"`
+	EndLineReg   string `yaml:"end"`
+	ValReg       string `yaml:"value"`
+}
+
+type config struct {
+	maskChar       string
+	valuesRegex    string
+	resourceRegex  string
+	env            string
+	multilineRules []multiLineRule
+}
+
+func parseRules(data []byte) ([]multiLineRule, error) {
+	var rules = struct {
+		Rules []multiLineRule `yaml:"rules"`
+	}{}
+	err := yaml.Unmarshal(data, &rules)
+	if err != nil {
+		return nil, fmt.Errorf("failed to parse yaml: %w", err)
+	}
+
+	return rules.Rules, nil
+}
 
+func getConfig() config {
+	cfg := config{}
 	// Character used to mask sensitive output
-	var tfmaskChar = getEnv("TFMASK_CHAR", "*")
+	cfg.maskChar = getEnv("TFMASK_CHAR", "*")
 	// Pattern representing sensitive output
-	var tfmaskValuesRegex = getEnv("TFMASK_VALUES_REGEX",
+	cfg.valuesRegex = getEnv("TFMASK_VALUES_REGEX",
 		"(?i)^.*[^a-zA-Z](oauth|secret|token|password|key|result|id).*$")
 	// Pattern representing sensitive resource
-	var tfmaskResourceRegex = getEnv("TFMASK_RESOURCES_REGEX",
+	cfg.resourceRegex = getEnv("TFMASK_RESOURCES_REGEX",
 		"(?i)^(random_id|random_string).*$")
 
-	// Default to tf 0.12, but users can override
-	var tfenv = getEnv("TFENV", "0.12")
-
-	reTfValues := regexp.MustCompile(tfmaskValuesRegex)
-	reTfResource := regexp.MustCompile(tfmaskResourceRegex)
-	scanner := bufio.NewScanner(os.Stdin)
-	versionedExpressions := versionedExpressions[tfenv]
-	// initialize currentResource once before scanning
-	currentResource := ""
-	for scanner.Scan() {
-		line := scanner.Text()
-		currentResource = getCurrentResource(versionedExpressions,
-			currentResource, line)
-		fmt.Println(processLine(versionedExpressions, reTfResource, reTfValues,
-			tfmaskChar, currentResource, line))
+	cfg.multilineRules = []multiLineRule{
+		{"BEGIN", "END", "[\\S]{1}"},
 	}
 
-	if err := scanner.Err(); err != nil {
-		fmt.Fprintln(os.Stderr, "error:", err)
-		os.Exit(1)
+	cfgPath := getEnv("TFMASK_RULES_PATH", "")
+	if _, err := os.Stat(cfgPath); err == nil {
+		data, err := ioutil.ReadFile(cfgPath)
+		if err != nil {
+			fmt.Printf("failed to read rules file: %v", err)
+		}
+		rules, err := parseRules(data)
+		if err != nil {
+			fmt.Printf("failed to parse rules: %v", err)
+		}
+		cfg.multilineRules = rules
 	}
+
+	// Default to tf 0.12, but users can override
+	cfg.env = getEnv("TFENV", "0.12")
+	return cfg
+}
+
+func main() {
+	log.SetFlags(0) // no timestamps on our logs
+
+	processFile(getConfig(), os.Stdin, os.Stdout, os.Stderr)
 }
 
 func getCurrentResource(expression expression, currentResource, line string) string {
@@ -144,6 +179,75 @@ func getCurrentResource(expression expression, currentResource, line string) str
 	return currentResource
 }
 
+func processByLine(cfg config, in io.Reader, out io.Writer, er io.Writer) {
+	reTfValues := regexp.MustCompile(cfg.valuesRegex)
+	reTfResource := regexp.MustCompile(cfg.resourceRegex)
+	scanner := bufio.NewScanner(in)
+	versionedExpressions := versionedExpressions[cfg.env]
+	// initialize currentResource once before scanning
+	currentResource := ""
+	for scanner.Scan() {
+		line := scanner.Text()
+		currentResource = getCurrentResource(versionedExpressions,
+			currentResource, line)
+		fmt.Fprintln(out, processLine(versionedExpressions, reTfResource, reTfValues,
+			cfg.maskChar, currentResource, line))
+	}
+
+	if err := scanner.Err(); err != nil {
+		fmt.Fprintln(er, "error:", err)
+		os.Exit(1)
+	}
+}
+
+func processFile(cfg config, in io.Reader, out io.Writer, er io.Writer) {
+	tempOut := &bytes.Buffer{}
+	processByLine(cfg, in, tempOut, er)
+
+	for _, rule := range cfg.multilineRules {
+		beginRegex := regexp.MustCompile(rule.BeginLineReg)
+		endRegex := regexp.MustCompile(rule.EndLineReg)
+		valRegex := regexp.MustCompile(rule.ValReg)
+
+		ruleOut := &bytes.Buffer{}
+		reader := bufio.NewReader(tempOut)
+		replacingMode := false
+		for {
+
+			line, _, err := reader.ReadLine()
+			if err == io.EOF {
+				break
+			}
+
+			if err != nil {
+				fmt.Fprintln(er, "error:", err)
+				os.Exit(1)
+			}
+
+			if replacingMode {
+				if endRegex.Match(line) {
+					fmt.Fprintf(ruleOut, "%s\n", line)
+
+					replacingMode = false
+					continue
+				}
+				newLine := valRegex.ReplaceAll(line, []byte(cfg.maskChar))
+				fmt.Fprintf(ruleOut, "%s\n", newLine)
+
+			} else {
+				fmt.Fprintf(ruleOut, "%s\n", line)
+
+				if beginRegex.Match(line) {
+					replacingMode = true
+					continue
+				}
+			}
+		}
+		tempOut = ruleOut
+	}
+	io.Copy(out, tempOut)
+}
+
 func processLine(expression expression, reTfResource,
 	reTfValues *regexp.Regexp, tfmaskChar, currentResource,
 	line string) string {

main_test.go

@@ -1,6 +1,12 @@
 package main
 
 import (
+	"bufio"
+	"bytes"
+	"io"
+	"io/ioutil"
+	"os"
+	"path/filepath"
 	"regexp"
 	"strings"
 	"testing"
@@ -75,6 +81,95 @@ func TestProcessLine(t *testing.T) {
 	}
 }
 
+func TestRulesParsing(t *testing.T) {
+	rulesStr := `
+rules:
+- begin: "BEGIN"
+  end: "END"
+  value: "[\\S]{1}"
+`
+	rules, err := parseRules([]byte(rulesStr))
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if len(rules) != 1 {
+		t.Fatalf("expected 1 rules found %v", len(rules))
+	}
+
+	exp := multiLineRule{
+		BeginLineReg: "BEGIN",
+		EndLineReg:   "END",
+		ValReg:       "[\\S]{1}",
+	}
+
+	if rules[0] != exp {
+		t.Fatalf("expected %v but received %v", exp, rules[0])
+	}
+}
+
+func TestProcessFile(t *testing.T) {
+	fs, err := ioutil.ReadDir("tests/cases")
+	if err != nil {
+		t.Fatal(err)
+	}
+	for _, f := range fs {
+		if f.IsDir() {
+			continue
+		}
+		filename := f.Name()
+
+		if filepath.Ext(filename) == ".in" {
+			outFname := strings.ReplaceAll(filename, ".in", ".out")
+
+			in, err := os.Open(filepath.Join("tests/cases", filename))
+			if err != nil {
+				t.Fatal(err)
+			}
+			outCheck, err := os.Open(filepath.Join("tests/cases", outFname))
+			if err != nil {
+				t.Fatal(err)
+			}
+			out := &bytes.Buffer{}
+			cfg := getConfig()
+			cfg.valuesRegex = "(?i)^.*[^a-zA-Z](oauth|secret|certificate|token|password|key|result).*$"
+			processFile(cfg, in, out, os.Stderr)
+
+			if err != nil {
+				t.Fatal(err)
+			}
+
+			expResultBuffer := bufio.NewReader(outCheck)
+			resultBuffer := bufio.NewReader(out)
+			for {
+				line, _, err1 := resultBuffer.ReadLine()
+				expLine, _, err2 := expResultBuffer.ReadLine()
+				if err1 == err2 && err1 == io.EOF {
+					break
+				}
+				if err != nil {
+					t.Fatal(err1)
+				}
+				if err != nil {
+					t.Fatal(err2)
+				}
+				lineStr := string(line)
+				expLineStr := string(expLine)
+
+				if lineStr != expLineStr {
+					t.Errorf("test %s failed", filename)
+					t.Error("expected line:")
+					t.Error(expLineStr)
+					t.Error("but received:")
+					t.Error(lineStr)
+				}
+
+			}
+
+		}
+	}
+}
+
 var currentResourceTests = []struct {
 	currentResource string
 	line            string

tests/cases/1.in

@@ -0,0 +1,29 @@
+  # module.k8s_clusters.vault_kubernetes_auth_backend_config.k8s_clusters["dev"] will be created
++ resource "vault_kubernetes_auth_backend_config" "k8s_clusters" {
+    + backend                = (known after apply)
+    + disable_iss_validation = true
+    + disable_local_ca_jwt   = (known after apply)
+    + id                     = (known after apply)
+    + issuer                 = "api"
+    + kubernetes_ca_cert     = <<-EOT
+          -----BEGIN CERTIFICATE-----
+            9b0194129108e8f860f4a58149b0194129108e8f860f4a5814w0BAQsFADAVMRM
+            cm5ldGVzMB4XDTIxMDgwOTE4NTYxM1oXDTMxMDgwNzE4NTYxM1owFTETMBEGA1UE
+            AxMKa3ViZXJuZXRlczCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAKuv
+            xz/WXGZtgnXx73S+n/C9b0194129108e8f860f4a5814RlJjY9u5uDqok1qWOyTM
+            sRsq4N45l69abnvmJZnlfHpqcQE09d+xi4qe5vc2mODDfL/ZVwczav0uGjyACEOa
+            9r88lpxJNIqyOrhbH6Dg3g53FgMefcOe88HC4WmRwppoIS7n7mXaUdzoxNtJRT2P
+            TYcjTbuJDj5xXJFFmGrrbgXWAj7pWl95zD0PIhhNA1XR4byvnPbrXWADgJH2wXZx
+            /fX9XR5BJsL8QKsmM4nibyxpLScu7i6FsrGJC4F/V+D7QGaTMmCtci0LBErUpQ1A
+            bdBqjFsf/oYd2x9b0194129108e8f860f4a5814H/BA9b0194129108GA1UdEwEB
+            /wQFMAMBAf8wHQYDVR0OBBYEFHfMZWhbqNN9wqeLVIIuGCMpaEUWMA0GCSqGSIb3
+            DQEBCwUAA4IBAQCpiJ9I7skg6psY3Wo8gyLPc0tDhpHs6+L7p+fBJMREqlJ1vgbX
+            4Pm2m770xXJSTvMbl45Y/RNT3B4CvNCqNt7mULQ9AkLmqSQaK28LRxsk+ZYaMwkY
+            oRKxNhHbpuwp/WaHayAwnh3eGmL+VJJAKXhNPUO3vc9b0194129108e8f860f4a5
+            KPnhFdWBL34hHd1VajGemqonDSn5zj8LyI4V944yeBuRt6g3j6MlvLmpW/NnTtFQ
+            22kVhlkD5kjBtkIl1dReg+OHtTZ77tHAmknxwo8m6luxiZsMX/LfjlfAfOJWwQKL
+            LD4U2hwN1bY36h5C3Djc9mXdc0Q1ghjK96vZ
+          -----END CERTIFICATE-----
+        EOT
+    + kubernetes_host        = "https://1.1.1.1"
+    }

tests/cases/2.in

@@ -0,0 +1,9 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIIBOgIBAAJBAKj34GkxFhD90vcNLYLInFEX6Ppy1tPf9Cnzj4p4WGeKLs1Pt8Qu
+KUpRKfFLfRYC9AIKjbJTWit+CqvjWYzvQwECAwEAAQJAIJLixBy2qpFoS4DSmoEm
+o3qGy0t6z09AIJtH+5OeRV1be+N4cDYJKffGzDa88vQENZiRm0GRq6a+HPGQMd2k
+TQIhAKMSvzIBnni7ot/OSie2TmJLY4SwTQAevXysE2RbFDYdAiEBCUEaRQnMnbp7
+9mxDXDf6AU0cN/RPBjb9qSHDcWZHGzUCIG2Es59z8ugGrDY+pxLQnwfotadxd+Uy
+v/Ow5T0q5gIJAiEAyS4RaI9YG8EWx/2w0T67ZUVAw8eOMB6BIUg0Xcu+3okCIBOs
+/5OiPgoTdSy7bcF9IGpSE8ZgGKzgYQVZeN97YE00
+-----END RSA PRIVATE KEY-----