From 193d603fdbe62803cbe2e09eb6b009677b085b5a Mon Sep 17 00:00:00 2001 From: Peter Booker Date: Tue, 8 Mar 2022 19:53:26 +0000 Subject: [PATCH 1/6] Add new faker fields --- internal/helpers/helpers.go | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/internal/helpers/helpers.go b/internal/helpers/helpers.go index 14a3d0d..eddf27b 100644 --- a/internal/helpers/helpers.go +++ b/internal/helpers/helpers.go @@ -21,8 +21,10 @@ func GetFakerFuncs() map[string]func(*sqlparser.SQLVal) *sqlparser.SQLVal { "phoneNumber": generatePhoneNumber, "addressFull": generateAddress, "addressStreet": generateStreetAddress, + "addressCity": generateCity, "addressPostCode": generatePostcode, "addressCountry": generateCountry, + "addressState": generateState, "paragraph": generateParagraph, "shortString": generateShortString, "ipv4": generateIPv4, @@ -85,6 +87,10 @@ func generateStreetAddress(value *sqlparser.SQLVal) *sqlparser.SQLVal { return sqlparser.NewStrVal([]byte(faker.Address().StreetAddress())) } +func generateCity(value *sqlparser.SQLVal) *sqlparser.SQLVal { + return sqlparser.NewStrVal([]byte(faker.Address().City())) +} + func generatePostcode(value *sqlparser.SQLVal) *sqlparser.SQLVal { return sqlparser.NewStrVal([]byte(faker.Address().Postcode())) } @@ -93,6 +99,10 @@ func generateCountry(value *sqlparser.SQLVal) *sqlparser.SQLVal { return sqlparser.NewStrVal([]byte(faker.Address().Country())) } +func generateState(value *sqlparser.SQLVal) *sqlparser.SQLVal { + return sqlparser.NewStrVal([]byte(faker.Address().State())) +} + func generateCreditCardNumber(value *sqlparser.SQLVal) *sqlparser.SQLVal { return sqlparser.NewStrVal([]byte(faker.Business().CreditCardNumber())) } From 01feb8cc1293d8d6e21d9a171ddb3f807caf0ee7 Mon Sep 17 00:00:00 2001 From: Peter Booker Date: Tue, 8 Mar 2022 19:54:44 +0000 Subject: [PATCH 2/6] Add presets flag and support --- internal/anonymize/anonymize.go | 4 +- internal/config/config.go | 42 ++- internal/embed/embed.go | 8 +- internal/embed/files/config.woocommerce.json | 336 ++++++++++++++++++ .../{config.default.json => default.json} | 0 internal/flag/flag.go | 12 +- 6 files changed, 387 insertions(+), 15 deletions(-) create mode 100644 internal/embed/files/config.woocommerce.json rename internal/embed/files/{config.default.json => default.json} (100%) diff --git a/internal/anonymize/anonymize.go b/internal/anonymize/anonymize.go index 35f1236..9c8a3b2 100644 --- a/internal/anonymize/anonymize.go +++ b/internal/anonymize/anonymize.go @@ -32,10 +32,10 @@ func Start(version, commit, date string) { log.SetOutput(os.Stderr) // Parse flags for custom config file. - configFile := flag.Parse(version, commit, date, config.ProcessName) + configFile, presets := flag.Parse(version, commit, date, config.ProcessName) // Parse config file. - config.ParseConfig(*configFile) + config.ParseConfig(*configFile, presets) // Error if Stdin is the terminal instead of pipe. stat, _ := os.Stdin.Stat() diff --git a/internal/config/config.go b/internal/config/config.go index 9fa923c..c47e881 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -63,11 +63,20 @@ func New(version, commit, date string) *Config { } // ParseConfig parses a default or user provided config file. -func (c *Config) ParseConfig(filepath string) { +func (c *Config) ParseConfig(filepath string, presets []string) { var jsonConfig []byte + var defaultConfig []byte + var jsonParser *json.Decoder + var jsonReader *strings.Reader var err error - jsonConfig = []byte(embed.DefaultConfig) + // Load default config. + defaultConfig, err = embed.Content.ReadFile("files/default.json") + if err != nil { + log.Fatalln("Failed reading default config.") + } + + jsonConfig = []byte(defaultConfig) if filepath != "" { jsonConfig, err = ioutil.ReadFile(filepath) @@ -76,12 +85,31 @@ func (c *Config) ParseConfig(filepath string) { } } - jsonReader := strings.NewReader(string(jsonConfig)) - jsonParser := json.NewDecoder(jsonReader) + jsonReader = strings.NewReader(string(jsonConfig)) + jsonParser = json.NewDecoder(jsonReader) err = jsonParser.Decode(c) - - // Make sure the JSON read is valid. if err != nil { - log.Fatalf("JSON file not valid!") + log.Fatalln("JSON file not valid!") + } + + if len(presets) != 0 { + var presetConfig []byte + for _, preset := range presets { + presetConfig, err = embed.Content.ReadFile("files/config." + preset + ".json") + if err != nil { + log.Fatalf("No preset config found for: %s.", preset) + } + + tempConfig := &Config{} + + jsonReader = strings.NewReader(string(presetConfig)) + jsonParser = json.NewDecoder(jsonReader) + err = jsonParser.Decode(tempConfig) + if err != nil { + log.Fatalln("JSON file not valid!") + } + + c.Patterns = append(c.Patterns, tempConfig.Patterns...) + } } } diff --git a/internal/embed/embed.go b/internal/embed/embed.go index 26de28a..6a3ca34 100644 --- a/internal/embed/embed.go +++ b/internal/embed/embed.go @@ -1,9 +1,9 @@ package embed import ( - _ "embed" + "embed" ) -//go:embed files/config.default.json -// DefaultConfig contains the contents of the default config file. -var DefaultConfig string +//go:embed files/* +// Content contains a filesystem of config files. +var Content embed.FS diff --git a/internal/embed/files/config.woocommerce.json b/internal/embed/files/config.woocommerce.json new file mode 100644 index 0000000..a2b854c --- /dev/null +++ b/internal/embed/files/config.woocommerce.json @@ -0,0 +1,336 @@ +{ + "patterns": [ + { + "tableName": "wp_woocommerce_sessions", + "purge": true, + "fields": [] + }, + { + "tableName": "wp_wc_customer_lookup", + "fields": [ + { + "field": "username", + "position": 3, + "type": "username", + "constraints": null + }, + { + "field": "first_name", + "position": 4, + "type": "firstName", + "constraints": null + }, + { + "field": "last_name", + "position": 5, + "type": "lastName", + "constraints": null + }, + { + "field": "email", + "position": 6, + "type": "email", + "constraints": null + }, + { + "field": "country", + "position": 9, + "type": "addressCountry", + "constraints": null + }, + { + "field": "postcode", + "position": 10, + "type": "addressPostCode", + "constraints": null + }, + { + "field": "city", + "position": 11, + "type": "addressCity", + "constraints": null + }, + { + "field": "state", + "position": 12, + "type": "addressState", + "constraints": null + } + ] + }, + { + "tableName": "wp_postmeta", + "fields": [ + { + "field": "meta_value", + "position": 4, + "type": "ipv4", + "constraints": [ + { + "field": "meta_key", + "position": 3, + "value": "_customer_ip_address" + } + ] + }, + { + "field": "meta_value", + "position": 4, + "type": "firstName", + "constraints": [ + { + "field": "meta_key", + "position": 3, + "value": "_billing_first_name" + } + ] + }, + { + "field": "meta_value", + "position": 4, + "type": "lastName", + "constraints": [ + { + "field": "meta_key", + "position": 3, + "value": "_billing_last_name" + } + ] + }, + { + "field": "meta_value", + "position": 4, + "type": "companyName", + "constraints": [ + { + "field": "meta_key", + "position": 3, + "value": "_billing_company" + } + ] + }, + { + "field": "meta_value", + "position": 4, + "type": "addressStreet", + "constraints": [ + { + "field": "meta_key", + "position": 3, + "value": "_billing_address_1" + } + ] + }, + { + "field": "meta_value", + "position": 4, + "type": "shortString", + "constraints": [ + { + "field": "meta_key", + "position": 3, + "value": "_billing_city" + } + ] + }, + { + "field": "meta_value", + "position": 4, + "type": "shortString", + "constraints": [ + { + "field": "meta_key", + "position": 3, + "value": "_billing_state" + } + ] + }, + { + "field": "meta_value", + "position": 4, + "type": "addressPostCode", + "constraints": [ + { + "field": "meta_key", + "position": 3, + "value": "_billing_postcode" + } + ] + }, + { + "field": "meta_value", + "position": 4, + "type": "addressPostCode", + "constraints": [ + { + "field": "meta_key", + "position": 3, + "value": "_billing_country" + } + ] + }, + { + "field": "meta_value", + "position": 4, + "type": "email", + "constraints": [ + { + "field": "meta_key", + "position": 3, + "value": "_billing_email" + } + ] + }, + { + "field": "meta_value", + "position": 4, + "type": "phoneNumber", + "constraints": [ + { + "field": "meta_key", + "position": 3, + "value": "_billing_phone" + } + ] + }, + { + "field": "meta_value", + "position": 4, + "type": "shortString", + "constraints": [ + { + "field": "meta_key", + "position": 3, + "value": "_billing_address_index" + } + ] + } + ] + }, + { + "tableName": "wp_usermeta", + "fields": [ + { + "field": "meta_value", + "position": 4, + "type": "firstName", + "constraints": [ + { + "field": "meta_key", + "position": 3, + "value": "billing_first_name" + } + ] + }, + { + "field": "meta_value", + "position": 4, + "type": "lastName", + "constraints": [ + { + "field": "meta_key", + "position": 3, + "value": "billing_last_name" + } + ] + }, + { + "field": "meta_value", + "position": 4, + "type": "companyName", + "constraints": [ + { + "field": "meta_key", + "position": 3, + "value": "billing_company" + } + ] + }, + { + "field": "meta_value", + "position": 4, + "type": "addressStreet", + "constraints": [ + { + "field": "meta_key", + "position": 3, + "value": "billing_address_1" + } + ] + }, + { + "field": "meta_value", + "position": 4, + "type": "shortString", + "constraints": [ + { + "field": "meta_key", + "position": 3, + "value": "billing_city" + } + ] + }, + { + "field": "meta_value", + "position": 4, + "type": "shortString", + "constraints": [ + { + "field": "meta_key", + "position": 3, + "value": "billing_state" + } + ] + }, + { + "field": "meta_value", + "position": 4, + "type": "addressPostCode", + "constraints": [ + { + "field": "meta_key", + "position": 3, + "value": "billing_postcode" + } + ] + }, + { + "field": "meta_value", + "position": 4, + "type": "addressPostCode", + "constraints": [ + { + "field": "meta_key", + "position": 3, + "value": "billing_country" + } + ] + }, + { + "field": "meta_value", + "position": 4, + "type": "email", + "constraints": [ + { + "field": "meta_key", + "position": 3, + "value": "billing_email" + } + ] + }, + { + "field": "meta_value", + "position": 4, + "type": "phoneNumber", + "constraints": [ + { + "field": "meta_key", + "position": 3, + "value": "billing_phone" + } + ] + } + ] + } + ] +} \ No newline at end of file diff --git a/internal/embed/files/config.default.json b/internal/embed/files/default.json similarity index 100% rename from internal/embed/files/config.default.json rename to internal/embed/files/default.json diff --git a/internal/flag/flag.go b/internal/flag/flag.go index bd966b9..ba35545 100644 --- a/internal/flag/flag.go +++ b/internal/flag/flag.go @@ -3,11 +3,12 @@ package flag import ( "fmt" "os" + "strings" "github.com/spf13/pflag" ) -func Parse(version, commit, date, processName string) *string { +func Parse(version, commit, date, processName string) (*string, []string) { helpText := `Anonymize MySQLDump is a database anonymization tool. Version: ` + version + ` Commit: ` + commit + ` @@ -24,6 +25,7 @@ Config: The anonymizer will use a default config suitable for WordPress, but you can override this by providing your own.` flagConfigFile := pflag.String("config", "", "Path to config file.") + flagPresets := pflag.String("presets", "", "Preset configs to use.") flagHelp := pflag.BoolP("help", "h", false, "") pflag.Parse() @@ -33,5 +35,11 @@ Config: os.Exit(1) } - return flagConfigFile + var presets []string + + if *flagPresets != "" { + presets = strings.Split(*flagPresets, ",") + } + + return flagConfigFile, presets } From 539642df766bc2bf5641c7fcb8ddde37a88b3ee3 Mon Sep 17 00:00:00 2001 From: Peter Booker Date: Tue, 8 Mar 2022 19:58:08 +0000 Subject: [PATCH 3/6] Add regex to table names --- internal/embed/files/config.woocommerce.json | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/internal/embed/files/config.woocommerce.json b/internal/embed/files/config.woocommerce.json index a2b854c..9735241 100644 --- a/internal/embed/files/config.woocommerce.json +++ b/internal/embed/files/config.woocommerce.json @@ -1,12 +1,12 @@ { "patterns": [ { - "tableName": "wp_woocommerce_sessions", + "tableName": ".*_woocommerce_sessions", "purge": true, "fields": [] }, { - "tableName": "wp_wc_customer_lookup", + "tableName": ".*_wc_customer_lookup", "fields": [ { "field": "username", @@ -59,7 +59,7 @@ ] }, { - "tableName": "wp_postmeta", + "tableName": ".*_postmeta", "fields": [ { "field": "meta_value", @@ -208,7 +208,7 @@ ] }, { - "tableName": "wp_usermeta", + "tableName": ".*_usermeta", "fields": [ { "field": "meta_value", From 04177e70e3360cc66ce7930b324e9088e9fb59bc Mon Sep 17 00:00:00 2001 From: Peter Booker Date: Tue, 8 Mar 2022 20:00:51 +0000 Subject: [PATCH 4/6] Update test --- internal/anonymize/anonymize_test.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/internal/anonymize/anonymize_test.go b/internal/anonymize/anonymize_test.go index 8e67365..4b80d2f 100644 --- a/internal/anonymize/anonymize_test.go +++ b/internal/anonymize/anonymize_test.go @@ -45,7 +45,8 @@ func init() { faker.Seed(432) jsonConfig = *config.New("", "", "") - jsonConfig.ParseConfig("") + presets := make([]string, 0) + jsonConfig.ParseConfig("", presets) // Get map of faker helper functions. transformationFunctionMap = helpers.GetFakerFuncs() From 1b757752af50f36dd28d887bd788732b3ac0f3ed Mon Sep 17 00:00:00 2001 From: Peter Booker Date: Sat, 12 Mar 2022 15:26:54 +0000 Subject: [PATCH 5/6] Add GiveWP preset --- internal/embed/files/config.give.json | 108 ++++++++++++++++++++++++++ 1 file changed, 108 insertions(+) create mode 100644 internal/embed/files/config.give.json diff --git a/internal/embed/files/config.give.json b/internal/embed/files/config.give.json new file mode 100644 index 0000000..19ba91b --- /dev/null +++ b/internal/embed/files/config.give.json @@ -0,0 +1,108 @@ +{ + "patterns": [ + { + "tableName": ".*_give_donationmeta", + "fields": [ + { + "field": "meta_value", + "position": 4, + "type": "firstName", + "constraints": [ + { + "field": "meta_key", + "position": 3, + "value": "_give_donor_billing_first_name" + } + ] + }, + { + "field": "meta_value", + "position": 4, + "type": "lastName", + "constraints": [ + { + "field": "meta_key", + "position": 3, + "value": "_give_donor_billing_last_name" + } + ] + }, + { + "field": "meta_value", + "position": 4, + "type": "email", + "constraints": [ + { + "field": "meta_key", + "position": 3, + "value": "_give_payment_donor_email" + } + ] + }, + { + "field": "meta_value", + "position": 4, + "type": "ipv4", + "constraints": [ + { + "field": "meta_key", + "position": 3, + "value": "_give_payment_donor_ip" + } + ] + } + ] + }, + { + "tableName": ".*_give_donormeta", + "fields": [ + { + "field": "meta_value", + "position": 4, + "type": "firstName", + "constraints": [ + { + "field": "meta_key", + "position": 3, + "value": "_give_donor_first_name" + } + ] + }, + { + "field": "meta_value", + "position": 4, + "type": "lastName", + "constraints": [ + { + "field": "meta_key", + "position": 3, + "value": "_give_donor_last_name" + } + ] + } + ] + }, + { + "tableName": ".*_give_donors", + "fields": [ + { + "field": "email", + "position": 3, + "type": "email", + "constraints": null + }, + { + "field": "name", + "position": 4, + "type": "name", + "constraints": null + } + ] + }, + { + "tableName": ".*_give_sessions", + "purge": true, + "fields": [] + } + ] +} \ No newline at end of file From 829561f0b6ccf4b9f8a4519cd3abbc0e9a4d69c1 Mon Sep 17 00:00:00 2001 From: Peter Booker Date: Tue, 15 Mar 2022 21:02:41 +0000 Subject: [PATCH 6/6] Add missing give_donationmeta fields --- internal/embed/files/config.give.json | 96 +++++++++++++++++++++++++++ 1 file changed, 96 insertions(+) diff --git a/internal/embed/files/config.give.json b/internal/embed/files/config.give.json index 19ba91b..c156d71 100644 --- a/internal/embed/files/config.give.json +++ b/internal/embed/files/config.give.json @@ -50,6 +50,102 @@ "value": "_give_payment_donor_ip" } ] + }, + { + "field": "meta_value", + "position": 4, + "type": "addressStreet", + "constraints": [ + { + "field": "meta_key", + "position": 3, + "value": "_give_donor_billing_address1" + } + ] + }, + { + "field": "meta_value", + "position": 4, + "type": "addressStreet", + "constraints": [ + { + "field": "meta_key", + "position": 3, + "value": "_give_donor_billing_address2" + } + ] + }, + { + "field": "meta_value", + "position": 4, + "type": "addressCity", + "constraints": [ + { + "field": "meta_key", + "position": 3, + "value": "_give_donor_billing_city" + } + ] + }, + { + "field": "meta_value", + "position": 4, + "type": "addressPostCode", + "constraints": [ + { + "field": "meta_key", + "position": 3, + "value": "_give_donor_billing_zip" + } + ] + }, + { + "field": "meta_value", + "position": 4, + "type": "addressState", + "constraints": [ + { + "field": "meta_key", + "position": 3, + "value": "_give_donor_billing_state" + } + ] + }, + { + "field": "meta_value", + "position": 4, + "type": "addressCountry", + "constraints": [ + { + "field": "meta_key", + "position": 3, + "value": "_give_donor_billing_country" + } + ] + }, + { + "field": "meta_value", + "position": 4, + "type": "companyName", + "constraints": [ + { + "field": "meta_key", + "position": 3, + "value": "_give_donation_company" + } + ] + }, + { + "field": "meta_value", + "position": 4, + "type": "shortString", + "constraints": [ + { + "field": "meta_key", + "position": 3, + "value": "_give_payment_transaction_id" + } + ] } ] },