From 4266c88b3943aa4521c28316aa9709293e8e86bb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ji=C5=99=C3=AD=20=C4=8Ctvrtka?= <62988319+JiriCtvrtka@users.noreply.github.com> Date: Thu, 7 Nov 2024 10:29:48 +0100 Subject: [PATCH] PMM-13132 Encryption rotation. (#3199) * PMM-13129 Tidy. * PMM-13129 Migration basics. * PMM-13129 Format. * PMM-13129 Encrypt, EncryptDB, Decrypt, DecryptDB, refactor. * PMM-13129 Encryption test workflow. * PMM-13129 Remove install. * PMM-13129 Encrypt/Decrypt agents. * PMM-13129 Changes. * PMM-13145 Fix for tests. * PMM-13129 Fix Mongo test. * PMM-13129 Fix. * PMM-13129 Encrypt fixture. * PMM-13129 Encryption test. * PMM-13129 File mode test. * PMM-13129 Fix credentials for test env. * PMM-13129 Clean. * PMM-13129 Correct DB for encryption test. * PMM-13129 Moved to utils folder. * PMM-13129 Empty password fix. * PMM-13129 Debug logs to warning level. * PMM-13129 Format. * PMM-13129 Small change in generated query. * PMM-13129 Password set check. * PMM-13129 Fix wrong field. * PMM-13129 Init in migration. * PMM-13129 Precheck if already encrypted, moved into managed utils. * PMM-13129 Migration. * PMM-13129 Fix for EncryptDB. Encrypt/Decrypt username. * PMM-13129 Formatting of encryption error, createAgent username fix. * PMM-13129 Remove unused method for now. * PMM-13129 Correct mode for cert file. * PMM-13129 Remove DB test, small refactor. * PMM-13129 Encryption for external exporter. * PMM-13129 Fix tests after external exporter encryption. * PMM-13129 Fix mongo tests. * PMM-13129 Fix another test to expect encrypted username. * PMM-13129 Another fix for tests to expect encrypted username. * PMM-13129 Fix for DecryptDB. * PMM-13129 Err if encryption is not initialized. * PMM-13129 Delimiter fix. * PMM-13129 Fix DecryptDB. * PMM-13129 Small change in agent test. * PMM-13129 Fix non related test to make it green for now. * PMM-13129 Add license headers. * PMM-13129 License. * PMM-13129 Lint. * PMM-13129 Another lint. * PMM-13129 Lint. * PMM-13129 Default encryption changes. * PMM-13129 Encrypt, decrypt all other secret, credentials in agents. * PMM-13129 Changes, some refactors. * PMM-13129 Another changes. * PMM-13129 Refactor. * PMM-13129 Fix. * PMM-13129 Changes. * PMM-13129 Changes. * PMM-13129 Save. * PMM-13129 Changes. * PMM-13129 Another changes. * PMM-13129 Refactor, another changes. * PMM-13129 Disable migration encryption until it is done. * PMM-13129 Basics for settings and migration. * PMM-13129 Original code for isPasswordSet. * PMM-13129 Fix current settings test. * PMM-13129 Basic changes to be able pass custom handlers. * PMM-13129 Handlers, PG handler. * PMM-13129 Refactor. * PMM-13129 Changes, refactor. * PMM-13129 Migrate and encrypt all possible fields. * PMM-13129 Fix for service info broker. * PMM-13129 Fix for settings helper test. * PMM-13129 Refactor. * PMM-13129 Lint. * PMM-13129 Lint. * PMM-13129 Format. * PMM-13129 Fix settings helpers test. * PMM-13129 License header. * PMM-13129 Another lint. * PMM-13129 Lint. * PMM-13129 Changes to fix tests. Refactor. * PMM-13129 Format. * PMM-13129 Fix. * PMM-13129 Encrypt items now receive opened DB connection, refactor. * PMM-13129 Lint (correct ctx). * PMM-13129 Refactor, lint. * PMM-13129 Check. * PMM-13129 Lint. * PMM-13129 Fix settings test. * PMM-13129 Fix to prevent double encryption on setup fixtures. * PMM-13129 Changes. * PMM-13129 Encrypt only basic fields in tests (migration). * PMM-13129 Test. * PMM-13129 Lint. * PMM-13129 Different encrypted columns for different migration versions. * PMM-13129 Fix. * PMM-13129 TODO. * PMM-13129 TODO. * PMM-13129 Check for nothing to encrypt. * PMM-13129 Encrypted fields based on migration version. * PMM-13129 Better debug. * PMM-13129 Lint. * PMM-13129 Fix, better debug. * PMM-13129 Exit in case of encryption initialization error. * PMM-13129 Handle nil migration version. * PMM-13129 Typo. * PMM-13129 Fix for service broker and connection check. * PMM-13129 Comments. * PMM-13129 Remove debug logging. * PMM-13129 Remove pointer in EncryptAgent, DecryptAgent. * PMM-13129 Fix. * PMM-13129 Fix for service_info_broker. * PMM-13129 Fix service_info_broker options pointer propagation. * PMM-13129 Fix for custom labels after removed pointer. * PMM-13129 Hide cipherText in error message. * PMM-13129 Panic in case of unavailable encryption. * PMM-13129 Remove CA certificates from encryption/decryption. * PMM-13129 Required refactor. * Update api/serverpb/server.proto Co-authored-by: Alex Demidoff * Update managed/models/database.go Co-authored-by: Alex Demidoff * Update managed/utils/encryption/encryption.go Co-authored-by: Alex Demidoff * Update managed/utils/encryption/models.go Co-authored-by: Alex Demidoff * Update managed/utils/encryption/models.go Co-authored-by: Alex Demidoff * Update managed/utils/encryption/helpers.go Co-authored-by: Alex Demidoff * PMM-13129 Gen. * PMM-13129 Identifiers word. * PMM-13129 Remove CAs from handlers. * Update managed/models/settings.go Co-authored-by: Alex Demidoff * Update managed/utils/encryption/encryption.go Co-authored-by: Alex Demidoff * Update managed/utils/encryption/encryption.go Co-authored-by: Alex Demidoff * PMM-13129 Dereference all DB options on encrypt/decrypt. * PMM-13129 Custom labels. * Revert "PMM-13129 Custom labels." This reverts commit 903b4ef4aa0ff84edde087ebf60d80b42ee62b1a. * Revert "PMM-13129 Dereference all DB options on encrypt/decrypt." This reverts commit fe3be3149b2d75924940176c725995d5d467f2d2. * Reapply "PMM-13129 Custom labels." This reverts commit 9fd89821fbef0d233674b20dbcde913f98027c8c. * Reapply "PMM-13129 Dereference all DB options on encrypt/decrypt." This reverts commit f9550405f8c4084e18615897ae1c4b39c52b682a. * PMM-13129 Remove old migrations tests, required refactor. * Revert "Reapply "PMM-13129 Custom labels."" This reverts commit 687a2e248496864a722156ead6be1adee98d89eb. * Revert "Reapply "PMM-13129 Dereference all DB options on encrypt/decrypt."" This reverts commit f09bef1207301238346e17ac96e454def8dab6d2. * PMM-13129 Logic change. * PMM-13129 Remove username, aws_access_key, aws_secret_key from enc. * PMM-13129 Env variable for custom encryption key. * PMM-13129 Custom key for main check. * PMM-13129 Remove decrypt agent from create agent methods. * PMM-13129 Change to skip empty values from encryption. * PMM-13129 Remove unused struct. * Update managed/models/database.go Co-authored-by: Nurlan Moldomurov * PMM-13129 Renaming of variable. * PMM-13129 Remove EncryptedItems field from settings proto. * PMM-13129 Workaround to create FB for now. Will be reverted. * PMM-13129 Fix connection checker dsn bug. * PMM-13129 Another dsn bug. * PMM-13129 Add back decrypt after insert to fix connection checker. * PMM-13129 Update reduct words. * PMM-13129 Fix for test after new redact word. * PMM-13132 Basics. * PMM-13132 Some changes. * PMM-13132 Make format. * PMM-13132 Mod fix, tidy. * PMM-13132 Fix. * PMM-13132 Changes. * PMM-13132 Changes. * PMM-13132 Rotation. * PMM-13132 Format. * PMM-13132 Changes. * PMM-13132 Fix. * PMM-13132 Backup and restore of previous key. * PMM-13132 Changes. * PMM-13132 Lint. * PMM-13132 Correct message. * PMM-13132 Changes related to tests. * PMM-13132 Test for whole cycle. * PMM-13132 Handle OS interuptions. * PMM-13132 Lint. * PMM-13132 Lint. * PMM-13132 Logger and logs. * PMM-13132 Test DB. * Revert "PMM-13132 Test DB." This reverts commit e8f94bf32a85fee0e987b224212f6121aacd24e5. * PMM-13132 Changes, CI. * PMM-13132 Fix in test. * PMM-13132 Changes. * PMM-13132 Skip encryption-rotation test in main test. * PMM-13132 Basic makefile for encryption-rotation. * PMM-13132 Remove duplicate defaults. * PMM-13132 Changes in workflow. * PMM-13132 Remove devcontainer from makefile. * PMM-13132 Add ENV variable for rotation key. * PMM-13132 Add PG. * PMM-13132 Remove user, pass in PG compose. * PMM-13132 Test of user. * PMM-13132 Change path for test. * PMM-13132 Test of simpler structure. * PMM-13132 Another changes in structure. * PMM-13132 Another changes to simplify rotation. * PMM-13132 Format. * PMM-13132 Improvements. * PMM-13132 Add command to makefile, lint. * PMM-13132 Lint. * PMM-13132 Lint. * PMM-13132 Wrappers around default on newly added methods. * PMM-13132 Move into cmd of pmm-managed. * PMM-13132 Suggested refactor. * PMM-13132 Another suggested refactor. * PMM-13132 Fix. * PMM-13132 Move encryption models into encryption file. * PMM-13132 Make. * PMM-13132 Migration to kong. * PMM-13132 Add consts. * PMM-13132 Lint. * PMM-13132 Specs. * PMM-13132 Move encryption rotation to services. * PMM-13132 Lint. * PMM-13132 Add interval and retries. * PMM-13132 Fix after conflicts. --------- Co-authored-by: Alex Demidoff Co-authored-by: Nurlan Moldomurov --- Makefile | 3 + .../rpm/server/SPECS/pmm-managed.spec | 5 + go.mod | 1 + go.sum | 2 + managed/Makefile | 3 + managed/cmd/pmm-encryption-rotation/main.go | 95 +++++++++++ managed/models/database.go | 91 +++++----- managed/models/settings_helpers.go | 3 +- .../encryption/encryption_rotation.go | 158 ++++++++++++++++++ .../encryption/encryption_rotation_test.go | 142 ++++++++++++++++ managed/utils/encryption/encryption.go | 120 +++++++++++-- managed/utils/encryption/helpers.go | 50 +----- managed/utils/encryption/models.go | 45 ----- 13 files changed, 582 insertions(+), 136 deletions(-) create mode 100644 managed/cmd/pmm-encryption-rotation/main.go create mode 100644 managed/services/encryption/encryption_rotation.go create mode 100644 managed/services/encryption/encryption_rotation_test.go delete mode 100644 managed/utils/encryption/models.go diff --git a/Makefile b/Makefile index e0f76fb515..6dbf9abe63 100644 --- a/Makefile +++ b/Makefile @@ -38,3 +38,6 @@ TARGET ?= _bash env: ## Run `make TARGET` in devcontainer (`make env TARGET=help`); TARGET defaults to bash COMPOSE_PROFILES=$(PROFILES) \ docker exec -it --workdir=/root/go/src/github.com/percona/pmm pmm-server make $(TARGET) + +rotate-encryption: ## Rotate encryption key + go run ./encryption-rotation/main.go diff --git a/build/packages/rpm/server/SPECS/pmm-managed.spec b/build/packages/rpm/server/SPECS/pmm-managed.spec index b1da81ea78..519a4ffc1c 100644 --- a/build/packages/rpm/server/SPECS/pmm-managed.spec +++ b/build/packages/rpm/server/SPECS/pmm-managed.spec @@ -51,6 +51,7 @@ install -d -p %{buildroot}%{_sbindir} install -d -p %{buildroot}%{_datadir}/%{name} install -d -p %{buildroot}%{_datadir}/pmm-ui install -p -m 0755 bin/pmm-managed %{buildroot}%{_sbindir}/pmm-managed +install -p -m 0755 bin/pmm-encryption-rotation %{buildroot}%{_sbindir}/pmm-encryption-rotation install -p -m 0755 bin/pmm-managed-init %{buildroot}%{_sbindir}/pmm-managed-init install -p -m 0755 bin/pmm-managed-starlark %{buildroot}%{_sbindir}/pmm-managed-starlark @@ -62,12 +63,16 @@ cp -pa ./ui/dist/. %{buildroot}%{_datadir}/pmm-ui %license src/%{provider}/LICENSE %doc src/%{provider}/README.md %{_sbindir}/pmm-managed +%{_sbindir}/pmm-encryption-rotation %{_sbindir}/pmm-managed-init %{_sbindir}/pmm-managed-starlark %{_datadir}/%{name} %{_datadir}/pmm-ui %changelog +* Mon Sep 23 2024 Jiri Ctvrtka - 3.0.0-1 +- PMM-13132 add PMM encryption rotation tool + * Fri Mar 22 2024 Matej Kubinec - 3.0.0-1 - PMM-11231 add pmm ui diff --git a/go.mod b/go.mod index 3c4a60cf81..8b56f60bed 100644 --- a/go.mod +++ b/go.mod @@ -67,6 +67,7 @@ require ( github.com/sirupsen/logrus v1.9.3 github.com/stretchr/objx v0.5.2 github.com/stretchr/testify v1.9.0 + github.com/tink-crypto/tink-go v0.0.0-20230613075026-d6de17e3f164 go.mongodb.org/mongo-driver v1.17.1 go.starlark.net v0.0.0-20230717150657-8a3343210976 golang.org/x/crypto v0.28.0 diff --git a/go.sum b/go.sum index 566408577f..54fe9f45af 100644 --- a/go.sum +++ b/go.sum @@ -505,6 +505,8 @@ github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= +github.com/tink-crypto/tink-go v0.0.0-20230613075026-d6de17e3f164 h1:yhVO0Yhq84FjdcotvFFvDJRNHJ7mO743G12VdcW4Evc= +github.com/tink-crypto/tink-go v0.0.0-20230613075026-d6de17e3f164/go.mod h1:HhtDVdE/PRZFRia834tkmcwuscnaAzda1RJUW9Pr3Rg= github.com/tv42/httpunix v0.0.0-20150427012821-b75d8614f926/go.mod h1:9ESjWnEqriFuLhtthL60Sar/7RFoluCcXsuvEwTV5KM= github.com/xdg-go/pbkdf2 v1.0.0 h1:Su7DPu48wXMwC3bs7MCNG+z4FhcyEuz5dlvchbq0B0c= github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI= diff --git a/managed/Makefile b/managed/Makefile index dcf35ec550..ce6c7ae65c 100644 --- a/managed/Makefile +++ b/managed/Makefile @@ -38,6 +38,9 @@ clean: ## Remove generated files release: ## Build pmm-managed release binaries env CGO_ENABLED=0 go build -v $(PMM_LD_FLAGS) -o $(PMM_RELEASE_PATH)/ ./cmd/... +release-encryption-rotation: ## Build PMM encryption rotation tool + env CGO_ENABLED=0 go build -v $(PMM_LD_FLAGS) -o $(PMM_RELEASE_PATH)/ ./cmd/pmm-encryption-rotation/... + release-starlark: env CGO_ENABLED=0 go build -v $(PMM_LD_FLAGS) -o $(PMM_RELEASE_PATH)/ ./cmd/pmm-managed-starlark/... $(PMM_RELEASE_PATH)/pmm-managed-starlark --version diff --git a/managed/cmd/pmm-encryption-rotation/main.go b/managed/cmd/pmm-encryption-rotation/main.go new file mode 100644 index 0000000000..1aa01325f0 --- /dev/null +++ b/managed/cmd/pmm-encryption-rotation/main.go @@ -0,0 +1,95 @@ +// Copyright (C) 2023 Percona LLC +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . +// Package main is the main package for encryption keys rotation. +package main + +import ( + "fmt" + "os" + "os/signal" + "syscall" + + "github.com/alecthomas/kong" + "github.com/sirupsen/logrus" + + "github.com/percona/pmm/managed/models" + encryptionService "github.com/percona/pmm/managed/services/encryption" + "github.com/percona/pmm/utils/logger" + "github.com/percona/pmm/version" +) + +const codeDBConnectionFailed = 1 + +func main() { + signal.Ignore(syscall.SIGINT, syscall.SIGTERM) // to prevent any interuptions during process + + logger.SetupGlobalLogger() + + logrus.Infof("PMM Encryption Rotation Tools version: %s", version.Version) + + sqlDB, err := models.OpenDB(setupParams()) + if err != nil { + logrus.Error(err) + os.Exit(codeDBConnectionFailed) + } + + statusCode := encryptionService.RotateEncryptionKey(sqlDB, "pmm-managed") + sqlDB.Close() //nolint:errcheck + + os.Exit(statusCode) +} + +type flags struct { + Address string `name:"postgres-addr" default:"${address}" help:"PostgreSQL address with port"` + DBName string `name:"postgres-name" default:"pmm-managed" help:"PostgreSQL database name"` + DBUsername string `name:"postgres-username" default:"pmm-managed" help:"PostgreSQL database username name"` + DBPassword string `name:"postgres-password" default:"pmm-managed" help:"PostgreSQL database password"` + SSLMode string `name:"postgres-ssl-mode" default:"${disable_sslmode}" help:"PostgreSQL SSL mode" enum:"${disable_sslmode}, ${require_sslmode},${verify_sslmode}, ${verify_full_sslmode}"` //nolint:lll + SSLCAPath string `name:"postgres-ssl-ca-path" help:"PostgreSQL SSL CA root certificate path" type:"path"` + SSLKeyPath string `name:"postgres-ssl-key-path" help:"PostgreSQL SSL key path" type:"path"` + SSLCertPath string `name:"postgres-ssl-cert-path" help:"PostgreSQL SSL certificate path" type:"path"` +} + +func setupParams() models.SetupDBParams { + var opts flags + kong.Parse( + &opts, + kong.Name("encryption-rotation"), + kong.Description(fmt.Sprintf("Version %s", version.Version)), + kong.UsageOnError(), + kong.ConfigureHelp(kong.HelpOptions{ + Compact: true, + NoExpandSubcommands: true, + }), + kong.Vars{ + "address": models.DefaultPostgreSQLAddr, + "disable_sslmode": models.DisableSSLMode, + "require_sslmode": models.RequireSSLMode, + "verify_sslmode": models.VerifyCaSSLMode, + "verify_full_sslmode": models.VerifyFullSSLMode, + }, + ) + + return models.SetupDBParams{ + Address: opts.Address, + Name: opts.DBName, + Username: opts.DBUsername, + Password: opts.DBPassword, + SSLMode: opts.SSLMode, + SSLCAPath: opts.SSLCAPath, + SSLKeyPath: opts.SSLKeyPath, + SSLCertPath: opts.SSLCertPath, + } +} diff --git a/managed/models/database.go b/managed/models/database.go index d2c2066c7f..d7a9778721 100644 --- a/managed/models/database.go +++ b/managed/models/database.go @@ -27,7 +27,6 @@ import ( "net" "net/url" "os" - "slices" "strconv" "strings" @@ -61,6 +60,25 @@ const ( VerifyFullSSLMode string = "verify-full" ) +// DefaultAgentEncryptionColumns contains all tables and it's columns to be encrypted in PMM Server DB. +var DefaultAgentEncryptionColumns = []encryption.Table{ + { + Name: "agents", + Identifiers: []string{"agent_id"}, + Columns: []encryption.Column{ + {Name: "username"}, + {Name: "password"}, + {Name: "aws_access_key"}, + {Name: "aws_secret_key"}, + {Name: "mongo_db_tls_options", CustomHandler: EncryptMongoDBOptionsHandler}, + {Name: "azure_options", CustomHandler: EncryptAzureOptionsHandler}, + {Name: "mysql_options", CustomHandler: EncryptMySQLOptionsHandler}, + {Name: "postgresql_options", CustomHandler: EncryptPostgreSQLOptionsHandler}, + {Name: "agent_password"}, + }, + }, +} + // databaseSchema maps schema version from schema_migrations table (id column) to a slice of DDL queries. var databaseSchema = [][]string{ 1: { @@ -1149,27 +1167,7 @@ func SetupDB(ctx context.Context, sqlDB *sql.DB, params SetupDBParams) (*reform. return nil, errCV } - agentColumnsToEncrypt := []encryption.Column{ - {Name: "username"}, - {Name: "password"}, - {Name: "aws_access_key"}, - {Name: "aws_secret_key"}, - {Name: "mongo_db_tls_options", CustomHandler: EncryptMongoDBOptionsHandler}, - {Name: "azure_options", CustomHandler: EncryptAzureOptionsHandler}, - {Name: "mysql_options", CustomHandler: EncryptMySQLOptionsHandler}, - {Name: "postgresql_options", CustomHandler: EncryptPostgreSQLOptionsHandler}, - {Name: "agent_password"}, - } - - itemsToEncrypt := []encryption.Table{ - { - Name: "agents", - Identifiers: []string{"agent_id"}, - Columns: agentColumnsToEncrypt, - }, - } - - if err := migrateDB(db, params, itemsToEncrypt); err != nil { + if err := migrateDB(db, params, DefaultAgentEncryptionColumns); err != nil { return nil, err } @@ -1177,8 +1175,20 @@ func SetupDB(ctx context.Context, sqlDB *sql.DB, params SetupDBParams) (*reform. } // EncryptDB encrypts a set of columns in a specific database and table. -func EncryptDB(tx *reform.TX, params SetupDBParams, itemsToEncrypt []encryption.Table) error { - if len(itemsToEncrypt) == 0 { +func EncryptDB(tx *reform.TX, database string, itemsToEncrypt []encryption.Table) error { + return dbEncryption(tx, database, itemsToEncrypt, encryption.EncryptItems, true) +} + +// DecryptDB decrypts a set of columns in a specific database and table. +func DecryptDB(tx *reform.TX, database string, itemsToEncrypt []encryption.Table) error { + return dbEncryption(tx, database, itemsToEncrypt, encryption.DecryptItems, false) +} + +func dbEncryption(tx *reform.TX, database string, items []encryption.Table, + encryptionHandler func(tx *reform.TX, tables []encryption.Table) error, + expectedState bool, +) error { + if len(items) == 0 { return nil } @@ -1186,42 +1196,47 @@ func EncryptDB(tx *reform.TX, params SetupDBParams, itemsToEncrypt []encryption. if err != nil { return err } - alreadyEncrypted := make(map[string]bool) + currentColumns := make(map[string]bool) for _, v := range settings.EncryptedItems { - alreadyEncrypted[v] = true + currentColumns[v] = true } - notEncrypted := []encryption.Table{} - newlyEncrypted := []string{} - for _, table := range itemsToEncrypt { + tables := []encryption.Table{} + prepared := []string{} + for _, table := range items { columns := []encryption.Column{} for _, column := range table.Columns { - dbTableColumn := fmt.Sprintf("%s.%s.%s", params.Name, table.Name, column.Name) - if alreadyEncrypted[dbTableColumn] { + dbTableColumn := fmt.Sprintf("%s.%s.%s", database, table.Name, column.Name) + if currentColumns[dbTableColumn] == expectedState { continue } columns = append(columns, column) - newlyEncrypted = append(newlyEncrypted, dbTableColumn) + prepared = append(prepared, dbTableColumn) } if len(columns) == 0 { continue } table.Columns = columns - notEncrypted = append(notEncrypted, table) + tables = append(tables, table) } - - if len(notEncrypted) == 0 { + if len(tables) == 0 { return nil } - err = encryption.EncryptItems(tx, notEncrypted) + err = encryptionHandler(tx, tables) if err != nil { return err } + + encryptedItems := []string{} + if expectedState { + encryptedItems = prepared + } + _, err = UpdateSettings(tx, &ChangeSettingsParams{ - EncryptedItems: slices.Concat(settings.EncryptedItems, newlyEncrypted), + EncryptedItems: encryptedItems, }) if err != nil { return err @@ -1325,7 +1340,7 @@ func migrateDB(db *reform.DB, params SetupDBParams, itemsToEncrypt []encryption. } } - err := EncryptDB(tx, params, itemsToEncrypt) + err := EncryptDB(tx, params.Name, itemsToEncrypt) if err != nil { return err } diff --git a/managed/models/settings_helpers.go b/managed/models/settings_helpers.go index fb5125f336..24ab1507a3 100644 --- a/managed/models/settings_helpers.go +++ b/managed/models/settings_helpers.go @@ -226,9 +226,10 @@ func UpdateSettings(q reform.DBTX, params *ChangeSettingsParams) (*Settings, err settings.DefaultRoleID = *params.DefaultRoleID } - if len(params.EncryptedItems) != 0 { + if params.EncryptedItems != nil { settings.EncryptedItems = params.EncryptedItems } + err = SaveSettings(q, settings) if err != nil { return nil, err diff --git a/managed/services/encryption/encryption_rotation.go b/managed/services/encryption/encryption_rotation.go new file mode 100644 index 0000000000..f73f6d96ed --- /dev/null +++ b/managed/services/encryption/encryption_rotation.go @@ -0,0 +1,158 @@ +// Copyright (C) 2023 Percona LLC +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +// Package encryption contains PMM encryption rotation functions. +package encryption + +import ( + "database/sql" + "fmt" + "os/exec" + "strings" + "time" + + "github.com/pkg/errors" + "github.com/sirupsen/logrus" + "gopkg.in/reform.v1" + "gopkg.in/reform.v1/dialects/postgresql" + + "github.com/percona/pmm/managed/models" + "github.com/percona/pmm/managed/utils/encryption" +) + +const ( + retries = 5 + interval = 5 * time.Second + statusRunning = "RUNNING" + statusStopped = "STOPPED" + codeOK = 0 + codePMMStopFailed = 2 + codeEncryptionFailed = 3 + codePMMStartFailed = 4 +) + +// RotateEncryptionKey will stop PMM server, decrypt data, create new encryption key and encrypt them and start PMM Server again. +func RotateEncryptionKey(sqlDB *sql.DB, dbName string) int { + db := reform.NewDB(sqlDB, postgresql.Dialect, nil) + + err := stopPMMServer() + if err != nil { + logrus.Errorf("Failed to stop PMM Server: %+v", err) + return codePMMStopFailed + } + + err = rotateEncryptionKey(db, dbName) + if err != nil { + logrus.Errorf("Failed to rotate encryption key: %+v", err) + return codeEncryptionFailed + } + + err = startPMMServer() + if err != nil { + logrus.Errorf("Failed to start PMM Server: %+v", err) + return codePMMStartFailed + } + + return codeOK +} + +func startPMMServer() error { + logrus.Infoln("Starting PMM Server") + if pmmServerStatus(statusRunning) { + return nil + } + + cmd := exec.Command("supervisorctl", "start pmm-managed") + output, err := cmd.CombinedOutput() + if err != nil { + return fmt.Errorf("%w: %s", err, output) + } + + if !pmmServerStatusWithRetries(statusRunning) { + return errors.New("cannot start pmm-managed") + } + + return nil +} + +func stopPMMServer() error { + logrus.Infoln("Stopping PMM Server") + if pmmServerStatus(statusStopped) { + return nil + } + + cmd := exec.Command("supervisorctl", "stop pmm-managed") + output, err := cmd.CombinedOutput() + if err != nil { + return fmt.Errorf("%w: %s", err, output) + } + + if !pmmServerStatusWithRetries(statusStopped) { + return errors.New("cannot stop pmm-managed") + } + + return nil +} + +func pmmServerStatus(status string) bool { + cmd := exec.Command("supervisorctl", "status pmm-managed") + output, _ := cmd.CombinedOutput() + + return strings.Contains(string(output), strings.ToUpper(status)) +} + +func pmmServerStatusWithRetries(status string) bool { + for i := 0; i < retries; i++ { + if !pmmServerStatus(status) { + logrus.Infoln("Retry...") + time.Sleep(interval) + continue + } + + return true + } + + return false +} + +func rotateEncryptionKey(db *reform.DB, dbName string) error { + return db.InTransaction(func(tx *reform.TX) error { + logrus.Infof("DB %s is being decrypted", dbName) + err := models.DecryptDB(tx, dbName, models.DefaultAgentEncryptionColumns) + if err != nil { + return err + } + logrus.Infof("DB %s is successfully decrypted", dbName) + + logrus.Infoln("Rotating encryption key") + err = encryption.RotateEncryptionKey() + if err != nil { + return err + } + logrus.Infof("New encryption key generated") + + logrus.Infof("DB %s is being encrypted", dbName) + err = models.EncryptDB(tx, dbName, models.DefaultAgentEncryptionColumns) + if err != nil { + if e := encryption.RestoreOldEncryptionKey(); e != nil { + return errors.Wrap(err, e.Error()) + } + return err + } + logrus.Infof("DB %s is successfully encrypted", dbName) + + return nil + }) +} diff --git a/managed/services/encryption/encryption_rotation_test.go b/managed/services/encryption/encryption_rotation_test.go new file mode 100644 index 0000000000..99a1579916 --- /dev/null +++ b/managed/services/encryption/encryption_rotation_test.go @@ -0,0 +1,142 @@ +// Copyright (C) 2023 Percona LLC +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package encryption + +import ( + "database/sql" + "os" + "testing" + "time" + + "github.com/AlekSi/pointer" + "github.com/pkg/errors" + "github.com/stretchr/testify/require" + + "github.com/percona/pmm/managed/models" + "github.com/percona/pmm/managed/utils/encryption" + "github.com/percona/pmm/managed/utils/testdb" +) + +const ( + encryptionKeyTestPath = "/srv/pmm-encryption-rotation-test.key" + originEncryptionKey = `CMatkOIIEmQKWAowdHlwZS5nb29nbGVhcGlzLmNvbS9nb29nbGUuY3J5cHRvLnRpbmsuQWVzR2NtS2V5EiIaIKDxOKZxwiJl5Hj6oPZ/unTzmAvfwHWzZ1Wli0vac15YGAEQARjGrZDiCCAB` + // pmm-managed-username encrypted with originEncryptionKey + originUsernameHash = `AYxEFsZsg7lp9+eSy6+wPFHlaNNy0ZpTbYN0NuCLPnQOZUYf2S6H9B+XJdF4+DscxC/pJwI=` + // pmm-managed-password encrypted with originEncryptionKey + originPasswordHash = `AYxEFsZuL5xZb5IxGGh8NI6GrjDxCzFGxIcHe94UXcg+dnZphu7GQSgmZm633XvZ8CBU2wo=` //nolint:gosec +) + +func TestEncryptionRotation(t *testing.T) { + db := testdb.Open(t, models.SkipFixtures, pointer.ToInt(88)) + defer db.Close() //nolint:errcheck + + err := createOriginEncryptionKey() + require.NoError(t, err) + + err = insertTestData(db) + require.NoError(t, err) + + statusCode := RotateEncryptionKey(db, "pmm-managed-dev") + require.Equal(t, 0, statusCode) + + newEncryptionKey, err := os.ReadFile(encryptionKeyTestPath) + require.NoError(t, err) + require.NotEqual(t, newEncryptionKey, []byte(originEncryptionKey)) + + err = checkNewlyEncryptedData(db) + require.NoError(t, err) + + err = os.Remove(encryptionKeyTestPath) + require.NoError(t, err) +} + +func createOriginEncryptionKey() error { + encryption.DefaultEncryptionKeyPath = encryptionKeyTestPath + err := os.WriteFile(encryptionKeyTestPath, []byte(originEncryptionKey), 0o600) + if err != nil { + return err + } + encryption.DefaultEncryption = encryption.New() + return nil +} + +//nolint:dupword +func insertTestData(db *sql.DB) error { + _, err := models.UpdateSettings(db, &models.ChangeSettingsParams{ + EncryptedItems: []string{"pmm-managed-dev.agents.username", "pmm-managed-dev.agents.password", "pmm-managed-dev.agents.aws_access_key", "pmm-managed-dev.agents.aws_secret_key", "pmm-managed-dev.agents.mongo_db_tls_options", "pmm-managed-dev.agents.azure_options", "pmm-managed-dev.agents.mysql_options", "pmm-managed-dev.agents.postgresql_options", "pmm-managed-dev.agents.agent_password"}, + }) + if err != nil { + return err + } + + now := time.Now() + _, err = db.Exec( + "INSERT INTO nodes (node_id, node_type, node_name, distro, node_model, az, address, created_at, updated_at) "+ + "VALUES ('1', 'generic', 'name', '', '', '', '', $1, $2)", + now, now) + if err != nil { + return err + } + _, err = db.Exec( + "INSERT INTO services (service_id, service_type, service_name, node_id, environment, cluster, replication_set, socket, external_group, created_at, updated_at) "+ + "VALUES ('1', 'mysql', 'name', '1', '', '', '', '/var/run/mysqld/mysqld.sock', '', $1, $2)", + now, now) + if err != nil { + return err + } + _, err = db.Exec( + "INSERT INTO agents (agent_id, agent_type, username, password, runs_on_node_id, pmm_agent_id, disabled, status, created_at, updated_at, tls, tls_skip_verify, max_query_length, query_examples_disabled, comments_parsing_disabled, max_query_log_size, table_count_tablestats_group_limit, rds_basic_metrics_disabled, rds_enhanced_metrics_disabled, push_metrics, expose_exporter) "+ + "VALUES ('1', 'pmm-agent', $1, $2, '1', NULL, false, '', $3, $4, false, false, 0, false, true, 0, 0, true, true, false, false)", + originUsernameHash, originPasswordHash, now, now) + if err != nil { + return err + } + + return nil +} + +func checkNewlyEncryptedData(db *sql.DB) error { + var newlyEncryptedUsername string + var newlyEncryptedPassword string + err := db.QueryRow(`SELECT username, password FROM agents WHERE agent_id = $1`, "1").Scan(&newlyEncryptedUsername, &newlyEncryptedPassword) + if err != nil { + return err + } + if newlyEncryptedUsername == originUsernameHash { + return errors.New("username hash not rotated properly") + } + if newlyEncryptedPassword == originPasswordHash { + return errors.New("password hash not rotated properly") + } + + username, err := encryption.Decrypt(newlyEncryptedUsername) + if err != nil { + return err + } + if username != "pmm-managed-username" { + return errors.New("username not properly decrypted") + } + + password, err := encryption.Decrypt(newlyEncryptedPassword) + if err != nil { + return err + } + if password != "pmm-managed-password" { + return errors.New("password not properly decrypted") + } + + return nil +} diff --git a/managed/utils/encryption/encryption.go b/managed/utils/encryption/encryption.go index a7cba048df..0396e73170 100644 --- a/managed/utils/encryption/encryption.go +++ b/managed/utils/encryption/encryption.go @@ -17,34 +17,64 @@ package encryption import ( + "bytes" "encoding/base64" + "fmt" "os" "slices" + "strings" + "sync" + "github.com/google/tink/go/tink" "github.com/pkg/errors" "github.com/sirupsen/logrus" + "github.com/tink-crypto/tink-go/aead" + "github.com/tink-crypto/tink-go/insecurecleartextkeyset" + "github.com/tink-crypto/tink-go/keyset" "gopkg.in/reform.v1" ) -// DefaultEncryptionKeyPath contains default PMM encryption key path. -const DefaultEncryptionKeyPath = "/srv/pmm-encryption.key" - var ( + // DefaultEncryptionKeyPath contains default PMM encryption key path. + DefaultEncryptionKeyPath = "/srv/pmm-encryption.key" // ErrEncryptionNotInitialized is error in case of encryption is not initialized. ErrEncryptionNotInitialized = errors.New("encryption is not initialized") // DefaultEncryption is the default implementation of encryption. - DefaultEncryption = New(DefaultEncryptionKeyPath) + DefaultEncryption = New() + defaultEncryptionMtx sync.Mutex ) +// Encryption contains fields required for encryption. +type Encryption struct { + Path string + Key string + Primitive tink.AEAD +} + +// Table represents table name, it's identifiers and columns to be encrypted/decrypted. +type Table struct { + Name string + Identifiers []string + Columns []Column +} + +// Column represents column name and column's custom handler (if needed). +type Column struct { + Name string + CustomHandler func(e *Encryption, val any) (any, error) +} + +// QueryValues represents query to update row after encrypt/decrypt. +type QueryValues struct { + Query string + SetValues [][]any + WhereValues [][]any +} + // New creates an encryption; if key on path doesn't exist, it will be generated. -func New(keyPath string) *Encryption { +func New() *Encryption { e := &Encryption{} - customKeyPath := os.Getenv("PMM_ENCRYPTION_KEY_PATH") - if customKeyPath != "" { - e.Path = customKeyPath - } else { - e.Path = keyPath - } + e.Path = encryptionKeyPath() bytes, err := os.ReadFile(e.Path) switch { @@ -68,6 +98,59 @@ func New(keyPath string) *Encryption { return e } +// RotateEncryptionKey is a wrapper around DefaultEncryption.RotateEncryptionKey. +func RotateEncryptionKey() error { + err := backupOldEncryptionKey() + if err != nil { + return err + } + + defaultEncryptionMtx.Lock() + DefaultEncryption = New() + defaultEncryptionMtx.Unlock() + + return nil +} + +// RestoreOldEncryptionKey is a wrapper around DefaultEncryption.RestoreOldEncryptionKey. +func RestoreOldEncryptionKey() error { + err := os.Rename(fmt.Sprintf("%s_old.key", strings.TrimSuffix(encryptionKeyPath(), ".key")), encryptionKeyPath()) + if err != nil { + return err + } + + return nil +} + +func backupOldEncryptionKey() error { + err := os.Rename(encryptionKeyPath(), fmt.Sprintf("%s_old.key", strings.TrimSuffix(encryptionKeyPath(), ".key"))) + if err != nil { + return err + } + + return nil +} + +func (e *Encryption) generateKey() error { + handle, err := keyset.NewHandle(aead.AES256GCMKeyTemplate()) + if err != nil { + return err + } + + buff := &bytes.Buffer{} + err = insecurecleartextkeyset.Write(handle, keyset.NewBinaryWriter(buff)) + if err != nil { + return err + } + e.Key = base64.StdEncoding.EncodeToString(buff.Bytes()) + + return e.saveKeyToFile() +} + +func (e *Encryption) saveKeyToFile() error { + return os.WriteFile(e.Path, []byte(e.Key), 0o644) //nolint:gosec +} + // Encrypt is a wrapper around DefaultEncryption.Encrypt. func Encrypt(secret string) (string, error) { return DefaultEncryption.Encrypt(secret) @@ -203,3 +286,18 @@ func (e *Encryption) DecryptItems(tx *reform.TX, tables []Table) error { return nil } + +func (e *Encryption) getPrimitive() (tink.AEAD, error) { //nolint:ireturn + serializedKeyset, err := base64.StdEncoding.DecodeString(e.Key) + if err != nil { + return nil, err + } + + binaryReader := keyset.NewBinaryReader(bytes.NewBuffer(serializedKeyset)) + parsedHandle, err := insecurecleartextkeyset.Read(binaryReader) + if err != nil { + return nil, err + } + + return aead.New(parsedHandle) +} diff --git a/managed/utils/encryption/helpers.go b/managed/utils/encryption/helpers.go index c8c11ab4e3..01a31f2762 100644 --- a/managed/utils/encryption/helpers.go +++ b/managed/utils/encryption/helpers.go @@ -16,21 +16,24 @@ package encryption import ( - "bytes" "database/sql" - "encoding/base64" "fmt" "os" "slices" "strings" - "github.com/google/tink/go/aead" - "github.com/google/tink/go/insecurecleartextkeyset" - "github.com/google/tink/go/keyset" - "github.com/google/tink/go/tink" "gopkg.in/reform.v1" ) +func encryptionKeyPath() string { + customKeyPath := os.Getenv("PMM_ENCRYPTION_KEY_PATH") + if customKeyPath != "" { + return customKeyPath + } + + return DefaultEncryptionKeyPath +} + func prepareRowPointers(rows *sql.Rows) ([]any, error) { columnTypes, err := rows.ColumnTypes() if err != nil { @@ -82,41 +85,6 @@ func decryptColumnStringHandler(e *Encryption, val any) (any, error) { return decrypted, nil } -func (e *Encryption) getPrimitive() (tink.AEAD, error) { //nolint:ireturn - serializedKeyset, err := base64.StdEncoding.DecodeString(e.Key) - if err != nil { - return nil, err - } - - binaryReader := keyset.NewBinaryReader(bytes.NewBuffer(serializedKeyset)) - parsedHandle, err := insecurecleartextkeyset.Read(binaryReader) - if err != nil { - return nil, err - } - - return aead.New(parsedHandle) -} - -func (e *Encryption) generateKey() error { - handle, err := keyset.NewHandle(aead.AES256GCMKeyTemplate()) - if err != nil { - return err - } - - buff := &bytes.Buffer{} - err = insecurecleartextkeyset.Write(handle, keyset.NewBinaryWriter(buff)) - if err != nil { - return err - } - e.Key = base64.StdEncoding.EncodeToString(buff.Bytes()) - - return e.saveKeyToFile() -} - -func (e *Encryption) saveKeyToFile() error { - return os.WriteFile(e.Path, []byte(e.Key), 0o644) //nolint:gosec -} - func (table Table) columnsList() []string { res := []string{} for _, c := range table.Columns { diff --git a/managed/utils/encryption/models.go b/managed/utils/encryption/models.go deleted file mode 100644 index 257b49b1de..0000000000 --- a/managed/utils/encryption/models.go +++ /dev/null @@ -1,45 +0,0 @@ -// Copyright (C) 2023 Percona LLC -// -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU Affero General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Affero General Public License for more details. -// -// You should have received a copy of the GNU Affero General Public License -// along with this program. If not, see . - -package encryption - -import "github.com/google/tink/go/tink" - -// Encryption contains fields required for encryption. -type Encryption struct { - Path string - Key string - Primitive tink.AEAD -} - -// Table represents table name, it's identifiers and columns to be encrypted/decrypted. -type Table struct { - Name string - Identifiers []string - Columns []Column -} - -// Column represents column name and column's custom handler (if needed). -type Column struct { - Name string - CustomHandler func(e *Encryption, val any) (any, error) -} - -// QueryValues represents query to update row after encrypt/decrypt. -type QueryValues struct { - Query string - SetValues [][]any - WhereValues [][]any -}