From ec6ee0b59162c97977c445c418fe1f9cdbbe0b17 Mon Sep 17 00:00:00 2001 From: edenkoveshi Date: Wed, 20 Oct 2021 21:07:09 +0300 Subject: [PATCH 1/2] Re-submitting changes from old fork --- internal/patroni/config.go | 12 + internal/patroni/config_test.go | 102 ++++++ internal/pgtune/pgtune.go | 320 +++++++++++++++++ internal/pgtune/pgtune_test.go | 335 ++++++++++++++++++ .../v1beta1/postgrescluster_types.go | 21 ++ 5 files changed, 790 insertions(+) create mode 100644 internal/pgtune/pgtune.go create mode 100644 internal/pgtune/pgtune_test.go diff --git a/internal/patroni/config.go b/internal/patroni/config.go index b045ebbafc..686e6695be 100644 --- a/internal/patroni/config.go +++ b/internal/patroni/config.go @@ -24,6 +24,7 @@ import ( "sigs.k8s.io/yaml" "github.com/crunchydata/postgres-operator/internal/naming" + "github.com/crunchydata/postgres-operator/internal/pgtune" "github.com/crunchydata/postgres-operator/internal/postgres" "github.com/crunchydata/postgres-operator/pkg/apis/postgres-operator.crunchydata.com/v1beta1" ) @@ -243,6 +244,17 @@ func DynamicConfiguration( } } } + + //Add PGTuneConfiguration if enabled. Do not override patroni.DynamicConfiguration values + if cluster.Spec.AutoPGTune != nil { + PGTuneConfig := pgtune.GetPGTuneConfigParameters(cluster) + for k, v := range PGTuneConfig { + if _, ok := parameters[k]; !ok { + parameters[k] = v + } + } + } + postgresql["parameters"] = parameters // Copy the "postgresql.pg_hba" section after any mandatory values. diff --git a/internal/patroni/config_test.go b/internal/patroni/config_test.go index a9c4ede784..56471cb7e5 100644 --- a/internal/patroni/config_test.go +++ b/internal/patroni/config_test.go @@ -867,3 +867,105 @@ func TestProbeTiming(t *testing.T) { assert.Assert(t, actual.FailureThreshold >= 1) // Minimum value is 1. } } + +func TestPatroniOverrides(t *testing.T) { + cluster := new(v1beta1.PostgresCluster) + cluster.Default() + cluster.Namespace = "some-namespace" + cluster.Name = "cluster-name" + cluster.Spec.PostgresVersion = 13 + cluster.Spec.AutoPGTune = &v1beta1.PostgresClusterPGTune{} + cluster.Spec.InstanceSets = []v1beta1.PostgresInstanceSetSpec{{Name: "test"}} + for i := range cluster.Spec.InstanceSets { + cluster.Spec.InstanceSets[i].Default(i) + cluster.Spec.InstanceSets[i].Resources = v1.ResourceRequirements{Requests: v1.ResourceList{ + "memory": resource.MustParse("4Gi"), + "cpu": resource.MustParse("2000m"), + }, + } + } + + patronidcInput := map[string]interface{}{ + "postgresql": map[string]interface{}{ + "parameters": map[string]interface{}{ + "wal_buffers": "patroni", + "work_mem": "overrides", + "shared_buffers": "pgtune", + }, + }, + } + + expected := map[string]interface{}{ + "loop_wait": int32(10), + "ttl": int32(30), + "postgresql": map[string]interface{}{ + "parameters": map[string]interface{}{ + "wal_buffers": "patroni", + "work_mem": "overrides", + "shared_buffers": "pgtune", + "max_parallel_workers": "2", + "max_parallel_workers_per_gather": "1", + "max_worker_processes": "2", + "max_parallel_maintenance_workers": "1", + "default_statistics_target": "100", + "effective_cache_size": "3072MB", + "maintenance_work_mem": "256MB", + "min_wal_size": "1GB", + "max_wal_size": "4GB", + }, + "pg_hba": []string{}, + "use_pg_rewind": true, + "use_slots": false, + }, + } + + actual := DynamicConfiguration(cluster, patronidcInput, postgres.HBAs{}, postgres.Parameters{}) + + assert.DeepEqual(t, expected, actual) +} + +func TestPatroniPGTuneDisabled(t *testing.T) { + cluster := new(v1beta1.PostgresCluster) + cluster.Default() + cluster.Namespace = "some-namespace" + cluster.Name = "cluster-name" + cluster.Spec.PostgresVersion = 13 + cluster.Spec.InstanceSets = []v1beta1.PostgresInstanceSetSpec{{Name: "test"}} + for i := range cluster.Spec.InstanceSets { + cluster.Spec.InstanceSets[i].Default(i) + cluster.Spec.InstanceSets[i].Resources = v1.ResourceRequirements{Requests: v1.ResourceList{ + "memory": resource.MustParse("4Gi"), + "cpu": resource.MustParse("2000m"), + }, + } + } + + patronidcInput := map[string]interface{}{ + "postgresql": map[string]interface{}{ + "parameters": map[string]interface{}{ + "wal_buffers": "pgtune", + "work_mem": "is", + "shared_buffers": "disabled", + }, + }, + } + + expected := map[string]interface{}{ + "loop_wait": int32(10), + "ttl": int32(30), + "postgresql": map[string]interface{}{ + "parameters": map[string]interface{}{ + "wal_buffers": "pgtune", + "work_mem": "is", + "shared_buffers": "disabled", + }, + "pg_hba": []string{}, + "use_pg_rewind": true, + "use_slots": false, + }, + } + + actual := DynamicConfiguration(cluster, patronidcInput, postgres.HBAs{}, postgres.Parameters{}) + + assert.DeepEqual(t, expected, actual) +} diff --git a/internal/pgtune/pgtune.go b/internal/pgtune/pgtune.go new file mode 100644 index 0000000000..eeacd997dd --- /dev/null +++ b/internal/pgtune/pgtune.go @@ -0,0 +1,320 @@ +package pgtune + +/* + Copyright 2021 Crunchy Data Solutions, Inc. + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +/* This package calculates recommended configuration settings according to pgTune (https://pgtune.leopard.in.ua/) +The settings are updated using Patroni Dynamic Conifguration which is already implemented, so this essentialy enables Patroni +This works only if PostgresCluster.Spec.autoPgTune property is set to true (default is false) +If both PostgresCluster.Spec.autoPgTune and PostgresCluster.Spec.Patroni.DynamicConfiguration are enabled, DynamicConfiguration will override this. +Memory and CPU values are taken from resources.requests propety, +Storage and Application Type values are CR fields. +Default Application Type is mixed, everything else will be ignored if not set. +Full credit goes to https://github.com/le0pard/pgtune */ + +import ( + "fmt" + "math" + + "github.com/crunchydata/postgres-operator/pkg/apis/postgres-operator.crunchydata.com/v1beta1" + "k8s.io/apimachinery/pkg/runtime" +) + +const ( + SizeKB = "KB" + SizeMB = "MB" + SizeGB = "GB" + + HDTypeSSD = "ssd" + HDTypeHDD = "hdd" + HDTypeSAN = "san" + + AppTypeWeb = "web" + AppTypeOLTP = "oltp" + AppTypeDW = "dw" + AppTypeDesktop = "desktop" + AppTypeMixed = "mixed" + + NameSharedBuffers = "shared_buffers" + NameWorkMem = "work_mem" + NameEffectiveCacheSize = "effective_cache_size" + NameMaintenanceWorkMem = "maintenance_work_mem" + NameWalBuffers = "wal_buffers" + NameDefaultStatisticsTarget = "default_statistics_target" + NameMinWalSize = "min_wal_size" + NameMaxWalSize = "max_wal_size" + NameMaxWorkerProcesses = "max_worker_processes" + NameMaxParallelWorkersPerGather = "max_parallel_workers_per_gather" + NameMaxParallelWorkers = "max_parallel_workers" + NameMaxParallelMaintenanceWorkers = "max_parallel_maintenance_workers" + NameRandomPageCost = "random_page_cost" + NameEffectiveIOConcurrency = "effective_io_concurrency" + + //Do not assign more then 10GB shared buffers + MaxSharedBuffers = 10 * 1024 + //Do not assign more then 2GB RAM for maintenance + MaxMaintenanceWorkMem = 2048 + MaxWalBuffersKB = 16 * 1024 //16MB at most, represented in KB + MinWalBuffersKB = 32 //32KB at least + MaxWorkersPerGather = 4 + MinWorkMem = 64 //64kb at least + DefaultWorkersPerGatherForWorkMem = 2 + + MaxConnections = 100 //This is already set up by Patroni +) + +func GetPGTuneConfigParameters(cluster *v1beta1.PostgresCluster) map[string]interface{} { + parameters := make(map[string]interface{}) + totalMemKB := TotalMemoryInKB(cluster) + + sharedBuffersVal := TuneSharedBuffers(cluster, parameters, totalMemKB) + TuneDefaultStatisticsTarget(cluster, parameters) + TuneEffectiveCacheSize(cluster, parameters, totalMemKB) + TuneMaintenanceWorkMem(cluster, parameters, totalMemKB) + TuneWalBuffers(cluster, parameters, sharedBuffersVal) + TuneMinWalSize(cluster, parameters) + TuneMaxWalSize(cluster, parameters) + TuneRandomPageCost(cluster, parameters) + TuneEffectiveIOConcurrency(cluster, parameters) + parallelWorkersPerGather := TuneParallelSettings(cluster, parameters) + TuneWorkMem(cluster, parameters, totalMemKB, sharedBuffersVal, parallelWorkersPerGather) + + return parameters +} + +func initPatroniForPGTune(cluster *v1beta1.PostgresCluster) { + if cluster.Spec.Patroni == nil { + cluster.Spec.Patroni = &v1beta1.PatroniSpec{DynamicConfiguration: runtime.RawExtension{}} + } + if cluster.Spec.Patroni.DynamicConfiguration.Raw == nil { + cluster.Spec.Patroni.DynamicConfiguration.Raw = []byte{} + } +} + +// TotalMemoryInKB returns memory request value in KB. 0 if no memory request has been made. +func TotalMemoryInKB(cluster *v1beta1.PostgresCluster) int64 { + totalMemInBytes := cluster.Spec.InstanceSets[0].Resources.Requests.Memory().Value() + totalMemInKB := totalMemInBytes / 1024 + return totalMemInKB +} + +//TotalCPUCores returns the number of CPU cores. This is validated by Kubernetes so nov validation is needed. +func TotalCPUCores(cluster *v1beta1.PostgresCluster) int64 { + totalCPU := cluster.Spec.InstanceSets[0].Resources.Requests.Cpu().Value() + return totalCPU +} + +/* + Each of the following functions calculate a unique property following PGTune recommendations. +*/ + +// TuneWorkMem calculates work_mem recommended configuration property +// This is the most complicated one and it will be tuned only if both cpu and memory +// and memory have been requested. cpu must be greather than 2 as it depends on parallel settings +func TuneWorkMem(cluster *v1beta1.PostgresCluster, params map[string]interface{}, totalMemKB int64, sharedBuffers int, parallelWorkersPerGather int64) { + // This will be tuned only if both memory and cpu has been requested. + if parallelWorkersPerGather == 0 { + parallelWorkersPerGather = DefaultWorkersPerGatherForWorkMem + } + if totalMemKB > 0 { //totalMemKB > 0 implies sharedBuffers > 0 + factor := 2 //mixed,web and dw + fmt.Println(cluster.Spec.AutoPGTune.ApplicationType) + switch cluster.Spec.AutoPGTune.ApplicationType { + case AppTypeOLTP: + factor = 1 + case AppTypeDesktop: + factor = 6 + } + workMem := int64((totalMemKB - KB(int64(sharedBuffers), SizeMB))) / (MaxConnections * 3) / parallelWorkersPerGather / int64(factor) + if workMem < MinWorkMem { + workMem = MinWorkMem + } + params[NameWorkMem] = fmt.Sprintf("%dkB", workMem) + } +} + +// TuneSharedBuffers calculates shared_buffers recommended configuration property which is Memory/4. +// returns sharedBuffers value or 0 if memory request is not set. +func TuneSharedBuffers(cluster *v1beta1.PostgresCluster, params map[string]interface{}, totalMemKB int64) int { + // if totalMemKB == 0, then memory request has not been set. Do not assign sharedBuffers in that case. + if totalMemKB > 0 { + factor := 1 + if cluster.Spec.AutoPGTune.ApplicationType == AppTypeDesktop { + factor = 4 + } + // divide by 16 for desktop, 4 for everything else + sharedBuffersVal := int(math.Min(float64(MB(totalMemKB/int64(4*factor), SizeKB)), MaxSharedBuffers)) + params[NameSharedBuffers] = fmt.Sprintf("%dMB", sharedBuffersVal) + return sharedBuffersVal + } + return 0 +} + +func TuneEffectiveCacheSize(cluster *v1beta1.PostgresCluster, params map[string]interface{}, totalMemKB int64) { + // if totalMemKB == 0, then memory request has not been set. Do not assign EffectiveCacheSize in that case. + if totalMemKB > 0 { + factor := 3 + if cluster.Spec.AutoPGTune.ApplicationType == AppTypeDesktop { + factor = 1 + } + // factor by 1/4 for desktop, 3/4 for anything else + params[NameEffectiveCacheSize] = fmt.Sprintf("%dMB", MB(totalMemKB*int64(factor)/4, SizeKB)) + } +} + +func TuneMaintenanceWorkMem(cluster *v1beta1.PostgresCluster, params map[string]interface{}, totalMemKB int64) { + // if totalMemKB == 0, then memory request has not been set. Do not assign MaintenanceWorkMem in that case. + if totalMemKB > 0 { + factor := 2 + if cluster.Spec.AutoPGTune.ApplicationType == AppTypeDW { + factor = 1 + } + // divide by 8 for DW, 16 for anything else + params[NameMaintenanceWorkMem] = fmt.Sprintf("%dMB", int(math.Min(float64(MB(totalMemKB/int64(8*factor), SizeKB)), MaxMaintenanceWorkMem))) //cap at 2GB + } +} + +func TuneWalBuffers(cluster *v1beta1.PostgresCluster, params map[string]interface{}, SharedBuffersMB int) { + //SharedBuffersMB == 0 if and only if requests.memory is not set + if SharedBuffersMB > 0 { + walBuffersValue := math.Ceil(3 * float64(KB(int64(SharedBuffersMB), SizeMB)) / 100) //3% of SharedBuffers value + walBuffersValue = math.Min(walBuffersValue, MaxWalBuffersKB) //at most MaxWalBuffers + walBuffersValue = math.Max(walBuffersValue, MinWalBuffersKB) //at least MinWalBuffers + if walBuffersValue >= 1024 { //format to MB + params[NameWalBuffers] = fmt.Sprintf("%dMB", MB(int64(walBuffersValue), SizeKB)) + } else { + params[NameWalBuffers] = fmt.Sprintf("%.fKB", walBuffersValue) + } + } +} + +func TuneDefaultStatisticsTarget(cluster *v1beta1.PostgresCluster, params map[string]interface{}) { + dst := 100 + if cluster.Spec.AutoPGTune.ApplicationType == AppTypeDW { + dst = 500 + } + params[NameDefaultStatisticsTarget] = fmt.Sprintf("%d", dst) +} + +func TuneMinWalSize(cluster *v1beta1.PostgresCluster, params map[string]interface{}) { + walSize := 1024 //mixed and web + switch cluster.Spec.AutoPGTune.ApplicationType { + case AppTypeDW: + walSize = 4096 + case AppTypeOLTP: + walSize = 2048 + case AppTypeDesktop: + walSize = 100 + } + params[NameMinWalSize] = fmt.Sprintf("%dMB", walSize) +} + +func TuneMaxWalSize(cluster *v1beta1.PostgresCluster, params map[string]interface{}) { + walSize := 4096 //mixed and web + switch cluster.Spec.AutoPGTune.ApplicationType { + case AppTypeDW: + walSize = 16384 + case AppTypeOLTP: + walSize = 8192 + case AppTypeDesktop: + walSize = 2048 + } + params[NameMaxWalSize] = fmt.Sprintf("%dMB", walSize) +} + +/* + TuneParallelSettings calculates all properties related to parallel execution + They will be tuned only if cpu request is defined and greather than/equal to 2 cores. + Returns the value of max_workers_per_gather property to be used later. +*/ +func TuneParallelSettings(cluster *v1beta1.PostgresCluster, params map[string]interface{}) int64 { + totalCPUCores := TotalCPUCores(cluster) + if totalCPUCores >= 2 { //Do not tune for less than 2 CPUS. 0 means no CPU has been requested + params[NameMaxWorkerProcesses] = fmt.Sprintf("%d", totalCPUCores) + params[NameMaxParallelWorkers] = fmt.Sprintf("%d", totalCPUCores) + + WorkersPerGather := int64(math.Ceil(float64(totalCPUCores) / 2)) + CappedWorkersPerGather := int64(math.Min(float64(WorkersPerGather), MaxWorkersPerGather)) + if cluster.Spec.AutoPGTune.ApplicationType != AppTypeDW { + // do not cap DW type application. + WorkersPerGather = CappedWorkersPerGather + } + params[NameMaxParallelWorkersPerGather] = fmt.Sprintf("%d", WorkersPerGather) + + if cluster.Spec.PostgresVersion >= 11 { + // cap any application type + params[NameMaxParallelMaintenanceWorkers] = fmt.Sprintf("%d", CappedWorkersPerGather) + } + return WorkersPerGather + } + return 0 +} + +func TuneRandomPageCost(cluster *v1beta1.PostgresCluster, params map[string]interface{}) { + if hdType := cluster.Spec.AutoPGTune.HDType; hdType != nil { + rpc := 1.1 + if *hdType == HDTypeHDD { + rpc = 4 + } + params[NameRandomPageCost] = fmt.Sprintf("%g", rpc) + } +} + +func TuneEffectiveIOConcurrency(cluster *v1beta1.PostgresCluster, params map[string]interface{}) { + if hdType := cluster.Spec.AutoPGTune.HDType; hdType != nil { + iocon := 0 + switch *hdType { + case HDTypeHDD: + iocon = 2 + case HDTypeSSD: + iocon = 200 + case HDTypeSAN: + iocon = 300 + } + if iocon > 0 { + params[NameEffectiveIOConcurrency] = fmt.Sprintf("%d", iocon) + } + } +} + +func GB(t int64, convertFrom string) int64 { + switch convertFrom { + case SizeMB: + return int64(math.Ceil(float64(t) / 1024)) + case SizeKB: + return int64(math.Ceil(float64(t) / (1024 * 1024))) + } + return t +} + +func MB(t int64, convertFrom string) int64 { + switch convertFrom { + case SizeGB: + return t * 1024 + case SizeKB: + return int64(math.Ceil(float64(t) / 1024)) + } + return t +} + +func KB(t int64, convertFrom string) int64 { + switch convertFrom { + case SizeMB: + return t * 1024 + case SizeGB: + return t * 1024 * 1024 + } + return t +} diff --git a/internal/pgtune/pgtune_test.go b/internal/pgtune/pgtune_test.go new file mode 100644 index 0000000000..5d862ea0ae --- /dev/null +++ b/internal/pgtune/pgtune_test.go @@ -0,0 +1,335 @@ +/* + Copyright 2021 Crunchy Data Solutions, Inc. + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package pgtune + +import ( + "testing" + + "github.com/crunchydata/postgres-operator/pkg/apis/postgres-operator.crunchydata.com/v1beta1" + "gotest.tools/assert" + v1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" +) + +func DefaultCluster() *v1beta1.PostgresCluster { + cluster := new(v1beta1.PostgresCluster) + cluster.Default() + cluster.Namespace = "some-namespace" + cluster.Name = "cluster-name" + cluster.Spec.PostgresVersion = 13 + cluster.Spec.InstanceSets = []v1beta1.PostgresInstanceSetSpec{{Name: "test"}} + for i := range cluster.Spec.InstanceSets { + cluster.Spec.InstanceSets[i].Default(i) + } + cluster.Spec.AutoPGTune = &v1beta1.PostgresClusterPGTune{ + ApplicationType: "mixed", + } + return cluster +} + +func TestGetPGTuneConfigParameters(t *testing.T) { + t.Parallel() + cluster := DefaultCluster() + for i := range cluster.Spec.InstanceSets { + cluster.Spec.InstanceSets[i].Default(i) + cluster.Spec.InstanceSets[i].Resources = v1.ResourceRequirements{Requests: v1.ResourceList{ + "memory": resource.MustParse("4Gi"), + "cpu": resource.MustParse("2000m"), + }, + } + } + + actual := GetPGTuneConfigParameters(cluster) + + expected := map[string]interface{}{ + "shared_buffers": "1024MB", + "max_parallel_workers": "2", + "max_parallel_workers_per_gather": "1", + "max_worker_processes": "2", + "max_parallel_maintenance_workers": "1", + "wal_buffers": "16MB", + "work_mem": "5242kB", + "default_statistics_target": "100", + "effective_cache_size": "3072MB", + "maintenance_work_mem": "256MB", + "min_wal_size": "1024MB", + "max_wal_size": "4096MB", + } + + assert.DeepEqual(t, expected, actual) +} + +func TestNoMemoryRequests(t *testing.T) { + cluster := DefaultCluster() + for i := range cluster.Spec.InstanceSets { + cluster.Spec.InstanceSets[i].Default(i) + cluster.Spec.InstanceSets[i].Resources = v1.ResourceRequirements{Requests: v1.ResourceList{ + "cpu": resource.MustParse("2000m"), + }, + } + } + + actual := GetPGTuneConfigParameters(cluster) + + expected := map[string]interface{}{ + "max_parallel_workers": "2", + "max_parallel_workers_per_gather": "1", + "max_worker_processes": "2", + "max_parallel_maintenance_workers": "1", + "default_statistics_target": "100", + "min_wal_size": "1024MB", + "max_wal_size": "4096MB", + } + + assert.DeepEqual(t, expected, actual) + +} + +func TestNoCPURequests(t *testing.T) { + cluster := DefaultCluster() + for i := range cluster.Spec.InstanceSets { + cluster.Spec.InstanceSets[i].Default(i) + cluster.Spec.InstanceSets[i].Resources = v1.ResourceRequirements{Requests: v1.ResourceList{ + "memory": resource.MustParse("4Gi"), + }, + } + } + + actual := GetPGTuneConfigParameters(cluster) + + expected := map[string]interface{}{ + "shared_buffers": "1024MB", + "wal_buffers": "16MB", + "default_statistics_target": "100", + "effective_cache_size": "3072MB", + "maintenance_work_mem": "256MB", + "min_wal_size": "1024MB", + "max_wal_size": "4096MB", + "work_mem": "2621kB", + } + + assert.DeepEqual(t, expected, actual) +} + +func TestStorageTypes(t *testing.T) { + for _, tt := range []struct { + hdtype string + expected map[string]interface{} + }{ + { + hdtype: HDTypeSSD, + expected: map[string]interface{}{ + "effective_io_concurrency": "200", + "random_page_cost": "1.1", + "max_wal_size": "4096MB", + "min_wal_size": "1024MB", + "default_statistics_target": "100", + }, + }, + { + hdtype: HDTypeHDD, + expected: map[string]interface{}{ + "effective_io_concurrency": "2", + "random_page_cost": "4", + "max_wal_size": "4096MB", + "min_wal_size": "1024MB", + "default_statistics_target": "100", + }, + }, + { + hdtype: HDTypeSAN, + expected: map[string]interface{}{ + "effective_io_concurrency": "300", + "random_page_cost": "1.1", + "max_wal_size": "4096MB", + "min_wal_size": "1024MB", + "default_statistics_target": "100", + }, + }, + } { + cluster := DefaultCluster() + for i := range cluster.Spec.InstanceSets { + cluster.Spec.InstanceSets[i].Default(i) + } + cluster.Spec.AutoPGTune.HDType = &tt.hdtype + actual := GetPGTuneConfigParameters(cluster) + + assert.DeepEqual(t, tt.expected, actual) + } +} + +func TestApplicationTypesDefaults(t *testing.T) { + for _, tt := range []struct { + apptype string + expected map[string]interface{} + }{ + { + apptype: AppTypeDW, + expected: map[string]interface{}{ + "min_wal_size": "4096MB", + "max_wal_size": "16384MB", + "default_statistics_target": "500", + }, + }, + { + apptype: AppTypeDesktop, + expected: map[string]interface{}{ + "min_wal_size": "100MB", + "max_wal_size": "2048MB", + "default_statistics_target": "100", + }, + }, + { + apptype: AppTypeOLTP, + expected: map[string]interface{}{ + "min_wal_size": "2048MB", + "max_wal_size": "8192MB", + "default_statistics_target": "100", + }, + }, + { + apptype: AppTypeWeb, + expected: map[string]interface{}{ + "min_wal_size": "1024MB", + "max_wal_size": "4096MB", + "default_statistics_target": "100", + }, + }, + { + apptype: AppTypeMixed, + expected: map[string]interface{}{ + "min_wal_size": "1024MB", + "max_wal_size": "4096MB", + "default_statistics_target": "100", + }, + }, + } { + cluster := DefaultCluster() + for i := range cluster.Spec.InstanceSets { + cluster.Spec.InstanceSets[i].Default(i) + } + cluster.Spec.AutoPGTune.ApplicationType = tt.apptype + actual := GetPGTuneConfigParameters(cluster) + + assert.DeepEqual(t, tt.expected, actual) + } +} + +func TestDifferentParameters(t *testing.T) { + for _, tt := range []struct { + name string + apptype string + hdtype string + cpu string + memory string + expected map[string]interface{} + }{ + { + name: "1-cpu-not-tuned", + apptype: "web", + hdtype: "ssd", + cpu: "1000m", + memory: "2Gi", + expected: map[string]interface{}{ + "shared_buffers": "512MB", + "effective_cache_size": "1536MB", + "maintenance_work_mem": "128MB", + "wal_buffers": "16MB", + "default_statistics_target": "100", + "random_page_cost": "1.1", + "effective_io_concurrency": "200", + "work_mem": "1310kB", + "min_wal_size": "1024MB", + "max_wal_size": "4096MB", + }, + }, + { + name: "no-app-type-assume-mixed", + apptype: "", + hdtype: "hdd", + cpu: "4000m", + memory: "8Gi", + expected: map[string]interface{}{ + "shared_buffers": "2048MB", + "effective_cache_size": "6144MB", + "maintenance_work_mem": "512MB", + "wal_buffers": "16MB", + "default_statistics_target": "100", + "random_page_cost": "4", + "effective_io_concurrency": "2", + "work_mem": "5242kB", + "min_wal_size": "1024MB", + "max_wal_size": "4096MB", + "max_parallel_maintenance_workers": "2", + "max_parallel_workers": "4", + "max_parallel_workers_per_gather": "2", + "max_worker_processes": "4", + }, + }, + { + name: "nothing-is-cool", + apptype: "", + hdtype: "", + cpu: "0", + memory: "0", + expected: map[string]interface{}{ + "default_statistics_target": "100", + "min_wal_size": "1024MB", + "max_wal_size": "4096MB", + }, + }, + { + name: "all-but-memory", + apptype: "oltp", + hdtype: "san", + cpu: "7000m", + memory: "0", + expected: map[string]interface{}{ + "default_statistics_target": "100", + "random_page_cost": "1.1", + "effective_io_concurrency": "300", + "min_wal_size": "2048MB", + "max_wal_size": "8192MB", + "max_parallel_maintenance_workers": "4", + "max_parallel_workers": "7", + "max_parallel_workers_per_gather": "4", + "max_worker_processes": "7", + }, + }, + } { + cluster := DefaultCluster() + for i := range cluster.Spec.InstanceSets { + cluster.Spec.InstanceSets[i].Default(i) + } + if tt.apptype != "" { + cluster.Spec.AutoPGTune.ApplicationType = tt.apptype + } + if tt.hdtype != "" { + cluster.Spec.AutoPGTune.HDType = &tt.hdtype + } + for i := range cluster.Spec.InstanceSets { + cluster.Spec.InstanceSets[i].Resources = v1.ResourceRequirements{} + cluster.Spec.InstanceSets[i].Resources = v1.ResourceRequirements{Requests: v1.ResourceList{ + "memory": resource.MustParse(tt.memory), + "cpu": resource.MustParse(tt.cpu), + }, + } + + actual := GetPGTuneConfigParameters(cluster) + assert.DeepEqual(t, tt.expected, actual) + } + } +} diff --git a/pkg/apis/postgres-operator.crunchydata.com/v1beta1/postgrescluster_types.go b/pkg/apis/postgres-operator.crunchydata.com/v1beta1/postgrescluster_types.go index 421852c295..d6451da749 100644 --- a/pkg/apis/postgres-operator.crunchydata.com/v1beta1/postgrescluster_types.go +++ b/pkg/apis/postgres-operator.crunchydata.com/v1beta1/postgrescluster_types.go @@ -161,6 +161,13 @@ type PostgresClusterSpec struct { // +listMapKey=name // +optional Users []PostgresUserSpec `json:"users,omitempty"` + + // If set, PostgreSQL will be automatically tuned according to its' resources. + // Memory and CPU values are tuned according to the instances' requests. + // Storage is tuned according to HDType property + // More information: https://pgtune.leopard.in.ua/#/ and https://github.com/le0pard/pgtune + // +optional + AutoPGTune *PostgresClusterPGTune `json:"autoPgTune,omitempty"` } // DataSource defines data sources for a new PostgresCluster. @@ -265,6 +272,20 @@ type PostgresClusterDataSource struct { Tolerations []corev1.Toleration `json:"tolerations,omitempty"` } +type PostgresClusterPGTune struct { + // Storage disk type, can be either ssd,hdd or san + // if not set, storage optimization will not be applied. + // +kubebuilder:validation:Enum={ssd,hdd,san} + HDType *string `json:"hdType,omitempty"` + // What is the purpose of the database's application? can be either + // "web" for web application, "oltp" for Online Transaction Processing, + // "dw" for Data Warehouse, "desktop" for Desktop Application or "mixed" + // if not set, "mixed" is assumed + // +kubebuilder:validation:Enum={web,oltp,dw,desktop,mixed} + // +kubebuilder:default="mixed" + ApplicationType string `json:"applicationType,omitempty"` +} + // Default defines several key default values for a Postgres cluster. func (s *PostgresClusterSpec) Default() { for i := range s.InstanceSets { From f23ec9c19e661792914699171b58dcb3279c9926 Mon Sep 17 00:00:00 2001 From: edenkoveshi Date: Wed, 20 Oct 2021 21:13:14 +0300 Subject: [PATCH 2/2] +optional flags --- .../v1beta1/postgrescluster_types.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pkg/apis/postgres-operator.crunchydata.com/v1beta1/postgrescluster_types.go b/pkg/apis/postgres-operator.crunchydata.com/v1beta1/postgrescluster_types.go index d6451da749..c521e56275 100644 --- a/pkg/apis/postgres-operator.crunchydata.com/v1beta1/postgrescluster_types.go +++ b/pkg/apis/postgres-operator.crunchydata.com/v1beta1/postgrescluster_types.go @@ -276,6 +276,7 @@ type PostgresClusterPGTune struct { // Storage disk type, can be either ssd,hdd or san // if not set, storage optimization will not be applied. // +kubebuilder:validation:Enum={ssd,hdd,san} + // +optional HDType *string `json:"hdType,omitempty"` // What is the purpose of the database's application? can be either // "web" for web application, "oltp" for Online Transaction Processing, @@ -283,6 +284,7 @@ type PostgresClusterPGTune struct { // if not set, "mixed" is assumed // +kubebuilder:validation:Enum={web,oltp,dw,desktop,mixed} // +kubebuilder:default="mixed" + // +optional ApplicationType string `json:"applicationType,omitempty"` }