From b9a482dd579839fe606a2815e90922820be87549 Mon Sep 17 00:00:00 2001 From: ericsyh Date: Sun, 22 Sep 2024 21:45:00 +0800 Subject: [PATCH 1/4] feat: add autoscaling on chart Signed-off-by: ericsyh --- .../templates/bookkeeper/bookkeeper-cluster.yaml | 11 +++++++++++ .../templates/broker/broker-cluster.yaml | 11 +++++++++++ charts/sn-platform-slim/values.yaml | 14 ++++++++++++++ 3 files changed, 36 insertions(+) diff --git a/charts/sn-platform-slim/templates/bookkeeper/bookkeeper-cluster.yaml b/charts/sn-platform-slim/templates/bookkeeper/bookkeeper-cluster.yaml index ccbd8685..a8cdc71b 100644 --- a/charts/sn-platform-slim/templates/bookkeeper/bookkeeper-cluster.yaml +++ b/charts/sn-platform-slim/templates/bookkeeper/bookkeeper-cluster.yaml @@ -33,6 +33,17 @@ spec: replicas: {{ .Values.bookkeeper.replicaCount }} image: "{{ .Values.images.bookie.repository }}:{{ .Values.images.bookie.tag }}" imagePullPolicy: {{ .Values.images.bookie.pullPolicy }} +{{- if .Values.bookkeeper.autoScaling.enabled }} + autoScalingPolicy: + minReplicas: {{ .Values.bookkeeper.autoScaling.minReplicas }} + maxReplicas: {{ .Values.bookkeeper.autoScaling.maxReplicas }} +{{- if .Values.bookkeeper.autoScaling.metrics }} + metrics: +{{- with .Values.bookkeeper.autoScaling.metrics }} +{{ toYaml . | indent 4 }} +{{- end }} +{{- end }} +{{- end }} logConfig: level: {{ .Values.bookkeeper.logConfig.level }} format: {{ .Values.bookkeeper.logConfig.format }} diff --git a/charts/sn-platform-slim/templates/broker/broker-cluster.yaml b/charts/sn-platform-slim/templates/broker/broker-cluster.yaml index e6872238..9971568a 100644 --- a/charts/sn-platform-slim/templates/broker/broker-cluster.yaml +++ b/charts/sn-platform-slim/templates/broker/broker-cluster.yaml @@ -35,6 +35,17 @@ spec: replicas: {{ .Values.broker.replicaCount }} image: "{{ .Values.images.broker.repository }}:{{ .Values.images.broker.tag }}" imagePullPolicy: {{ .Values.images.broker.pullPolicy }} +{{- if .Values.broker.autoScaling.enabled }} + autoScalingPolicy: + minReplicas: {{ .Values.broker.autoScaling.minReplicas }} + maxReplicas: {{ .Values.broker.autoScaling.maxReplicas }} +{{- if .Values.broker.autoScaling.metrics }} + metrics: +{{- with .Values.broker.autoScaling.metrics }} +{{ toYaml . | indent 4 }} +{{- end }} +{{- end }} +{{- end }} logConfig: level: {{ .Values.broker.logConfig.level }} format: {{ .Values.broker.logConfig.format }} diff --git a/charts/sn-platform-slim/values.yaml b/charts/sn-platform-slim/values.yaml index 894c1094..f83f9d8d 100644 --- a/charts/sn-platform-slim/values.yaml +++ b/charts/sn-platform-slim/values.yaml @@ -788,6 +788,13 @@ bookkeeper: # annotations that will be added on the BookKeeperCluster CR only. annotations: {} + # The field autoScaling will track the average metrics usage of the Pulsar nodes and adjusts the nodes to keep at the target metrics usage level. + # If the average metrics usage for nodes is over the target, scaling controller will scale out to bring more nodes and if the average metrics for the nodes is less than the target, scaling controller will downscale nodes to save resources. + autoScaling: + enabled: false + minReplicas: 1 + maxReplicas: 4 + metrics: {} # The field logConfig can be used to change the log level and log format of pods. # The logConfig field is optional. If it is not specified, the component will use the default log configuration /pulsar/conf/log4j2.yaml. # f it is specified will dynamically change the log level and log format of the component by changing the CR. @@ -1025,6 +1032,13 @@ broker: # annotations that will be added on the PulsarBroker CR only. annotations: {} + # The field autoScaling will track the average metrics usage of the Pulsar nodes and adjusts the nodes to keep at the target metrics usage level. + # If the average metrics usage for nodes is over the target, scaling controller will scale out to bring more nodes and if the average metrics for the nodes is less than the target, scaling controller will downscale nodes to save resources. + autoScaling: + enabled: false + minReplicas: 1 + maxReplicas: 4 + metrics: {} # The field logConfig can be used to change the log level and log format of pods. # The logConfig field is optional. If it is not specified, the component will use the default log configuration /pulsar/conf/log4j2.yaml. # If it is specified will dynamically change the log level and log format of the component by changing the CR. From 10785027e332b237419e627da138cca4783d847b Mon Sep 17 00:00:00 2001 From: ericsyh Date: Sun, 22 Sep 2024 21:48:47 +0800 Subject: [PATCH 2/4] apply autoscaling on sn-platform Signed-off-by: ericsyh --- .../templates/bookkeeper/bookkeeper-cluster.yaml | 11 +++++++++++ .../templates/broker/broker-cluster.yaml | 11 +++++++++++ charts/sn-platform/values.yaml | 16 ++++++++++++++++ 3 files changed, 38 insertions(+) diff --git a/charts/sn-platform/templates/bookkeeper/bookkeeper-cluster.yaml b/charts/sn-platform/templates/bookkeeper/bookkeeper-cluster.yaml index c9560f59..94af4a76 100644 --- a/charts/sn-platform/templates/bookkeeper/bookkeeper-cluster.yaml +++ b/charts/sn-platform/templates/bookkeeper/bookkeeper-cluster.yaml @@ -33,6 +33,17 @@ spec: replicas: {{ .Values.bookkeeper.replicaCount }} image: "{{ .Values.images.bookie.repository }}:{{ .Values.images.bookie.tag }}" imagePullPolicy: {{ .Values.images.bookie.pullPolicy }} +{{- if .Values.bookkeeper.autoScaling.enabled }} + autoScalingPolicy: + minReplicas: {{ .Values.bookkeeper.autoScaling.minReplicas }} + maxReplicas: {{ .Values.bookkeeper.autoScaling.maxReplicas }} +{{- if .Values.bookkeeper.autoScaling.metrics }} + metrics: +{{- with .Values.bookkeeper.autoScaling.metrics }} +{{ toYaml . | indent 4 }} +{{- end }} +{{- end }} +{{- end }} logConfig: level: {{ .Values.bookkeeper.logConfig.level }} format: {{ .Values.bookkeeper.logConfig.format }} diff --git a/charts/sn-platform/templates/broker/broker-cluster.yaml b/charts/sn-platform/templates/broker/broker-cluster.yaml index eff81fce..b8704cea 100644 --- a/charts/sn-platform/templates/broker/broker-cluster.yaml +++ b/charts/sn-platform/templates/broker/broker-cluster.yaml @@ -35,6 +35,17 @@ spec: replicas: {{ .Values.broker.replicaCount }} image: "{{ .Values.images.broker.repository }}:{{ .Values.images.broker.tag }}" imagePullPolicy: {{ .Values.images.broker.pullPolicy }} +{{- if .Values.broker.autoScaling.enabled }} + autoScalingPolicy: + minReplicas: {{ .Values.broker.autoScaling.minReplicas }} + maxReplicas: {{ .Values.broker.autoScaling.maxReplicas }} +{{- if .Values.broker.autoScaling.metrics }} + metrics: +{{- with .Values.broker.autoScaling.metrics }} +{{ toYaml . | indent 4 }} +{{- end }} +{{- end }} +{{- end }} logConfig: level: {{ .Values.broker.logConfig.level }} format: {{ .Values.broker.logConfig.format }} diff --git a/charts/sn-platform/values.yaml b/charts/sn-platform/values.yaml index 6794800c..44faf8f6 100644 --- a/charts/sn-platform/values.yaml +++ b/charts/sn-platform/values.yaml @@ -862,6 +862,14 @@ bookkeeper: # annotations that will be added on the BookKeeperCluster CR only. annotations: {} + # The field autoScaling will track the average metrics usage of the Pulsar nodes and adjusts the nodes to keep at the target metrics usage level. + # If the average metrics usage for nodes is over the target, scaling controller will scale out to bring more nodes and if the average metrics for the nodes is less than the target, scaling controller will downscale nodes to save resources. + autoScaling: + enabled: false + minReplicas: 1 + maxReplicas: 4 + metrics: {} + # The field logConfig can be used to change the log level and log format of pods. # The logConfig field is optional. If it is not specified, the component will use the default log configuration /pulsar/conf/log4j2.yaml. # If it is specified will dynamically change the log level and log format of the component by changing the CR. @@ -1099,6 +1107,14 @@ broker: # annotations that will be added on the PulsarBroker CR only. annotations: {} + # The field autoScaling will track the average metrics usage of the Pulsar nodes and adjusts the nodes to keep at the target metrics usage level. + # If the average metrics usage for nodes is over the target, scaling controller will scale out to bring more nodes and if the average metrics for the nodes is less than the target, scaling controller will downscale nodes to save resources. + autoScaling: + enabled: false + minReplicas: 1 + maxReplicas: 4 + metrics: {} + # The field logConfig can be used to change the log level and log format of pods. # The logConfig field is optional. If it is not specified, the component will use the default log configuration /pulsar/conf/log4j2.yaml. # If it is specified will dynamically change the log level and log format of the component by changing the CR. From cd358fa51735788eccaea7e6549a24569696792a Mon Sep 17 00:00:00 2001 From: ericsyh Date: Thu, 10 Oct 2024 11:13:42 +0800 Subject: [PATCH 3/4] add autoscaling examples Signed-off-by: ericsyh --- .../autoscaling-custom-metrics.yaml | 59 ++++++++++++++ .../autoscaling-resource-metrics.yaml | 29 +++++++ examples/sn-platform/prometheus.-adapter.yaml | 79 +++++++++++++++++++ 3 files changed, 167 insertions(+) create mode 100644 examples/sn-platform/autoscaling-custom-metrics.yaml create mode 100644 examples/sn-platform/autoscaling-resource-metrics.yaml create mode 100644 examples/sn-platform/prometheus.-adapter.yaml diff --git a/examples/sn-platform/autoscaling-custom-metrics.yaml b/examples/sn-platform/autoscaling-custom-metrics.yaml new file mode 100644 index 00000000..7d9a0bed --- /dev/null +++ b/examples/sn-platform/autoscaling-custom-metrics.yaml @@ -0,0 +1,59 @@ +broker: + autoScaling: + enabled: true + minReplicas: 1 + maxReplicas: 4 + # Deploy the https://prometheus.io/ and https://github.com/kubernetes-sigs/prometheus-adapter to support custom metrics. + # Refer the prometheus.-adapter.yaml how to setup the custom metrics. + metrics: + - pods: + metric: + name: cpu_usage + target: + averageValue: "75" + type: AverageValue + type: Pods + - pods: + metric: + name: network_in_rate_kb + target: + averageValue: "204800" + type: AverageValue + type: Pods + - pods: + metric: + name: network_out_rate_kb + target: + averageValue: "204800" + type: AverageValue + type: Pods + +bookkeeper: + autoScaling: + enabled: true + minReplicas: 1 + maxReplicas: 4 + # Deploy the https://prometheus.io/ and https://github.com/kubernetes-sigs/prometheus-adapter to support custom metrics. + # Refer the prometheus.-adapter.yaml how to setup the custom metrics. + metrics: + - pods: + metric: + name: cpu_usage + target: + averageValue: "75" + type: AverageValue + type: Pods + - pods: + metric: + name: network_in_rate_kb + target: + averageValue: "204800" + type: AverageValue + type: Pods + - pods: + metric: + name: network_out_rate_kb + target: + averageValue: "204800" + type: AverageValue + type: Pods \ No newline at end of file diff --git a/examples/sn-platform/autoscaling-resource-metrics.yaml b/examples/sn-platform/autoscaling-resource-metrics.yaml new file mode 100644 index 00000000..7bf2e169 --- /dev/null +++ b/examples/sn-platform/autoscaling-resource-metrics.yaml @@ -0,0 +1,29 @@ +broker: + autoScaling: + enabled: true + minReplicas: 1 + maxReplicas: 4 + # Deploy the https://github.com/kubernetes-sigs/metrics-server to support resource metrics. + # Metrics server supports the cpu and memory PodMetrics. + metrics: + type: Resource + resource: + name: cpu + target: + type: Utilization + averageUtilization: 70 + +bookkeeper: + autoScaling: + enabled: true + minReplicas: 1 + maxReplicas: 4 + # Deploy the https://github.com/kubernetes-sigs/metrics-server to support resource metrics. + # Metrics server supports the cpu and memory PodMetrics. + metrics: + type: Resource + resource: + name: cpu + target: + type: Utilization + averageUtilization: 70 \ No newline at end of file diff --git a/examples/sn-platform/prometheus.-adapter.yaml b/examples/sn-platform/prometheus.-adapter.yaml new file mode 100644 index 00000000..058d5bc9 --- /dev/null +++ b/examples/sn-platform/prometheus.-adapter.yaml @@ -0,0 +1,79 @@ +prometheus: + url: http://prometheus-server.monitor.svc + port: 80 +listenPort: 8443 +rules: + default: false + custom: + - seriesQuery: '{namespace!="",__name__!~"^container_.*"}' + resources: + template: "<<.Resource>>" + name: + matches: "^(.*)_total" + as: "" + metricsQuery: | + sum by (<<.GroupBy>>) ( + irate ( + <<.Series>>{<<.LabelMatchers>>}[1m] + ) + ) + - seriesQuery: 'container_cpu_usage_seconds_total{namespace!~"(sn-system|kube-system|olm|cert-manager)"}' + seriesFilters: [] + resources: + overrides: + pod: + resource: pod + namespace: + resource: namespace + name: + matches: "container_cpu_usage_seconds_total" + as: "cpu_usage" + metricsQuery: sum(rate(container_cpu_usage_seconds_total{<<.LabelMatchers>>}[5m])) by (<<.GroupBy>>) / (sum(container_spec_cpu_shares{<<.LabelMatchers>>}/1000) by (<<.GroupBy>>)) * 100 + - seriesQuery: 'container_network_receive_bytes_total{namespace!~"(sn-system|kube-system|olm|cert-manager)"}' + seriesFilters: [] + resources: + overrides: + pod: + resource: pod + namespace: + resource: namespace + name: + matches: "container_network_receive_bytes_total" + as: "network_in_rate_kb" + metricsQuery: rate(container_network_receive_bytes_total{<<.LabelMatchers>>}[5m]) / 1024 + - seriesQuery: 'container_network_transmit_bytes_total{namespace!~"(sn-system|kube-system|olm|cert-manager)"}' + seriesFilters: [] + resources: + overrides: + pod: + resource: pod + namespace: + resource: namespace + name: + matches: "container_network_transmit_bytes_total" + as: "network_out_rate_kb" + metricsQuery: rate(container_network_transmit_bytes_total{<<.LabelMatchers>>}[5m]) / 1024 + - seriesQuery: 'container_fs_reads_bytes_total{namespace!~"(sn-system|kube-system|olm|cert-manager)"}' + seriesFilters: [] + resources: + overrides: + pod: + resource: pod + namespace: + resource: namespace + name: + matches: "container_fs_reads_bytes_total" + as: "disk_read_rate_kb" + metricsQuery: sum(rate(container_fs_reads_bytes_total{<<.LabelMatchers>>}[5m])) by (<<.GroupBy>>) / 1024 + - seriesQuery: 'container_fs_writes_bytes_total{namespace!~"(sn-system|kube-system|olm|cert-manager)"}' + seriesFilters: [] + resources: + overrides: + pod: + resource: pod + namespace: + resource: namespace + name: + matches: "container_fs_writes_bytes_total" + as: "disk_write_rate_kb" + metricsQuery: sum(rate(container_fs_writes_bytes_total{<<.LabelMatchers>>}[5m])) by (<<.GroupBy>>) / 1024 \ No newline at end of file From 355481c74fcc1cae0c4f3afac32c7b1b55783348 Mon Sep 17 00:00:00 2001 From: ericsyh Date: Thu, 10 Oct 2024 13:35:53 +0800 Subject: [PATCH 4/4] fix name Signed-off-by: ericsyh --- .../{prometheus.-adapter.yaml => prometheus-adapter.yaml} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename examples/sn-platform/{prometheus.-adapter.yaml => prometheus-adapter.yaml} (100%) diff --git a/examples/sn-platform/prometheus.-adapter.yaml b/examples/sn-platform/prometheus-adapter.yaml similarity index 100% rename from examples/sn-platform/prometheus.-adapter.yaml rename to examples/sn-platform/prometheus-adapter.yaml