From 4658de4040766df6cfd7d6c12e2fd2a5abefca16 Mon Sep 17 00:00:00 2001 From: Kalle Fagerberg Date: Thu, 6 Oct 2022 12:08:35 +0200 Subject: [PATCH 1/7] Added YAML manifests to use in-cluster prometheus scraping Signed-off-by: Kalle Fagerberg --- monitor-pv.yaml | 114 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 114 insertions(+) create mode 100644 monitor-pv.yaml diff --git a/monitor-pv.yaml b/monitor-pv.yaml new file mode 100644 index 0000000..3170ea0 --- /dev/null +++ b/monitor-pv.yaml @@ -0,0 +1,114 @@ +# monitor-pv will launch as a daemonset. +# Make sure to update namespace on ClusterRoleBinding below. +apiVersion: v1 +kind: ServiceAccount +metadata: + name: openebs-monitor-pv +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: openebs-monitor-pv +rules: +- apiGroups: + - "" + resources: + - persistentvolumes + verbs: + - list + - get +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: openebs-monitor-pv +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: openebs-monitor-pv +subjects: +- kind: ServiceAccount + name: openebs-monitor-pv + namespace: openebs # NOTE: Update namespace here if deploying to a different namespace +--- +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: openebs-monitor-pv +spec: + selector: + matchLabels: + app: openebs-monitor-pv + template: + metadata: + labels: + app: openebs-monitor-pv + annotations: + prometheus.io/path: /scrape.txt + prometheus.io/port: "80" + prometheus.io/scrape: "true" + name: openebs-monitor-pv + spec: + serviceAccount: openebs-monitor-pv + containers: + - name: monitor-pv + image: openebs/monitor-pv:ci + imagePullPolicy: Always + env: + - name: TEXTFILE_PATH + value: /shared_vol/scrape.txt + - name: COLLECT_INTERVAL + value: "10" + - name: PROVISIONER_WHITELIST + value: "openebs.io/provisioner-iscsi,openebs.io/local" + command: + - /bin/bash + args: + - -c + - ./textfile_collector.sh + resources: + requests: + memory: "40M" + cpu: "200m" + limits: + memory: "100M" + cpu: "200m" + volumeMounts: + - mountPath: /host/proc + name: proc + - mountPath: /host/sys + name: sys + - mountPath: /host/root + mountPropagation: HostToContainer + name: root + readOnly: true + - mountPath: /shared_vol + name: shared + - image: nginx:alpine + name: nginx + ports: + - containerPort: 80 + name: scrape + resources: + requests: + memory: "60M" + cpu: "1m" + limits: + memory: "100M" + cpu: "10m" + ## Uncomment this to also run on control-plane nodes + #tolerations: + # - effect: NoSchedule + # operator: Exists + volumes: + - name: proc + hostPath: + path: /proc + - name: sys + hostPath: + path: /sys + - name: root + hostPath: + path: / + - name: shared + emptyDir: {} From 93f630751a7e40071275802952fb18033f1508cb Mon Sep 17 00:00:00 2001 From: Kalle Fagerberg Date: Thu, 6 Oct 2022 12:08:43 +0200 Subject: [PATCH 2/7] Updated script Signed-off-by: Kalle Fagerberg --- textfile_collector.sh | 46 ++++++++++++++++++++++--------------------- 1 file changed, 24 insertions(+), 22 deletions(-) diff --git a/textfile_collector.sh b/textfile_collector.sh index 9b3f3af..e5b74cb 100755 --- a/textfile_collector.sh +++ b/textfile_collector.sh @@ -1,12 +1,11 @@ #!/bin/bash -FILEPATH=${TEXTFILE_PATH:=/shared_vol} +FILEPATH=${TEXTFILE_PATH:=/shared_vol/scrape.txt} INTERVAL=${COLLECT_INTERVAL:=10} PROVISIONERS=${PROVISIONER_WHITELIST:=openebs.io/local} ## calculate_pv_capacity obtains the size of a PV in bytes function calculate_pv_capacity(){ - if [[ ${size_in_spec} =~ "i" ]]; then unit=$(echo "${size_in_spec: -2}") else @@ -30,29 +29,17 @@ function calculate_pv_capacity(){ ## collect_pv_capacity_metrics collects the PV capacity metrics function collect_pv_capacity_metrics(){ - - ##TODO: We clear the file and then proceed to derive the metrics in the for loop below. - ## If, the block below takes time, it may cause a few seconds of "no-metrics". - ## This needs to be optimized. Preferable approach is to replace values v/s recreating the file. - > ${FILEPATH}/pv_size.prom - for i in ${pv_list[@]} do pvc_name=$(kubectl get pv ${i} -o custom-columns=:spec.claimRef.name --no-headers | tr '[:upper:]' '[:lower:]') size_in_spec=$(kubectl get pv ${i} -o custom-columns=:spec.capacity.storage --no-headers | tr '[:upper:]' '[:lower:]') size_in_bytes=$(calculate_pv_capacity ${size_in_spec};) - echo "pv_capacity_bytes{persistentvolume=\"${i}\",persistentvolumeclaim=\"${pvc_name}\"} ${size_in_bytes}" >> ${FILEPATH}/pv_size.prom + echo "pv_capacity_bytes{persistentvolume=\"${i}\",persistentvolumeclaim=\"${pvc_name}\"} ${size_in_bytes}" done } ## collect_pv_utilization_metrics collects the PV utilization metrics function collect_pv_utilization_metrics(){ - - ##TODO: We clear the file and then proceed to derive the metrics in the for loop below. - ## If, the block below takes time, it may cause a few seconds of "no-metrics". - ## This needs to be optimized. Preferable approach is to replace values v/s recreating the file. - > ${FILEPATH}/pv_used.prom - declare -a pv_mount_list=() for i in ${pv_list[@]} @@ -60,7 +47,7 @@ function collect_pv_utilization_metrics(){ pv_mount_list+=($(findmnt --df | grep ${i} | grep '/var/lib/kubelet/pods' | head -1 | awk '{print $NF}')) done - echo "mount list: ${pv_mount_list[@]}" + echo "mount list: ${pv_mount_list[@]}" >&2 for i in ${pv_mount_list[@]} do ## Get mount point utilization in bytes @@ -68,10 +55,26 @@ function collect_pv_utilization_metrics(){ utilization=$(echo ${mount_data}| cut -d " " -f 1) pv_name=$(basename $(echo ${mount_data} | cut -d " " -f 2)) pvc_name=$(kubectl get pv ${pv_name} -o custom-columns=:spec.claimRef.name --no-headers | tr '[:upper:]' '[:lower:]') - echo "pv_utilization_bytes{persistentvolume=\"${pv_name}\",persistentvolumeclaim=\"${pvc_name}\"} ${utilization}" >> ${FILEPATH}/pv_used.prom + echo "pv_utilization_bytes{persistentvolume=\"${pv_name}\",persistentvolumeclaim=\"${pvc_name}\"} ${utilization}" done } +collect_and_write_to_file() { + cap_metrics="$(collect_pv_capacity_metrics)" + util_metrics="$(collect_pv_utilization_metrics)" + + cat < "${FILEPATH}" +# Prometheus metrics generated by monitor-pv at $(date) + +# Capacity metrics +${cap_metrics} + +# Utilization metrics +${util_metrics} +END + +} + while true do provisioner_list=$(echo ${PROVISIONERS} | tr ',' ' ') @@ -87,14 +90,13 @@ do pv_list+=(${i}) fi done - + echo "No. of PVs by specified provisioners: ${#pv_list[@]}" - + if [[ ${#pv_list[@]} -ne 0 ]]; then echo "PV List: ${pv_list[@]}" - collect_pv_capacity_metrics; - collect_pv_utilization_metrics; + collect_and_write_to_file fi - + sleep ${INTERVAL} done From e8096dc902bf5573c165d9e50551d9679a2da657 Mon Sep 17 00:00:00 2001 From: Kalle Fagerberg Date: Thu, 6 Oct 2022 12:08:54 +0200 Subject: [PATCH 3/7] Removed old node-exporter manifests Signed-off-by: Kalle Fagerberg --- node-exporter-pv-metrics.yaml | 102 ---------------------------------- 1 file changed, 102 deletions(-) delete mode 100644 node-exporter-pv-metrics.yaml diff --git a/node-exporter-pv-metrics.yaml b/node-exporter-pv-metrics.yaml deleted file mode 100644 index 3a26f7c..0000000 --- a/node-exporter-pv-metrics.yaml +++ /dev/null @@ -1,102 +0,0 @@ -# monitor-pv will be launch as daemonset. -# choose appropriate namespace and serviceaccount. defaults to ns: openebs, sa: openebs-maya-operator -apiVersion: apps/v1 -kind: DaemonSet -metadata: - name: monitor-pv - namespace: openebs -spec: - selector: - matchLabels: - app: monitor-pv - template: - metadata: - labels: - app: monitor-pv - name: monitor-pv - spec: - serviceAccount: openebs-maya-operator - containers: - - image: quay.io/prometheus/node-exporter:v0.18.1 - args: - - --path.procfs=/host/proc - - --path.sysfs=/host/sys - - --path.rootfs=/host/root - - --collector.filesystem.ignored-mount-points=^/(dev|proc|sys|var/lib|run|boot|home/kubernetes/.+)($|/) - - --collector.filesystem.ignored-fs-types=^(autofs|binfmt_misc|cgroup|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|mqueue|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|sysfs|tracefs)$ - - --collector.textfile.directory=/shared_vol - name: node-exporter - ports: - - containerPort: 9100 - hostPort: 9100 - name: scrape - resources: - requests: - # A memory request of 250M means it will try to ensure minimum - # 250MB RAM . - memory: "128M" - # A cpu request of 128m means it will try to ensure minimum - # .125 CPU; where 1 CPU means : - # 1 *Hyperthread* on a bare-metal Intel processor with Hyperthreading - cpu: "128m" - limits: - memory: "700M" - cpu: "500m" - volumeMounts: - # All the application data stored in data-disk - - name: proc - mountPath: /host/proc - readOnly: false - # Root disk is where OS(Node) is installed - - name: sys - mountPath: /host/sys - readOnly: false - - name: root - mountPath: /host/root - mountPropagation: HostToContainer - readOnly: true - - name: tmpvol - mountPath: /shared_vol - - name: monitor-pv - image: openebs/monitor-pv:ci - imagePullPolicy: Always - env: - - name: TEXTFILE_PATH - value: /shared_vol - - name: COLLECT_INTERVAL - value: "10" - - name: PROVISIONER_WHITELIST - value: "openebs.io/provisioner-iscsi,openebs.io/local" - command: - - /bin/bash - args: - - -c - - ./textfile_collector.sh - volumeMounts: - - mountPath: /host/proc - name: proc - - mountPath: /host/sys - name: sys - - mountPath: /host/root - mountPropagation: HostToContainer - name: root - readOnly: true - - mountPath: /shared_vol - name: tmpvol - tolerations: - - effect: NoSchedule - operator: Exists - volumes: - - name: proc - hostPath: - path: /proc - - name: sys - hostPath: - path: /sys - - name: root - hostPath: - path: / - - name: tmpvol - emptyDir: {} - hostNetwork: true - hostPID: true From 841c440b391b70db36f6b763c1cd90bae6e0a61b Mon Sep 17 00:00:00 2001 From: Kalle Fagerberg Date: Thu, 6 Oct 2022 12:16:51 +0200 Subject: [PATCH 4/7] Updated README.md Signed-off-by: Kalle Fagerberg --- README.md | 80 ++++++++++++++++++++----------------------------------- 1 file changed, 29 insertions(+), 51 deletions(-) diff --git a/README.md b/README.md index ebd54b5..254c1bd 100644 --- a/README.md +++ b/README.md @@ -1,68 +1,46 @@ # monitor-pv + custom stats collector for OpenEBS persistent volumes (jiva, localpv) -### Pre-requisite +## Pre-requisite + - openebs (https://docs.openebs.io/) -### How to use -- Run `kubectl apply -f node-exporter-pv-metrics.yaml` +## How to use + +- Run `kubectl apply -f monitor-pv.yaml` - Verify if the pods are up and running. -
+
+```console
 $ kubectl get pods -n openebs 
-NAME                                                              READY   STATUS    RESTARTS   AGE
-cspc-operator-6c4cc7c64d-698ps                                    1/1     Running   0          6d23h
-cvc-operator-77d749c559-9phff                                     1/1     Running   0          6d23h
-maya-apiserver-5fb947d74d-r9skp                                   1/1     Running   0          6d23h
-monitor-pv-bvzgv                                                  2/2     Running   0          8s
-monitor-pv-nk76b                                                  2/2     Running   0          11s
-monitor-pv-wnvp5                                                  2/2     Running   0          8s
-openebs-admission-server-6c4b4998f8-zcg9n                         1/1     Running   0          6d23h
-openebs-localpv-provisioner-5b744fc789-5wr8d                      1/1     Running   0          6d23h
-openebs-ndm-g474w                                                 1/1     Running   0          6d23h
-openebs-ndm-k2nnp                                                 1/1     Running   0          6d23h
-openebs-ndm-operator-b58c79cc5-z8zw6                              1/1     Running   1          6d23h
-openebs-ndm-rwzrb                                                 1/1     Running   0          6d23h
-openebs-provisioner-54d45b55db-rt5rv                              1/1     Running   0          6d23h
-openebs-snapshot-operator-6d4f5d7688-6g7zw                        2/2     Running   0          6d23h
-pvc-dd03f0ae-731c-4f78-bdbf-86485f32ab3d-ctrl-89b44f6cb-pbnmk     2/2     Running   0          40h
-pvc-dd03f0ae-731c-4f78-bdbf-86485f32ab3d-rep-1-857b65c68d-qrdx9   1/1     Running   0          40h
-pvc-dd03f0ae-731c-4f78-bdbf-86485f32ab3d-rep-2-58c4f54f7-m6n45    1/1     Running   0          40h
+NAME                READY   STATUS    RESTARTS   AGE
+...
+monitor-pv-bvzgv    2/2     Running   0          8s
+monitor-pv-nk76b    2/2     Running   0          11s
+monitor-pv-wnvp5    2/2     Running   0          8s
+...
+```
 
-
+## How it works -### How it works -The monitor PV daemonset pods consist of two containers i.e node-exporter and monitor-pv. The monitor-pv collects the PV size and PV utilization information & places it in a file on the shared mount. The node exporter uses its text-file collector to expose this data as metrics. +The monitor PV daemonset pods consist of two containers i.e nginx and monitor-pv. The monitor-pv collects the PV size via `kubectl` and PV utilization information via `du`, and then places it in a text file on the shared emptydir mount. The nginx exposes this text file via HTTP so that Prometheus can scrape it as metrics. It exposes two metrics **pv_capacity_bytes** and **pv_utilization_bytes**. -### Prometheus Configuration -To scrape the metrics in prometheus add this configuration in prometheus configuration file: -
-    - job_name: 'monitor-pv'
-      kubernetes_sd_configs:
-      - role: pod
-      relabel_configs:
-      - source_labels: [__meta_kubernetes_pod_label_app]
-        regex: monitor-pv
-        action: keep
-
+The pods are configured with Prometheus annotations so that a Prometheus instance installed in the cluster knows how to scrape: + +```yaml +apiVersion: v1 +kind: Pod +metadata: + annotations: + prometheus.io/path: /scrape.txt + prometheus.io/port: "80" + prometheus.io/scrape: "true" +``` -If you want to scrape only monitor-pv metrics and drop all other node-exporter metrics use the below configuration: -
-    - job_name: 'monitor-pv'
-      kubernetes_sd_configs:
-      - role: pod
-      relabel_configs:
-      - source_labels: [__meta_kubernetes_pod_label_app]
-        regex: monitor-pv
-        action: keep
-      metric_relabel_configs:
-      - source_labels: [__name__]
-        regex: '(pv_capacity_bytes|pv_utilization_bytes)'
-        action: keep
-
+## Example -### Example: ![monitor-pv-1](https://user-images.githubusercontent.com/29499601/81772797-67141a80-9504-11ea-901b-fe165900d60c.png) ![monitor-pv-2](https://user-images.githubusercontent.com/29499601/81772848-8a3eca00-9504-11ea-8d0b-e7a572a06aef.png) From f40f707c5ce5e37737d1447c54f8190419a0f5c7 Mon Sep 17 00:00:00 2001 From: Kalle Fagerberg Date: Thu, 6 Oct 2022 12:21:34 +0200 Subject: [PATCH 5/7] Removed trailing spaces Signed-off-by: Kalle Fagerberg --- textfile_collector.sh | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/textfile_collector.sh b/textfile_collector.sh index e5b74cb..93f64a8 100755 --- a/textfile_collector.sh +++ b/textfile_collector.sh @@ -10,10 +10,10 @@ function calculate_pv_capacity(){ unit=$(echo "${size_in_spec: -2}") else unit=$(echo "${size_in_spec: -1}") - fi + fi + + case "${unit}" in - case "${unit}" in - g|gi) echo $((1024*1024*1024*$(echo $1 | tr -d ${unit}))) ;; m|mi) echo $((1024*1024*$(echo $1 | tr -d ${unit}))) @@ -62,7 +62,7 @@ function collect_pv_utilization_metrics(){ collect_and_write_to_file() { cap_metrics="$(collect_pv_capacity_metrics)" util_metrics="$(collect_pv_utilization_metrics)" - + cat < "${FILEPATH}" # Prometheus metrics generated by monitor-pv at $(date) @@ -72,7 +72,7 @@ ${cap_metrics} # Utilization metrics ${util_metrics} END - + } while true @@ -86,17 +86,17 @@ do ## Select only those PVs that are provisioned by the whitelisted provisioners ## Nested conditions in jsonpath filters are not supported yet. Ref: https://github.com/kubernetes/kubernetes/issues/20352 if [[ ${provisioner_list} =~ $(kubectl get pv ${i} -o jsonpath='{.metadata.annotations.pv\.kubernetes\.io/provisioned-by}') ]] - then + then pv_list+=(${i}) fi done - + echo "No. of PVs by specified provisioners: ${#pv_list[@]}" - + if [[ ${#pv_list[@]} -ne 0 ]]; then echo "PV List: ${pv_list[@]}" collect_and_write_to_file fi - + sleep ${INTERVAL} done From 678d18b5cdc3a011bd27bb215949f58fe0f10623 Mon Sep 17 00:00:00 2001 From: Kalle Fagerberg Date: Thu, 6 Oct 2022 12:35:25 +0200 Subject: [PATCH 6/7] Use numfmt instead of own function Signed-off-by: Kalle Fagerberg --- textfile_collector.sh | 25 +------------------------ 1 file changed, 1 insertion(+), 24 deletions(-) diff --git a/textfile_collector.sh b/textfile_collector.sh index 93f64a8..675fe03 100755 --- a/textfile_collector.sh +++ b/textfile_collector.sh @@ -4,36 +4,13 @@ FILEPATH=${TEXTFILE_PATH:=/shared_vol/scrape.txt} INTERVAL=${COLLECT_INTERVAL:=10} PROVISIONERS=${PROVISIONER_WHITELIST:=openebs.io/local} -## calculate_pv_capacity obtains the size of a PV in bytes -function calculate_pv_capacity(){ - if [[ ${size_in_spec} =~ "i" ]]; then - unit=$(echo "${size_in_spec: -2}") - else - unit=$(echo "${size_in_spec: -1}") - fi - - case "${unit}" in - - g|gi) echo $((1024*1024*1024*$(echo $1 | tr -d ${unit}))) - ;; - m|mi) echo $((1024*1024*$(echo $1 | tr -d ${unit}))) - ;; - k|ki) echo $((1024*$(echo $1 | tr -d ${unit}))) - ;; - b|bi) echo $1 | tr -d ${unit} - ;; - *) echo 0 - ;; - esac -} - ## collect_pv_capacity_metrics collects the PV capacity metrics function collect_pv_capacity_metrics(){ for i in ${pv_list[@]} do pvc_name=$(kubectl get pv ${i} -o custom-columns=:spec.claimRef.name --no-headers | tr '[:upper:]' '[:lower:]') size_in_spec=$(kubectl get pv ${i} -o custom-columns=:spec.capacity.storage --no-headers | tr '[:upper:]' '[:lower:]') - size_in_bytes=$(calculate_pv_capacity ${size_in_spec};) + size_in_bytes=$(numfmt --from=auto --to=none ${size_in_spec};) echo "pv_capacity_bytes{persistentvolume=\"${i}\",persistentvolumeclaim=\"${pvc_name}\"} ${size_in_bytes}" done } From 53870db44931adfe70d9d95f86efe98f913b0eb8 Mon Sep 17 00:00:00 2001 From: Kalle Fagerberg Date: Fri, 7 Oct 2022 11:06:54 +0200 Subject: [PATCH 7/7] Fixed to write to file even if no PV's Signed-off-by: Kalle Fagerberg --- textfile_collector.sh | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/textfile_collector.sh b/textfile_collector.sh index 675fe03..4be8ad3 100755 --- a/textfile_collector.sh +++ b/textfile_collector.sh @@ -36,9 +36,9 @@ function collect_pv_utilization_metrics(){ done } -collect_and_write_to_file() { - cap_metrics="$(collect_pv_capacity_metrics)" - util_metrics="$(collect_pv_utilization_metrics)" +write_to_file() { + cap_metrics="${1:-""}" + util_metrics="${2:-""}" cat < "${FILEPATH}" # Prometheus metrics generated by monitor-pv at $(date) @@ -49,7 +49,6 @@ ${cap_metrics} # Utilization metrics ${util_metrics} END - } while true @@ -72,7 +71,11 @@ do if [[ ${#pv_list[@]} -ne 0 ]]; then echo "PV List: ${pv_list[@]}" - collect_and_write_to_file + cap_metrics="$(collect_pv_capacity_metrics)" + util_metrics="$(collect_pv_utilization_metrics)" + write_to_file "$cap_metrics" "$util_metrics" + else + write_to_file "" "" fi sleep ${INTERVAL}