From 83a064f15940db7cdade098bb5a3e201d04b6586 Mon Sep 17 00:00:00 2001
From: Chris Grindstaff <chris@gstaff.org>
Date: Fri, 1 Nov 2024 08:15:15 -0400
Subject: [PATCH] ci: Harvest should lint metrics with promtool

---
 Makefile                                      |  2 +
 .../zapi/plugins/systemnode/systemnode.go     |  4 --
 integration/Jenkinsfile                       |  2 +
 integration/test/promtool_metrics_test.go     | 52 +++++++++++++++++++
 integration/test/utils/utils.go               |  1 +
 5 files changed, 57 insertions(+), 4 deletions(-)
 create mode 100644 integration/test/promtool_metrics_test.go

diff --git a/Makefile b/Makefile
index 73a0131a7..49505d51b 100644
--- a/Makefile
+++ b/Makefile
@@ -177,4 +177,6 @@ endif
 	VERSION=${VERSION} INSTALL_DOCKER=1 ./integration/test/test.sh
 	VERSION=${VERSION} REGRESSION=1 ./integration/test/test.sh
 	VERSION=${VERSION} ANALYZE_DOCKER_LOGS=1 ./integration/test/test.sh
+	VERSION=${VERSION} CHECK_METRICS=1 ./integration/test/test.sh
+
 
diff --git a/cmd/collectors/zapi/plugins/systemnode/systemnode.go b/cmd/collectors/zapi/plugins/systemnode/systemnode.go
index 8a5375bc4..b04440fdc 100644
--- a/cmd/collectors/zapi/plugins/systemnode/systemnode.go
+++ b/cmd/collectors/zapi/plugins/systemnode/systemnode.go
@@ -68,10 +68,6 @@ func (s *SystemNode) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, *
 		inst.SetLabel("bmc_firmware_version", serviceProcessorMap[nodeName])
 	}
 
-	// update node metrics with partner info
-	for _, metric := range data.GetMetrics() {
-		metric.SetLabel("ha_partner", partnerNameMap[metric.GetLabel("node")])
-	}
 	return nil, s.client.Metadata, nil
 }
 
diff --git a/integration/Jenkinsfile b/integration/Jenkinsfile
index 0d83fb084..3e237e9ce 100644
--- a/integration/Jenkinsfile
+++ b/integration/Jenkinsfile
@@ -207,6 +207,7 @@ pipeline {
                                         zip -r docker_logs.zip /var/log/harvest
                                         cd $WORKSPACE/docker
                                         ANALYZE_DOCKER_LOGS=1 bash $WORKSPACE/harvest/integration/test/test.sh
+                                        CHECK_METRICS=1 bash $WORKSPACE/harvest/integration/test/test.sh
                                         docker ps -q | xargs docker stop | xargs docker rm --force
                                     """
                                     archiveArtifacts artifacts: "docker_logs.zip", fingerprint: true
@@ -267,6 +268,7 @@ pipeline {
                                         zip -r docker_fips_logs.zip /var/log/harvest
                                         cd $WORKSPACE/docker
                                         ANALYZE_DOCKER_LOGS=1 bash $WORKSPACE/harvest/integration/test/test.sh
+                                        CHECK_METRICS=1 bash $WORKSPACE/harvest/integration/test/test.sh
                                         docker ps -q | xargs docker stop | xargs docker rm --force
                                     """
                                     archiveArtifacts artifacts: "docker_fips_logs.zip", fingerprint: true
diff --git a/integration/test/promtool_metrics_test.go b/integration/test/promtool_metrics_test.go
new file mode 100644
index 000000000..65fa3c103
--- /dev/null
+++ b/integration/test/promtool_metrics_test.go
@@ -0,0 +1,52 @@
+package main
+
+import (
+	"errors"
+	"fmt"
+	"github.com/Netapp/harvest-automation/test/utils"
+	"os/exec"
+	"strings"
+	"testing"
+)
+
+func TestPrometheusMetrics(t *testing.T) {
+	utils.SkipIfMissing(t, utils.CheckMetrics)
+
+	ports := []int{12990, 12992, 12993, 12994}
+	for _, port := range ports {
+		checkMetrics(t, port)
+	}
+}
+
+func checkMetrics(t *testing.T, port int) {
+	cli := fmt.Sprintf(`curl -s http://localhost:%d/metrics | tee /tmp/metrics:%d.txt | promtool check metrics`, port, port)
+	command := exec.Command("bash", "-c", cli)
+	output, err := command.CombinedOutput()
+
+	if err != nil {
+		var ee *exec.ExitError
+		if !errors.As(err, &ee) {
+			// An exit code can't be used since we need to ignore metrics that are not valid but can't change
+			t.Errorf("ERR checking metrics cli=%s err=%v output=%s", cli, err, string(output))
+			return
+		}
+	}
+
+	if len(output) == 0 {
+		return
+	}
+
+	// Read the output, line by line, and check for errors, non-errors are ignored
+
+	lines := strings.Split(string(output), "\n")
+	for _, line := range lines {
+		line = strings.TrimSpace(line)
+		if line == "" {
+			continue
+		}
+
+		if strings.Contains(line, "error while linting: ") {
+			t.Errorf("promtool: %s", line)
+		}
+	}
+}
diff --git a/integration/test/utils/utils.go b/integration/test/utils/utils.go
index f2b2894b9..7faf72924 100644
--- a/integration/test/utils/utils.go
+++ b/integration/test/utils/utils.go
@@ -27,6 +27,7 @@ const (
 	GrafanaTokeKey = "grafana_api_token"
 
 	AnalyzeDockerLogs = "ANALYZE_DOCKER_LOGS"
+	CheckMetrics      = "CHECK_METRICS"
 	CopyDockerLogs    = "COPY_DOCKER_LOGS"
 	InstallDocker     = "INSTALL_DOCKER"
 	InstallNative     = "INSTALL_NATIVE"