diff --git a/Makefile b/Makefile index 73a0131a7..49505d51b 100644 --- a/Makefile +++ b/Makefile @@ -177,4 +177,6 @@ endif VERSION=${VERSION} INSTALL_DOCKER=1 ./integration/test/test.sh VERSION=${VERSION} REGRESSION=1 ./integration/test/test.sh VERSION=${VERSION} ANALYZE_DOCKER_LOGS=1 ./integration/test/test.sh + VERSION=${VERSION} CHECK_METRICS=1 ./integration/test/test.sh + diff --git a/cmd/collectors/zapi/plugins/systemnode/systemnode.go b/cmd/collectors/zapi/plugins/systemnode/systemnode.go index 8a5375bc4..b04440fdc 100644 --- a/cmd/collectors/zapi/plugins/systemnode/systemnode.go +++ b/cmd/collectors/zapi/plugins/systemnode/systemnode.go @@ -68,10 +68,6 @@ func (s *SystemNode) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, * inst.SetLabel("bmc_firmware_version", serviceProcessorMap[nodeName]) } - // update node metrics with partner info - for _, metric := range data.GetMetrics() { - metric.SetLabel("ha_partner", partnerNameMap[metric.GetLabel("node")]) - } return nil, s.client.Metadata, nil } diff --git a/integration/Jenkinsfile b/integration/Jenkinsfile index 0d83fb084..3e237e9ce 100644 --- a/integration/Jenkinsfile +++ b/integration/Jenkinsfile @@ -207,6 +207,7 @@ pipeline { zip -r docker_logs.zip /var/log/harvest cd $WORKSPACE/docker ANALYZE_DOCKER_LOGS=1 bash $WORKSPACE/harvest/integration/test/test.sh + CHECK_METRICS=1 bash $WORKSPACE/harvest/integration/test/test.sh docker ps -q | xargs docker stop | xargs docker rm --force """ archiveArtifacts artifacts: "docker_logs.zip", fingerprint: true @@ -267,6 +268,7 @@ pipeline { zip -r docker_fips_logs.zip /var/log/harvest cd $WORKSPACE/docker ANALYZE_DOCKER_LOGS=1 bash $WORKSPACE/harvest/integration/test/test.sh + CHECK_METRICS=1 bash $WORKSPACE/harvest/integration/test/test.sh docker ps -q | xargs docker stop | xargs docker rm --force """ archiveArtifacts artifacts: "docker_fips_logs.zip", fingerprint: true diff --git a/integration/test/promtool_metrics_test.go b/integration/test/promtool_metrics_test.go new file mode 100644 index 000000000..65fa3c103 --- /dev/null +++ b/integration/test/promtool_metrics_test.go @@ -0,0 +1,52 @@ +package main + +import ( + "errors" + "fmt" + "github.com/Netapp/harvest-automation/test/utils" + "os/exec" + "strings" + "testing" +) + +func TestPrometheusMetrics(t *testing.T) { + utils.SkipIfMissing(t, utils.CheckMetrics) + + ports := []int{12990, 12992, 12993, 12994} + for _, port := range ports { + checkMetrics(t, port) + } +} + +func checkMetrics(t *testing.T, port int) { + cli := fmt.Sprintf(`curl -s http://localhost:%d/metrics | tee /tmp/metrics:%d.txt | promtool check metrics`, port, port) + command := exec.Command("bash", "-c", cli) + output, err := command.CombinedOutput() + + if err != nil { + var ee *exec.ExitError + if !errors.As(err, &ee) { + // An exit code can't be used since we need to ignore metrics that are not valid but can't change + t.Errorf("ERR checking metrics cli=%s err=%v output=%s", cli, err, string(output)) + return + } + } + + if len(output) == 0 { + return + } + + // Read the output, line by line, and check for errors, non-errors are ignored + + lines := strings.Split(string(output), "\n") + for _, line := range lines { + line = strings.TrimSpace(line) + if line == "" { + continue + } + + if strings.Contains(line, "error while linting: ") { + t.Errorf("promtool: %s", line) + } + } +} diff --git a/integration/test/utils/utils.go b/integration/test/utils/utils.go index f2b2894b9..7faf72924 100644 --- a/integration/test/utils/utils.go +++ b/integration/test/utils/utils.go @@ -27,6 +27,7 @@ const ( GrafanaTokeKey = "grafana_api_token" AnalyzeDockerLogs = "ANALYZE_DOCKER_LOGS" + CheckMetrics = "CHECK_METRICS" CopyDockerLogs = "COPY_DOCKER_LOGS" InstallDocker = "INSTALL_DOCKER" InstallNative = "INSTALL_NATIVE"