Merge branch 'main' into rg2-workload-dash

docs: incorporate PaulS panel descriptions. Thanks Paul!

.github/workflows/go.yml

@@ -1,7 +1,7 @@
 name: Build, Test, Lint License
 
 env:
-  GO_VERSION: "1.20.5"
+  GO_VERSION: "1.20.6"
 
 on:
   push:

.github/workflows/golangci-lint.yml

@@ -19,7 +19,7 @@ jobs:
     steps:
       - uses: actions/setup-go@v4
         with:
-          go-version: '1.20.3'
+          go-version: '1.20.6'
       - uses: actions/checkout@v3
       - name: golangci-lint
         uses: golangci/golangci-lint-action@v3

.golangci.yml

@@ -28,6 +28,8 @@ linters:
 issues:
   max-issues-per-linter: 0
   max-same-issues: 0
+  include:
+    - EXC0009
 
 output:
   format: github-actions

cmd/admin/admin.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"crypto/tls"
 	"encoding/json"
+	"errors"
 	"fmt"
 	"github.com/netapp/harvest/v2/pkg/conf"
 	"github.com/netapp/harvest/v2/pkg/logging"
@@ -68,15 +69,15 @@ func (a *Admin) startServer() {
 		Msg("Admin node started")
 
 	if a.httpSD.TLS.KeyFile != "" {
-		if err := server.ListenAndServeTLS(a.httpSD.TLS.CertFile, a.httpSD.TLS.KeyFile); err != nil && err != http.ErrServerClosed {
+		if err := server.ListenAndServeTLS(a.httpSD.TLS.CertFile, a.httpSD.TLS.KeyFile); err != nil && !errors.Is(err, http.ErrServerClosed) {
 			a.logger.Fatal().Err(err).
 				Str("listen", a.listen).
 				Str("ssl_cert", a.httpSD.TLS.CertFile).
 				Str("ssl_key", a.httpSD.TLS.KeyFile).
 				Msg("Admin node could not listen")
 		}
 	} else {
-		if err := server.ListenAndServe(); err != nil && err != http.ErrServerClosed {
+		if err := server.ListenAndServe(); err != nil && !errors.Is(err, http.ErrServerClosed) {
 			a.logger.Fatal().Err(err).
 				Str("listen", a.listen).
 				Msg("Admin node could not listen")

cmd/collectors/commonutils.go

@@ -169,8 +169,9 @@ func SetNameservice(nsDB, nsSource, nisDomain string, instance *matrix.Instance)
 }
 
 // IsTimestampOlderThanDuration - timestamp units are micro seconds
-func IsTimestampOlderThanDuration(timestamp float64, duration time.Duration) bool {
-	return time.Since(time.UnixMicro(int64(timestamp))) > duration
+// The `begin` argument lets us virtualize time without requiring sleeps in test code
+func IsTimestampOlderThanDuration(nowish time.Time, timestamp float64, duration time.Duration) bool {
+	return nowish.Sub(time.UnixMicro(int64(timestamp))) > duration
 }
 
 func UpdateLagTime(instance *matrix.Instance, lastTransferSize *matrix.Metric, lagTime *matrix.Metric, logger *logging.Logger) {

cmd/collectors/commonutils_test.go

@@ -238,7 +238,7 @@ func testWithNoPolicyTypeNoRelationshipType(t *testing.T, instance *matrix.Insta
 func testOlderTimestampThanDuration(t *testing.T) {
 	timestamp := float64(time.Now().Add(-20 * time.Minute).UnixMicro())
 	duration := 5 * time.Minute
-	isOlder := IsTimestampOlderThanDuration(timestamp, duration)
+	isOlder := IsTimestampOlderThanDuration(time.Now(), timestamp, duration)
 
 	if isOlder {
 		// OK
@@ -250,7 +250,7 @@ func testOlderTimestampThanDuration(t *testing.T) {
 func testNewerTimestampThanDuration(t *testing.T) {
 	timestamp := float64(time.Now().Add(-1 * time.Hour).UnixMicro())
 	duration := 2 * time.Hour
-	isOlder := IsTimestampOlderThanDuration(timestamp, duration)
+	isOlder := IsTimestampOlderThanDuration(time.Now(), timestamp, duration)
 
 	if !isOlder {
 		// OK

cmd/collectors/ems/ems.go

@@ -351,7 +351,7 @@ func (e *Ems) PollData() (map[string]*matrix.Matrix, error) {
 	e.Logger.Debug().Msg("starting data poll")
 
 	// Update cache for bookend ems
-	e.updateMatrix()
+	e.updateMatrix(time.Now())
 
 	startTime = time.Now()
 
@@ -465,7 +465,6 @@ func (e *Ems) HandleResults(result []gjson.Result, prop map[string][]*emsProp) (
 			instanceKey string
 		)
 
-		var instanceLabelCount uint64
 		if !instanceData.IsObject() {
 			e.Logger.Warn().Str("type", instanceData.Type.String()).Msg("Instance data is not object, skipping")
 			continue
@@ -537,10 +536,20 @@ func (e *Ems) HandleResults(result []gjson.Result, prop map[string][]*emsProp) (
 				mx = m[msgName]
 			}
 
-			//parse ems properties for the instance
+			// Check matches at all same name ems
 			isMatch := false
+			// Check matches at each ems
+			isMatchPs := false
+			// Check instance count at all same name ems
+			instanceLabelCount := uint64(0)
+			// Check instance count at each ems
+			instanceLabelCountPs := uint64(0)
+
+			// parse ems properties for the instance
 			if ps, ok := prop[msgName]; ok {
 				for _, p := range ps {
+					isMatchPs = false
+					instanceLabelCountPs = 0
 					instanceKey = e.getInstanceKeys(p, instanceData)
 					instance := mx.GetInstance(instanceKey)
 
@@ -571,7 +580,7 @@ func (e *Ems) HandleResults(result []gjson.Result, prop map[string][]*emsProp) (
 							} else {
 								instance.SetLabel(display, value.String())
 							}
-							instanceLabelCount++
+							instanceLabelCountPs++
 						} else {
 							e.Logger.Warn().Str("Instance key", instanceKey).Str("label", label).Msg("Missing label value")
 						}
@@ -584,12 +593,12 @@ func (e *Ems) HandleResults(result []gjson.Result, prop map[string][]*emsProp) (
 
 					//matches filtering
 					if len(p.Matches) == 0 {
-						isMatch = true
+						isMatchPs = true
 					} else {
 						for _, match := range p.Matches {
 							if value := instance.GetLabel(match.Name); value != "" {
 								if value == match.value {
-									isMatch = true
+									isMatchPs = true
 									break
 								}
 							} else {
@@ -602,8 +611,8 @@ func (e *Ems) HandleResults(result []gjson.Result, prop map[string][]*emsProp) (
 							}
 						}
 					}
-					if !isMatch {
-						instanceLabelCount = 0
+					if !isMatchPs {
+						instanceLabelCountPs = 0
 						continue
 					}
 
@@ -634,6 +643,8 @@ func (e *Ems) HandleResults(result []gjson.Result, prop map[string][]*emsProp) (
 								Msg("Unable to find metric")
 						}
 					}
+					instanceLabelCount += instanceLabelCountPs
+					isMatch = isMatch || isMatchPs
 				}
 			}
 			if !isMatch {
@@ -670,7 +681,7 @@ func (e *Ems) getInstanceKeys(p *emsProp, instanceData gjson.Result) string {
 	return instanceKey
 }
 
-func (e *Ems) updateMatrix() {
+func (e *Ems) updateMatrix(begin time.Time) {
 	tempMap := make(map[string]*matrix.Matrix)
 	// store the bookend ems metric in tempMap
 	for _, issuingEmsList := range e.bookendEmsMap {
@@ -715,7 +726,7 @@ func (e *Ems) updateMatrix() {
 
 			// check instance timestamp and remove it after given resolve_after duration
 			if metricTimestamp, ok := timestampMetric.GetValueFloat64(instance); ok {
-				if collectors.IsTimestampOlderThanDuration(metricTimestamp, e.resolveAfter[issuingEms]) {
+				if collectors.IsTimestampOlderThanDuration(begin, metricTimestamp, e.resolveAfter[issuingEms]) {
 					// Set events metric value as 0 and export instance to true with label autoresolved as true.
 					if err := eventMetric.SetValueFloat64(instance, 0); err != nil {
 						e.Logger.Error().Err(err).Str("key", "events").

cmd/collectors/ems/ems_test.go

@@ -17,6 +17,9 @@ import (
 // Bookend EMS testing: Simulated bookend issuing ems "wafl.vvol.offline" and ems "hm.alert.raised" with alert_id value as "RaidLeftBehindAggrAlert"
 var issuingEmsNames = []string{"wafl.vvol.offline", "hm.alert.raised"}
 
+// Default labels per ems is 5, "hm.alert.raised" ems has 10 labels and "wafl.vvol.offline" has 4 labels, total instance labels would be 24
+const expectedInstanceLabelCount = 24
+
 // Auto resolve EMS testing: Simulated bookend issuing ems "LUN.offline" and ems "monitor.fan.critical"
 var autoresolveEmsNames = []string{"LUN.offline", "monitor.fan.critical"}
 
@@ -76,12 +79,12 @@ func emsParams(emsConfigPath string) *node.Node {
 }
 
 func (e *Ems) testBookendIssuingEms(t *testing.T, path string) {
-	e.updateMatrix()
+	e.updateMatrix(time.Now())
 
 	results := collectors.JSONToGson(path, true)
 	// Polling ems collector to handle results
-	if _, emsCount, _ := e.HandleResults(results, e.emsProp); emsCount == 0 {
-		t.Fatal("Failed to fetch data")
+	if _, emsCount, _ := e.HandleResults(results, e.emsProp); emsCount != expectedInstanceLabelCount {
+		t.Fatalf("Instance labels count mismatch detected. Expected labels: %d actual labels: %d", expectedInstanceLabelCount, emsCount)
 	}
 
 	// Check and evaluate ems events
@@ -114,7 +117,7 @@ func (e *Ems) testBookendIssuingEms(t *testing.T, path string) {
 }
 
 func (e *Ems) testBookendResolvingEms(t *testing.T, path string) {
-	e.updateMatrix()
+	e.updateMatrix(time.Now())
 
 	// Simulated bookend resolving ems "wafl.vvol.online" and ems "hm.alert.cleared" with alert_id value as "RaidLeftBehindAggrAlert"
 	results := collectors.JSONToGson(path, true)
@@ -156,7 +159,7 @@ func (e *Ems) testBookendResolvingEms(t *testing.T, path string) {
 
 func (e *Ems) testAutoResolvingEms(t *testing.T, path string) {
 	var notGeneratedEmsNames, notAutoResolvedEmsNames []string
-	e.updateMatrix()
+	e.updateMatrix(time.Now())
 
 	results := collectors.JSONToGson(path, true)
 	// Polling ems collector to handle results
@@ -184,9 +187,8 @@ func (e *Ems) testAutoResolvingEms(t *testing.T, path string) {
 	}
 
 	// Evaluate the cache for existence of these auto resolve ems.
-	// Sleep for 1 second and check LUN.offline ems got auto resolved
-	time.Sleep(1 * time.Second)
-	e.updateMatrix()
+	// Simulate one second in the future and check that the LUN.offline ems event is auto resolved
+	e.updateMatrix(time.Now().Add(1 * time.Second))
 	// Check and evaluate bookend ems events got auto resolved successfully.
 	for generatedEmsName, mx := range e.Matrix {
 		if util.Contains(autoresolveEmsNames, generatedEmsName) {
@@ -206,9 +208,9 @@ func (e *Ems) testAutoResolvingEms(t *testing.T, path string) {
 		t.Fatalf("These Bookend Ems haven't been auto resolved: %s", notAutoResolvedEmsNames)
 	}
 
-	// Sleep for another 1 second and check "LUN.offline" ems got removed from cache and "monitor.fan.critical" got auto resolved
-	time.Sleep(1 * time.Second)
-	e.updateMatrix()
+	// Simulate two seconds in future and check that the "LUN.offline" ems event was removed from the cache and
+	// "monitor.fan.critical" is auto resolved
+	e.updateMatrix(time.Now().Add(2 * time.Second))
 	notAutoResolvedEmsNames = make([]string, 0)
 	// Check bookend ems event got removed from cache successfully.
 	if e.Matrix["LUN.offline"] != nil {

cmd/collectors/rest/plugins/certificate/certificate.go

@@ -139,6 +139,8 @@ func (my *Certificate) setCertificateIssuerType(instance *matrix.Instance) {
 				// Any verification exception means it is not signed with the give key. i.e. not self-signed
 				instance.SetLabel("certificateIssuerType", "ca_signed")
 			}
+		} else {
+			instance.SetLabel("certificateIssuerType", "ca_signed")
 		}
 	}
 }

cmd/collectors/rest/plugins/qtree/qtree.go

@@ -187,7 +187,7 @@ func (my *Qtree) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, error
 		Msg("Collected")
 
 	// metrics with qtree prefix and quota prefix are available to support backward compatibility
-	qtreePluginData := my.data.Clone(true, true, true)
+	qtreePluginData := my.data.Clone(matrix.With{Data: true, Metrics: true, Instances: true, ExportInstances: true})
 	qtreePluginData.UUID = my.Parent + ".Qtree"
 	qtreePluginData.Object = "qtree"
 	qtreePluginData.Identifier = "qtree"

cmd/collectors/rest/plugins/securityaccount/securityaccount.go

@@ -61,7 +61,6 @@ func (s *SecurityAccount) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matr
 	)
 
 	data := dataMap[s.Object]
-
 	href := rest.BuildHref("", "applications", nil, "", "", "", "", s.query)
 
 	if result, err = collectors.InvokeRestCall(s.client, href, s.Logger); err != nil {
@@ -91,7 +90,7 @@ func (s *SecurityAccount) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matr
 			}
 		}
 
-		securityAccountKey := username + svm
+		securityAccountKey := svm + username
 		if securityAccountInstance := data.GetInstance(securityAccountKey); securityAccountInstance != nil {
 			securityAccountInstance.SetExportable(false)
 

cmd/collectors/rest/plugins/svm/svm.go

@@ -12,11 +12,15 @@ import (
 	"github.com/netapp/harvest/v2/pkg/errs"
 	"github.com/netapp/harvest/v2/pkg/matrix"
 	"github.com/tidwall/gjson"
+	"regexp"
 	"sort"
+	"strconv"
 	"strings"
 	"time"
 )
 
+var weakCiphers = regexp.MustCompile("(.*)_cbc.*")
+
 type SVM struct {
 	*plugin.AbstractPlugin
 	nsswitchInfo        map[string]nsswitch
@@ -154,6 +158,10 @@ func (my *SVM) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, error)
 		if iscsiAuthenticationType, ok := my.iscsiCredentialInfo[svmName]; ok {
 			svmInstance.SetLabel("iscsi_authentication_type", iscsiAuthenticationType)
 		}
+
+		ciphersVal := svmInstance.GetLabel("ciphers")
+		insecured := weakCiphers.MatchString(ciphersVal)
+		svmInstance.SetLabel("insecured", strconv.FormatBool(insecured))
 	}
 	return nil, nil
 }

cmd/collectors/restperf/plugins/volume/volume.go

@@ -57,7 +57,7 @@ func (v *Volume) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, error
 	}
 	v.Logger.Trace().Msgf("added metric: (%s) %v", metricName, metric)
 
-	cache := data.Clone(false, true, false)
+	cache := data.Clone(matrix.With{Data: false, Metrics: true, Instances: false, ExportInstances: true})
 	cache.UUID += ".Volume"
 
 	// create flexgroup instance cache

cmd/collectors/restperf/plugins/vscan/vscan.go

@@ -65,7 +65,7 @@ func (v *Vscan) aggregatePerScanner(data *matrix.Matrix) ([]*matrix.Matrix, erro
 	// 		scan_request_dispatched_rate
 
 	// create per scanner instance cache
-	cache := data.Clone(false, true, false)
+	cache := data.Clone(matrix.With{Data: false, Metrics: true, Instances: false, ExportInstances: true})
 	cache.UUID += ".Vscan"
 
 	for _, i := range data.GetInstances() {

cmd/collectors/restperf/restperf.go

@@ -649,7 +649,7 @@ func (r *RestPerf) pollData(startTime time.Time, perfRecords []rest.PerfRecord)
 
 	prevMat = r.Matrix[r.Object]
 	// clone matrix without numeric data
-	curMat = prevMat.Clone(false, true, true)
+	curMat = prevMat.Clone(matrix.With{Data: false, Metrics: true, Instances: true, ExportInstances: true})
 	curMat.Reset()
 	instanceKeys = r.Prop.InstanceKeys
 
@@ -973,7 +973,7 @@ func (r *RestPerf) pollData(startTime time.Time, perfRecords []rest.PerfRecord)
 	r.Logger.Debug().Msg("starting delta calculations from previous cache")
 
 	// cache raw data for next poll
-	cachedData := curMat.Clone(true, true, true)
+	cachedData := curMat.Clone(matrix.With{Data: true, Metrics: true, Instances: true, ExportInstances: true})
 
 	orderedNonDenominatorMetrics := make([]*matrix.Metric, 0, len(curMat.GetMetrics()))
 	orderedNonDenominatorKeys := make([]string, 0, len(orderedNonDenominatorMetrics))