Merge branch 'main' into rg2-workload-dash

.clabot

@@ -18,6 +18,6 @@
         "$$$ Bots $$$",
         "dependabot",
         "dependabot[bot]",
-        "renovate"
+        "renovate[bot]"
     ]
 }

.github/workflows/go.yml

@@ -1,7 +1,7 @@
 name: Build, Test, Lint License
 
 env:
-  GO_VERSION: "1.20.3"
+  GO_VERSION: "1.20.5"
 
 on:
   push:

cmd/collectors/ems/ems.go

@@ -25,6 +25,7 @@ const defaultSeverityFilter = "alert|emergency|error|informational|notice"
 const MaxBookendInstances = 1000
 const DefaultBookendResolutionDuration = 28 * 24 * time.Hour // 28 days == 672 hours
 const Hyphen = "-"
+const AutoResolved = "autoresolved"
 
 type Ems struct {
 	*rest2.Rest    // provides: AbstractCollector, Client, Object, Query, TemplateFn, TemplateType
@@ -340,11 +341,11 @@ func (e *Ems) PollInstance() (map[string]*matrix.Matrix, error) {
 func (e *Ems) PollData() (map[string]*matrix.Matrix, error) {
 
 	var (
-		count        uint64
-		apiD, parseD time.Duration
-		startTime    time.Time
-		err          error
-		records      []gjson.Result
+		count, instanceCount uint64
+		apiD, parseD         time.Duration
+		startTime            time.Time
+		err                  error
+		records              []gjson.Result
 	)
 
 	e.Logger.Debug().Msg("starting data poll")
@@ -395,29 +396,15 @@ func (e *Ems) PollData() (map[string]*matrix.Matrix, error) {
 
 	apiD = time.Since(startTime)
 
-	if len(records) == 0 {
-		e.lastFilterTime = toTime
-		_ = e.Metadata.LazySetValueInt64("api_time", "data", apiD.Microseconds())
-		_ = e.Metadata.LazySetValueInt64("parse_time", "data", parseD.Microseconds())
-		_ = e.Metadata.LazySetValueUint64("metrics", "data", 0)
-		_ = e.Metadata.LazySetValueUint64("instances", "data", 0)
-		return nil, nil
-	}
-
 	startTime = time.Now()
-	_, count = e.HandleResults(records, e.emsProp)
+	_, count, instanceCount = e.HandleResults(records, e.emsProp)
 
 	parseD = time.Since(startTime)
 
-	var instanceCount int
-	for _, v := range e.Matrix {
-		instanceCount += len(v.GetInstances())
-	}
-
 	_ = e.Metadata.LazySetValueInt64("api_time", "data", apiD.Microseconds())
 	_ = e.Metadata.LazySetValueInt64("parse_time", "data", parseD.Microseconds())
 	_ = e.Metadata.LazySetValueUint64("metrics", "data", count)
-	_ = e.Metadata.LazySetValueUint64("instances", "data", uint64(instanceCount))
+	_ = e.Metadata.LazySetValueUint64("instances", "data", instanceCount)
 
 	e.AddCollectCount(count)
 
@@ -464,11 +451,11 @@ func parseProperties(instanceData gjson.Result, property string) gjson.Result {
 }
 
 // HandleResults function is used for handling the rest response for parent as well as endpoints calls,
-func (e *Ems) HandleResults(result []gjson.Result, prop map[string][]*emsProp) (map[string]*matrix.Matrix, uint64) {
+func (e *Ems) HandleResults(result []gjson.Result, prop map[string][]*emsProp) (map[string]*matrix.Matrix, uint64, uint64) {
 	var (
-		err   error
-		count uint64
-		mx    *matrix.Matrix
+		err                  error
+		count, instanceCount uint64
+		mx                   *matrix.Matrix
 	)
 
 	var m = e.Matrix
@@ -568,6 +555,10 @@ func (e *Ems) HandleResults(result []gjson.Result, prop map[string][]*emsProp) (
 					instance.SetExportable(true)
 
 					for label, display := range p.InstanceLabels {
+						if label == AutoResolved {
+							instance.SetLabel(display, "")
+							continue
+						}
 						value := parseProperties(instanceData, label)
 						if value.Exists() {
 							if value.IsArray() {
@@ -652,7 +643,16 @@ func (e *Ems) HandleResults(result []gjson.Result, prop map[string][]*emsProp) (
 			count += instanceLabelCount
 		}
 	}
-	return m, count
+
+	for _, v := range e.Matrix {
+		for _, i := range v.GetInstances() {
+			if i.IsExportable() {
+				instanceCount++
+			}
+		}
+	}
+
+	return m, count, instanceCount
 }
 
 func (e *Ems) getInstanceKeys(p *emsProp, instanceData gjson.Result) string {
@@ -671,12 +671,6 @@ func (e *Ems) getInstanceKeys(p *emsProp, instanceData gjson.Result) string {
 }
 
 func (e *Ems) updateMatrix() {
-	var (
-		ok                           bool
-		val                          float64
-		eventMetric, timestampMetric *matrix.Metric
-	)
-
 	tempMap := make(map[string]*matrix.Matrix)
 	// store the bookend ems metric in tempMap
 	for _, issuingEmsList := range e.bookendEmsMap {
@@ -693,15 +687,17 @@ func (e *Ems) updateMatrix() {
 	e.Matrix[e.Object] = mat
 
 	for issuingEms, mx := range tempMap {
-		if eventMetric, ok = mx.GetMetrics()["events"]; !ok {
+		eventMetric, ok := mx.GetMetrics()["events"]
+		if !ok {
 			e.Logger.Error().
 				Str("issuingEms", issuingEms).
 				Str("name", "events").
 				Msg("failed to get metric")
 			continue
 		}
 
-		if timestampMetric, ok = mx.GetMetrics()["timestamp"]; !ok {
+		timestampMetric, ok := mx.GetMetrics()["timestamp"]
+		if !ok {
 			e.Logger.Error().
 				Str("issuingEms", issuingEms).
 				Str("name", "timestamp").
@@ -712,14 +708,22 @@ func (e *Ems) updateMatrix() {
 			// set export to false
 			instance.SetExportable(false)
 
-			if val, ok = eventMetric.GetValueFloat64(instance); ok && val == 0 {
+			if val, exist := eventMetric.GetValueFloat64(instance); exist && val == 0 {
 				mx.RemoveInstance(instanceKey)
 				continue
 			}
+
 			// check instance timestamp and remove it after given resolve_after duration
 			if metricTimestamp, ok := timestampMetric.GetValueFloat64(instance); ok {
 				if collectors.IsTimestampOlderThanDuration(metricTimestamp, e.resolveAfter[issuingEms]) {
-					mx.RemoveInstance(instanceKey)
+					// Set events metric value as 0 and export instance to true with label autoresolved as true.
+					if err := eventMetric.SetValueFloat64(instance, 0); err != nil {
+						e.Logger.Error().Err(err).Str("key", "events").
+							Msg("Unable to set float key on metric")
+						continue
+					}
+					instance.SetExportable(true)
+					instance.SetLabel(AutoResolved, "true")
 				}
 			}
 		}

cmd/collectors/ems/ems_test.go

@@ -11,11 +11,15 @@ import (
 	"github.com/rs/zerolog/log"
 	"os"
 	"testing"
+	"time"
 )
 
 // Bookend EMS testing: Simulated bookend issuing ems "wafl.vvol.offline" and ems "hm.alert.raised" with alert_id value as "RaidLeftBehindAggrAlert"
 var issuingEmsNames = []string{"wafl.vvol.offline", "hm.alert.raised"}
 
+// Auto resolve EMS testing: Simulated bookend issuing ems "LUN.offline" and ems "monitor.fan.critical"
+var autoresolveEmsNames = []string{"LUN.offline", "monitor.fan.critical"}
+
 func Test_Ems(t *testing.T) {
 	// Initialize the Ems collector
 	e := NewEms()
@@ -30,6 +34,9 @@ func BookendEmsTest(t *testing.T, e *Ems) {
 
 	// Step 2: Generate Bookend resolving ems
 	e.testBookendResolvingEms(t, "testdata/resolvingEms.json")
+
+	// Step 3: Generate Bookend issuing ems and validate auto resolution functionality
+	e.testAutoResolvingEms(t, "testdata/autoresolveEms.json")
 }
 
 func NewEms() *Ems {
@@ -49,6 +56,9 @@ func NewEms() *Ems {
 	if err := e.Init(ac); err != nil {
 		log.Fatal().Err(err)
 	}
+	// Changed the resolve_after for 2 issuing ems for auto resolve testing
+	e.resolveAfter["LUN.offline"] = 1 * time.Second
+	e.resolveAfter["monitor.fan.critical"] = 2 * time.Second
 	return e
 }
 
@@ -70,7 +80,7 @@ func (e *Ems) testBookendIssuingEms(t *testing.T, path string) {
 
 	results := collectors.JSONToGson(path, true)
 	// Polling ems collector to handle results
-	if _, emsCount := e.HandleResults(results, e.emsProp); emsCount == 0 {
+	if _, emsCount, _ := e.HandleResults(results, e.emsProp); emsCount == 0 {
 		t.Fatal("Failed to fetch data")
 	}
 
@@ -79,8 +89,7 @@ func (e *Ems) testBookendIssuingEms(t *testing.T, path string) {
 	for generatedEmsName, mx := range e.Matrix {
 		if util.Contains(issuingEmsNames, generatedEmsName) {
 			metr, ok := mx.GetMetrics()["events"]
-			// e.Matrix map would have one entry of Ems(parent) in map, skipping that as it's not required for testing.
-			if !ok && generatedEmsName != "Ems" {
+			if !ok {
 				t.Fatalf("Failed to get netric for Ems %s", generatedEmsName)
 			}
 			for _, instance := range mx.GetInstances() {
@@ -110,7 +119,7 @@ func (e *Ems) testBookendResolvingEms(t *testing.T, path string) {
 	// Simulated bookend resolving ems "wafl.vvol.online" and ems "hm.alert.cleared" with alert_id value as "RaidLeftBehindAggrAlert"
 	results := collectors.JSONToGson(path, true)
 	// Polling ems collector to handle results
-	if _, emsCount := e.HandleResults(results, e.emsProp); emsCount == 0 {
+	if _, emsCount, _ := e.HandleResults(results, e.emsProp); emsCount == 0 {
 		t.Fatal("Failed to fetch data")
 	}
 
@@ -119,8 +128,7 @@ func (e *Ems) testBookendResolvingEms(t *testing.T, path string) {
 	for generatedEmsName, mx := range e.Matrix {
 		if util.Contains(issuingEmsNames, generatedEmsName) {
 			metr, ok := mx.GetMetrics()["events"]
-			// e.Matrix map would have one entry of Ems(parent) in map, skipping that as it's not required for testing.
-			if !ok && generatedEmsName != "Ems" {
+			if !ok {
 				t.Fatalf("Failed to get netric for Ems %s", generatedEmsName)
 			}
 			for _, instance := range mx.GetInstances() {
@@ -145,3 +153,82 @@ func (e *Ems) testBookendResolvingEms(t *testing.T, path string) {
 		t.Errorf("These Bookend Ems haven't been resolved: %s", notResolvedEmsNames)
 	}
 }
+
+func (e *Ems) testAutoResolvingEms(t *testing.T, path string) {
+	var notGeneratedEmsNames, notAutoResolvedEmsNames []string
+	e.updateMatrix()
+
+	results := collectors.JSONToGson(path, true)
+	// Polling ems collector to handle results
+	if _, emsCount, _ := e.HandleResults(results, e.emsProp); emsCount == 0 {
+		t.Fatal("Failed to fetch data")
+	}
+
+	// Check and evaluate ems events
+	for generatedEmsName, mx := range e.Matrix {
+		if util.Contains(autoresolveEmsNames, generatedEmsName) {
+			if metr, ok := mx.GetMetrics()["events"]; ok {
+				for _, instance := range mx.GetInstances() {
+					// If value not exist for that instance or metric value 0 indicate ems hasn't been raised.
+					if val, ok := metr.GetValueFloat64(instance); !ok || val == 0 {
+						notGeneratedEmsNames = append(notGeneratedEmsNames, generatedEmsName)
+					}
+				}
+			} else {
+				t.Fatalf("Failed to get netric for Ems %s", generatedEmsName)
+			}
+		}
+	}
+	if len(notGeneratedEmsNames) > 0 {
+		t.Fatalf("These Bookend Ems haven't been raised: %s", notGeneratedEmsNames)
+	}
+
+	// Evaluate the cache for existence of these auto resolve ems.
+	// Sleep for 1 second and check LUN.offline ems got auto resolved
+	time.Sleep(1 * time.Second)
+	e.updateMatrix()
+	// Check and evaluate bookend ems events got auto resolved successfully.
+	for generatedEmsName, mx := range e.Matrix {
+		if util.Contains(autoresolveEmsNames, generatedEmsName) {
+			if metr, ok := mx.GetMetrics()["events"]; ok {
+				for _, instance := range mx.GetInstances() {
+					// If value not exist for that instance or metric value 0 indicate ems hasn't been raised.
+					if val, ok := metr.GetValueFloat64(instance); !ok || val == 1 {
+						notAutoResolvedEmsNames = append(notAutoResolvedEmsNames, generatedEmsName)
+					}
+				}
+			} else {
+				t.Fatalf("Failed to get netric for Ems %s", generatedEmsName)
+			}
+		}
+	}
+	if util.Contains(notAutoResolvedEmsNames, "LUN.offline") {
+		t.Fatalf("These Bookend Ems haven't been auto resolved: %s", notAutoResolvedEmsNames)
+	}
+
+	// Sleep for another 1 second and check "LUN.offline" ems got removed from cache and "monitor.fan.critical" got auto resolved
+	time.Sleep(1 * time.Second)
+	e.updateMatrix()
+	notAutoResolvedEmsNames = make([]string, 0)
+	// Check bookend ems event got removed from cache successfully.
+	if e.Matrix["LUN.offline"] != nil {
+		t.Fatalf("These Bookend Ems haven't been removed from cache: %s", "LUN.offline")
+	}
+	for generatedEmsName, mx := range e.Matrix {
+		if util.Contains(autoresolveEmsNames, generatedEmsName) {
+			if metr, ok := mx.GetMetrics()["events"]; ok {
+				for _, instance := range mx.GetInstances() {
+					// If value not exist for that instance or metric value 0 indicate ems hasn't been raised.
+					if val, ok := metr.GetValueFloat64(instance); !ok || val == 1 {
+						notAutoResolvedEmsNames = append(notAutoResolvedEmsNames, generatedEmsName)
+					}
+				}
+			} else {
+				t.Fatalf("Failed to get netric for Ems %s", generatedEmsName)
+			}
+		}
+	}
+	if util.Contains(notAutoResolvedEmsNames, "monitor.fan.critical") {
+		t.Fatalf("These Bookend Ems haven't been auto resolved: %s", notAutoResolvedEmsNames)
+	}
+}

cmd/collectors/ems/templating.go

@@ -158,4 +158,7 @@ func (e *Ems) ParseResolveEms(resolveEvent *node.Node, issueEmsProp emsProp) {
 	}
 	e.bookendEmsMap[resolveEmsName].Add(issueEmsProp.Name)
 	e.emsProp[resolveEmsName] = append(e.emsProp[resolveEmsName], &prop)
+
+	// add autoresolved label in issuingEms labels
+	issueEmsProp.InstanceLabels[AutoResolved] = AutoResolved
 }