diff --git a/cmd/collectors/zapi/plugins/aggregate/aggregate.go b/cmd/collectors/zapi/plugins/aggregate/aggregate.go index cc4105957..938745cc0 100644 --- a/cmd/collectors/zapi/plugins/aggregate/aggregate.go +++ b/cmd/collectors/zapi/plugins/aggregate/aggregate.go @@ -63,11 +63,11 @@ func (a *Aggregate) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, er } // update aggregate instance label with cloud stores info - for aggrUUID, aggr := range data.GetInstances() { + for uuid, aggr := range data.GetInstances() { if !aggr.IsExportable() { continue } - aggr.SetLabel("cloud_stores", strings.Join(a.aggrCloudStoresMap[aggrUUID], ",")) + aggr.SetLabel("cloud_stores", strings.Join(a.aggrCloudStoresMap[uuid], ",")) // Handling aggr footprint metrics aggrName := aggr.GetLabel("aggr") diff --git a/cmd/poller/collector/helpers.go b/cmd/poller/collector/helpers.go index 7e8dc6609..042113dd9 100644 --- a/cmd/poller/collector/helpers.go +++ b/cmd/poller/collector/helpers.go @@ -13,6 +13,7 @@ import ( "github.com/hashicorp/go-version" "github.com/netapp/harvest/v2/cmd/poller/plugin" "github.com/netapp/harvest/v2/cmd/poller/plugin/aggregator" + "github.com/netapp/harvest/v2/cmd/poller/plugin/changelog" "github.com/netapp/harvest/v2/cmd/poller/plugin/labelagent" "github.com/netapp/harvest/v2/cmd/poller/plugin/max" "github.com/netapp/harvest/v2/cmd/poller/plugin/metricagent" @@ -214,5 +215,9 @@ func GetBuiltinPlugin(name string, abc *plugin.AbstractPlugin) plugin.Plugin { return metricagent.New(abc) } + if name == "ChangeLog" { + return changelog.New(abc) + } + return nil } diff --git a/cmd/poller/plugin/changelog/change_log.go b/cmd/poller/plugin/changelog/change_log.go new file mode 100644 index 000000000..80e5e58e9 --- /dev/null +++ b/cmd/poller/plugin/changelog/change_log.go @@ -0,0 +1,298 @@ +package changelog + +import ( + "github.com/netapp/harvest/v2/cmd/poller/plugin" + "github.com/netapp/harvest/v2/pkg/matrix" + "github.com/netapp/harvest/v2/pkg/set" + "github.com/netapp/harvest/v2/pkg/tree/yaml" + "maps" + "strconv" + "time" +) + +/*The changelog feature is only applicable to labels and requires a UUID for the label name to exist. +A default configuration for volume, SVM, and node is available, but the DSL can be overwritten as needed. +The shape of the change_log is specific to each label change and is only applicable to matrix collected by the collector. +*/ + +// Constants for ChangeLog metrics and labels +const ( + changeLog = "change" + objectLabel = "object" + opLabel = "op" + create = "create" + update = "update" + del = "delete" + track = "track" + oldValue = "old_value" + newValue = "new_value" + indexLabel = "index" +) + +// Metrics to be used in ChangeLog +var metrics = []string{ + "log", +} + +// ChangeLog represents the main structure of the ChangeLog plugin +type ChangeLog struct { + *plugin.AbstractPlugin + matrixName string + previousData *matrix.Matrix + changeLogConfig Entry + index int + metricsCount int +} + +// Change represents a single change entry in the ChangeLog +type Change struct { + key string + object string + op string + labels map[string]string + track string + oldValue string + newValue string + time int64 +} + +// New initializes a new instance of the ChangeLog plugin +func New(p *plugin.AbstractPlugin) plugin.Plugin { + return &ChangeLog{AbstractPlugin: p} +} + +// Init initializes the ChangeLog plugin +func (c *ChangeLog) Init() error { + + // Initialize the abstract plugin + if err := c.AbstractPlugin.Init(); err != nil { + return err + } + + object := c.ParentParams.GetChildS("object") + c.matrixName = object.GetContentS() + "_" + changeLog + + return c.populateChangeLogConfig() +} + +// populateChangeLogConfig populates the ChangeLog configuration from the plugin parameters +func (c *ChangeLog) populateChangeLogConfig() error { + var err error + changeLogYaml, err := yaml.Dump(c.Params) + if err != nil { + return err + } + + c.changeLogConfig, err = getChangeLogConfig(c.ParentParams, changeLogYaml, c.Logger) + if err != nil { + return err + } + return nil +} + +// initMatrix initializes a new matrix with the given name +func (c *ChangeLog) initMatrix() (map[string]*matrix.Matrix, error) { + changeLogMap := make(map[string]*matrix.Matrix) + changeLogMap[c.matrixName] = matrix.New(c.Parent+c.matrixName, changeLog, c.matrixName) + for _, changeLogMatrix := range changeLogMap { + changeLogMatrix.SetExportOptions(matrix.DefaultExportOptions()) + } + for _, k := range metrics { + err := matrix.CreateMetric(k, changeLogMap[c.matrixName]) + if err != nil { + c.Logger.Warn().Err(err).Str("key", k).Msg("error while creating metric") + return nil, err + } + } + return changeLogMap, nil +} + +// Run processes the data and generates ChangeLog instances +func (c *ChangeLog) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, error) { + + data := dataMap[c.Object] + changeLogMap, err := c.initMatrix() + if err != nil { + c.Logger.Warn().Err(err).Msg("error while init matrix") + return nil, err + } + + // reset metric count + c.metricsCount = 0 + + // if this is the first poll + if c.previousData == nil { + c.copyPreviousData(data) + return nil, nil + } + + changeMat := changeLogMap[c.matrixName] + + changeMat.SetGlobalLabels(data.GetGlobalLabels()) + object := data.Object + if c.changeLogConfig.Object == "" { + c.Logger.Warn().Str("object", object).Msg("ChangeLog is not supported. Missing correct configuration") + return nil, nil + } + + prevMat := c.previousData + oldInstances := set.New() + prevInstancesUUIDKey := make(map[string]string) + for key, prevInstance := range prevMat.GetInstances() { + uuid := prevInstance.GetLabel("uuid") + if uuid == "" { + c.Logger.Warn().Str("object", object).Str("key", key).Msg("missing uuid") + continue + } + prevInstancesUUIDKey[uuid] = key + oldInstances.Add(key) + } + + currentTime := time.Now().Unix() + + // loop over current instances + for key, instance := range data.GetInstances() { + uuid := instance.GetLabel("uuid") + if uuid == "" { + c.Logger.Warn().Str("object", object).Str("key", key).Msg("missing uuid. ChangeLog is not supported") + continue + } + + prevKey := prevInstancesUUIDKey[uuid] + if prevKey != "" { + // instance already in cache + oldInstances.Remove(prevKey) + } + + prevInstance := c.previousData.GetInstance(prevKey) + + if prevInstance == nil { + //instance created + change := &Change{ + key: uuid + "_" + object, + object: object, + op: create, + labels: make(map[string]string), + time: currentTime, + } + c.updateChangeLogLabels(object, instance, change) + c.createChangeLogInstance(changeMat, change) + } else { + // check for any modification + cur, old := instance.CompareDiffs(prevInstance, c.changeLogConfig.Track) + if len(cur) > 0 { + for currentLabel, nVal := range cur { + change := &Change{ + key: uuid + "_" + object + "_" + currentLabel, + object: object, + op: update, + labels: make(map[string]string), + track: currentLabel, + oldValue: old[currentLabel], + newValue: nVal, + time: currentTime, + } + c.updateChangeLogLabels(object, instance, change) + // add changed track and its old, new value + change.labels[track] = currentLabel + change.labels[oldValue] = change.oldValue + change.labels[newValue] = nVal + c.createChangeLogInstance(changeMat, change) + } + } + } + } + // create deleted instances change_log + for key := range oldInstances.Iter() { + prevInstance := prevMat.GetInstance(key) + uuid := prevInstance.GetLabel("uuid") + if uuid == "" { + c.Logger.Warn().Str("object", object).Str("key", key).Msg("missing uuid. ChangeLog is not supported") + continue + } + if prevInstance != nil { + change := &Change{ + key: uuid + "_" + object, + object: object, + op: del, + labels: make(map[string]string), + time: currentTime, + } + c.updateChangeLogLabels(object, prevInstance, change) + c.createChangeLogInstance(changeMat, change) + } else { + c.Logger.Warn().Str("object", object).Str("key", key).Msg("missing instance") + } + } + + var matricesArray []*matrix.Matrix + matricesArray = append(matricesArray, changeMat) + + c.copyPreviousData(data) + if len(changeMat.GetInstances()) > 0 { + // The `index` variable is used to differentiate between changes to the same label in a Grafana dashboard. + // It has a value between 0 and 100 and is used in the `change_log` query as `last_over_time`. + c.index = (c.index + 1) % 100 + c.Logger.Info().Int("instances", len(changeMat.GetInstances())). + Int("metrics", c.metricsCount). + Int("index", c.index). + Msg("Collected") + } + + return matricesArray, nil +} + +// copyPreviousData creates a copy of the previous data for comparison +func (c *ChangeLog) copyPreviousData(cur *matrix.Matrix) { + labels := c.changeLogConfig.PublishLabels + labels = append(labels, c.changeLogConfig.Track...) + labels = append(labels, "uuid") + c.previousData = cur.Clone(matrix.With{Data: true, Metrics: false, Instances: true, ExportInstances: false, Labels: labels}) +} + +// createChangeLogInstance creates a new ChangeLog instance with the given change data +func (c *ChangeLog) createChangeLogInstance(mat *matrix.Matrix, change *Change) { + cInstance, err := mat.NewInstance(change.key) + if err != nil { + c.Logger.Warn().Str("object", change.object).Str("key", change.key).Msg("error while creating instance") + return + } + // copy keys + cInstance.SetLabel(objectLabel, change.object) + cInstance.SetLabel(opLabel, change.op) + cInstance.SetLabel(indexLabel, strconv.Itoa(c.index)) + for k, v := range change.labels { + cInstance.SetLabel(k, v) + } + c.metricsCount += len(cInstance.GetLabels()) + m := mat.GetMetric("log") + if m == nil { + if m, err = mat.NewMetricFloat64("log"); err != nil { + c.Logger.Warn().Err(err).Str("key", "log").Msg("error while creating metric") + return + } + } + if err = m.SetValueInt64(cInstance, change.time); err != nil { + c.Logger.Error().Err(err).Int64("val", change.time).Msg("Unable to set value on metric") + return + } +} + +// updateChangeLogLabels populates change log labels +func (c *ChangeLog) updateChangeLogLabels(object string, instance *matrix.Instance, change *Change) { + cl := c.changeLogConfig + if len(cl.PublishLabels) > 0 { + for _, l := range cl.PublishLabels { + labelValue := instance.GetLabel(l) + if labelValue == "" { + c.Logger.Warn().Str("object", object).Str("label", l).Msg("Missing label") + } else { + change.labels[l] = labelValue + } + } + } else if cl.includeAll { + maps.Copy(change.labels, instance.GetLabels()) + } else { + c.Logger.Warn().Str("object", object).Msg("missing publish labels") + } +} diff --git a/cmd/poller/plugin/changelog/change_log_helper.go b/cmd/poller/plugin/changelog/change_log_helper.go new file mode 100644 index 000000000..448310a99 --- /dev/null +++ b/cmd/poller/plugin/changelog/change_log_helper.go @@ -0,0 +1,112 @@ +package changelog + +import ( + "github.com/netapp/harvest/v2/pkg/logging" + "github.com/netapp/harvest/v2/pkg/tree/node" + "gopkg.in/yaml.v3" + "slices" + "strconv" +) + +// Entry represents a single ChangeLog entry +type Entry struct { + Object string `yaml:"object"` + Track []string `yaml:"track"` + PublishLabels []string `yaml:"publish_labels"` + includeAll bool +} + +// Config represents the structure of the ChangeLog configuration +type Config struct { + ChangeLogs []Entry `yaml:"ChangeLog"` +} + +// defaultChangeLogTemplate is the default ChangeLog configuration +const defaultChangeLogTemplate = ` +ChangeLog: + - object: svm + track: + - svm + - state + - type + - anti_ransomware_state + - object: node + track: + - node + - location + - healthy + - object: volume + track: + - node + - volume + - svm + - style + - type + - aggr + - state + - status +` + +// getChangeLogConfig returns a map of ChangeLog entries for the given object +func getChangeLogConfig(parentParams *node.Node, overwriteConfig []byte, logger *logging.Logger) (Entry, error) { + var ( + config Config + entry Entry + err error + ) + object := parentParams.GetChildS("object").GetContentS() + + if len(overwriteConfig) > 0 { + entry, err = preprocessOverwrite(object, overwriteConfig) + if err != nil { + logger.Warn().Err(err).Str("template", string(overwriteConfig)).Msg("failed to parse changelog dsl. Trying default") + } else { + return entry, nil + } + } + + err = yaml.Unmarshal([]byte(defaultChangeLogTemplate), &config) + if err != nil { + return Entry{}, err + } + + i := slices.IndexFunc(config.ChangeLogs, func(entry Entry) bool { + return entry.Object == object + }) + if i == -1 { + return Entry{}, nil + } + entry = config.ChangeLogs[i] + // populate publish_labels if they are empty + if entry.PublishLabels == nil { + if exportOption := parentParams.GetChildS("export_options"); exportOption != nil { + if exportedKeys := exportOption.GetChildS("instance_keys"); exportedKeys != nil { + entry.PublishLabels = append(entry.PublishLabels, exportedKeys.GetAllChildContentS()...) + } else if x := exportOption.GetChildContentS("include_all_labels"); x != "" { + if includeAllLabels, err := strconv.ParseBool(x); err != nil { + logger.Logger.Error().Err(err).Msg("parameter: include_all_labels") + } else { + if includeAllLabels { + entry.includeAll = true + } + } + } + } + } + + return entry, nil +} + +// preprocessOverwrite updates the ChangeLog configuration by adding the given object and its properties +func preprocessOverwrite(object string, configStr []byte) (Entry, error) { + var entry Entry + + err := yaml.Unmarshal(configStr, &entry) + if err != nil { + return entry, err + } + + entry.Object = object + return entry, nil + +} diff --git a/cmd/poller/plugin/changelog/change_log_test.go b/cmd/poller/plugin/changelog/change_log_test.go new file mode 100644 index 000000000..5902d61ba --- /dev/null +++ b/cmd/poller/plugin/changelog/change_log_test.go @@ -0,0 +1,208 @@ +package changelog + +import ( + "github.com/netapp/harvest/v2/cmd/poller/options" + "github.com/netapp/harvest/v2/cmd/poller/plugin" + "github.com/netapp/harvest/v2/pkg/matrix" + "github.com/netapp/harvest/v2/pkg/tree/node" + "testing" +) + +func newChangeLog(object string, includeAll bool) *ChangeLog { + params := node.NewS("ChangeLog") + parentParams := node.NewS("parent") + parentParams.NewChildS("object", object) + exportOptions := parentParams.NewChildS("export_options", "") + if includeAll { + exportOptions.NewChildS("include_all_labels", "true") + } else { + instanceKeys := exportOptions.NewChildS("instance_keys", "") + instanceKeys.NewChildS("", "svm") + } + + return createChangeLog(params, parentParams) +} + +func createChangeLog(params, parentParams *node.Node) *ChangeLog { + abc := plugin.New("Test", nil, params, parentParams, "", nil) + p := &ChangeLog{AbstractPlugin: abc} + p.Options = &options.Options{ + Poller: "Test", + } + p.Object = "svm" + + if err := p.Init(); err != nil { + panic(err) + } + return p +} + +func newChangeLogUnsupportedTrack(object string) *ChangeLog { + params := node.NewS("ChangeLog") + t := params.NewChildS("Track", "") + t.NewChildS("", "abcd") + parentParams := node.NewS("parent") + parentParams.NewChildS("object", object) + + return createChangeLog(params, parentParams) +} + +func checkChangeLogInstances(t *testing.T, o []*matrix.Matrix, expectedInstances, expectedLabels int, expectedOpLabel, opLabel string) { + if len(o) == 1 { + cl := o[0] + if len(cl.GetInstances()) != expectedInstances { + t.Errorf("ChangeLog instances size expected %d, actual %d", expectedInstances, len(cl.GetInstances())) + } else { + for _, i := range cl.GetInstances() { + if i.GetLabel(opLabel) != expectedOpLabel { + t.Errorf("ChangeLog %s label expected %s, actual %s", opLabel, expectedOpLabel, i.GetLabel(opLabel)) + } + if len(i.GetLabels()) != expectedLabels { + t.Errorf("ChangeLog number of labels expected %d, actual %d", expectedLabels, len(i.GetLabels())) + } + } + } + } else { + t.Error("ChangeLog slice size is wrong") + } +} + +func TestChangeLogModified(t *testing.T) { + p := newChangeLog("svm", true) + m := matrix.New("TestChangeLog", "svm", "svm") + data := map[string]*matrix.Matrix{ + "svm": m, + } + instance, _ := m.NewInstance("0") + instance.SetLabel("uuid", "u1") + instance.SetLabel("svm", "s1") + instance.SetLabel("type", "t1") + + _, _ = p.Run(data) + + m1 := matrix.New("TestChangeLog", "svm", "svm") + data1 := map[string]*matrix.Matrix{ + "svm": m1, + } + instance1, _ := m1.NewInstance("0") + instance1.SetLabel("uuid", "u1") + instance1.SetLabel("svm", "s2") + instance1.SetLabel("type", "t2") + + o, _ := p.Run(data1) + + checkChangeLogInstances(t, o, 2, 9, update, opLabel) +} + +func TestChangeLogCreated(t *testing.T) { + p := newChangeLog("svm", false) + m := matrix.New("TestChangeLog", "svm", "svm") + data := map[string]*matrix.Matrix{ + "svm": m, + } + instance, _ := m.NewInstance("0") + instance.SetLabel("uuid", "u1") + instance.SetLabel("svm", "s1") + instance.SetLabel("type", "t1") + + _, _ = p.Run(data) + + m1 := matrix.New("TestChangeLog", "svm", "svm") + data1 := map[string]*matrix.Matrix{ + "svm": m1, + } + instance1, _ := m1.NewInstance("1") + instance1.SetLabel("uuid", "u2") + instance1.SetLabel("svm", "s2") + instance1.SetLabel("type", "t2") + + instance2, _ := m1.NewInstance("0") + instance2.SetLabel("uuid", "u1") + instance2.SetLabel("svm", "s1") + instance2.SetLabel("type", "t1") + + o, _ := p.Run(data1) + + checkChangeLogInstances(t, o, 1, 4, create, opLabel) +} + +func TestChangeLogDeleted(t *testing.T) { + p := newChangeLog("svm", false) + m := matrix.New("TestChangeLog", "svm", "svm") + data := map[string]*matrix.Matrix{ + "svm": m, + } + instance, _ := m.NewInstance("0") + instance.SetLabel("uuid", "u1") + instance.SetLabel("svm", "s1") + instance.SetLabel("type", "t1") + + _, _ = p.Run(data) + + m1 := matrix.New("TestChangeLog", "svm", "svm") + data1 := map[string]*matrix.Matrix{ + "svm": m1, + } + + o, _ := p.Run(data1) + + checkChangeLogInstances(t, o, 1, 4, del, opLabel) +} + +func TestChangeLogUnsupported(t *testing.T) { + p := newChangeLog("lun", false) + m := matrix.New("TestChangeLog", "lun", "lun") + data := map[string]*matrix.Matrix{ + "svm": m, + } + instance, _ := m.NewInstance("0") + instance.SetLabel("uuid", "u1") + instance.SetLabel("lun", "l1") + + _, _ = p.Run(data) + + m1 := matrix.New("TestChangeLog", "lun", "lun") + data1 := map[string]*matrix.Matrix{ + "svm": m1, + } + instance1, _ := m1.NewInstance("1") + instance1.SetLabel("uuid", "u2") + instance1.SetLabel("lun", "l2") + + instance2, _ := m1.NewInstance("0") + instance2.SetLabel("uuid", "u1") + instance2.SetLabel("lun", "l3") + + o, _ := p.Run(data1) + + if len(o) != 0 { + t.Errorf("ChangeLog mEtric size expected %d, actual %d", 0, len(o)) + } +} + +func TestChangeLogModifiedUnsupportedTrack(t *testing.T) { + p := newChangeLogUnsupportedTrack("svm") + + m := matrix.New("TestChangeLog", "svm", "svm") + data := map[string]*matrix.Matrix{ + "svm": m, + } + instance, _ := m.NewInstance("0") + instance.SetLabel("uuid", "u1") + instance.SetLabel("svm", "s1") + + _, _ = p.Run(data) + + m1 := matrix.New("TestChangeLog", "svm", "svm") + data1 := map[string]*matrix.Matrix{ + "svm": m1, + } + + instance1, _ := m1.NewInstance("0") + instance1.SetLabel("uuid", "u1") + instance1.SetLabel("svm", "s2") + + o, _ := p.Run(data1) + + checkChangeLogInstances(t, o, 0, 0, "", "") +} diff --git a/cmd/tools/template/template_test.go b/cmd/tools/template/template_test.go index 3fae46e51..864c6301f 100644 --- a/cmd/tools/template/template_test.go +++ b/cmd/tools/template/template_test.go @@ -596,6 +596,7 @@ func findCustomPlugins(path string, template *node.Node, model *TemplateModel) e "Aggregator": true, "Max": true, "Tenant": true, + "ChangeLog": true, } for _, child := range plug[0].Children { name := child.GetNameS() diff --git a/conf/rest/9.10.0/svm.yaml b/conf/rest/9.10.0/svm.yaml index 2f2ccb1e1..5f58f2d03 100644 --- a/conf/rest/9.10.0/svm.yaml +++ b/conf/rest/9.10.0/svm.yaml @@ -9,6 +9,7 @@ counters: - ^anti_ransomware_default_volume_state => anti_ransomware_state - ^operational_state => state - ^type => type + - ^uuid => uuid endpoints: - query: api/svm/svms @@ -58,6 +59,7 @@ plugins: - new_status state online online `0` - ldap_signed ldap_session_security sign sign `0` - ldap_encrypted ldap_session_security seal seal `0` +# - ChangeLog export_options: instance_keys: diff --git a/conf/rest/9.10.0/volume.yaml b/conf/rest/9.10.0/volume.yaml index 544c286cd..e92909953 100644 --- a/conf/rest/9.10.0/volume.yaml +++ b/conf/rest/9.10.0/volume.yaml @@ -20,6 +20,7 @@ counters: - ^state => state - ^style => style - ^type => type + - ^uuid - autosize.grow_threshold => autosize_grow_threshold_percent - autosize.maximum => autosize_maximum_size - snapshot_count @@ -134,6 +135,7 @@ plugins: # - volume `MDV_CRS_.+` # # Exclude Metadata volumes, Audit volumes have a “MDV_aud_” prefix # - volume `MDV_aud_.+` +# - ChangeLog export_options: instance_keys: diff --git a/conf/rest/9.12.0/disk.yaml b/conf/rest/9.12.0/disk.yaml index 01f27b96b..792df660a 100644 --- a/conf/rest/9.12.0/disk.yaml +++ b/conf/rest/9.12.0/disk.yaml @@ -4,7 +4,7 @@ query: api/storage/disks object: disk counters: - - ^^uid + - ^^uid => uuid - ^bay => shelf_bay - ^container_type - ^home_node.name => owner_node @@ -25,7 +25,7 @@ counters: endpoints: - query: api/private/cli/disk counters: - - ^^uid => uid + - ^^uid => uuid - ^type - disk_io_kbps_total => stats_io_kbps - sectors_read => stats_sectors_read diff --git a/conf/rest/9.12.0/lif.yaml b/conf/rest/9.12.0/lif.yaml index a723aa9c0..e5aef78db 100644 --- a/conf/rest/9.12.0/lif.yaml +++ b/conf/rest/9.12.0/lif.yaml @@ -15,6 +15,7 @@ counters: - ^services => services - ^state => status - ^subnet.name => subnet + - ^uuid => uuid endpoints: - query: api/private/cli/network/interface diff --git a/conf/rest/9.12.0/node.yaml b/conf/rest/9.12.0/node.yaml index 9b11de92d..abebce45d 100644 --- a/conf/rest/9.12.0/node.yaml +++ b/conf/rest/9.12.0/node.yaml @@ -13,6 +13,7 @@ counters: - ^model - ^serial_number => serial - ^state + - ^uuid - ^version.full => version - controller.failed_fan.count => failed_fan - controller.failed_power_supply.count => failed_power @@ -35,6 +36,7 @@ plugins: - new_status healthy true up `0` join: - warnings `; ` failed_fan_message,failed_power_message,over_temperature +# - ChangeLog export_options: instance_keys: diff --git a/conf/rest/9.12.0/shelf.yaml b/conf/rest/9.12.0/shelf.yaml index c9fdeaba4..bfa1b51a5 100644 --- a/conf/rest/9.12.0/shelf.yaml +++ b/conf/rest/9.12.0/shelf.yaml @@ -10,6 +10,7 @@ counters: - ^module_type => module_type - ^name => shelf - ^state => state + - ^uid => uuid - disk_count => disk_count endpoints: diff --git a/conf/rest/9.12.0/volume_analytics.yaml b/conf/rest/9.12.0/volume_analytics.yaml index 339b4059e..a3c006c9f 100644 --- a/conf/rest/9.12.0/volume_analytics.yaml +++ b/conf/rest/9.12.0/volume_analytics.yaml @@ -3,7 +3,7 @@ query: api/storage/volumes object: volume counters: - - ^^uuid => instance_uuid + - ^^uuid => uuid - ^name => volume - ^svm.name => svm - filter: diff --git a/conf/rest/9.9.0/svm.yaml b/conf/rest/9.9.0/svm.yaml index 2d2957d11..6e56853df 100644 --- a/conf/rest/9.9.0/svm.yaml +++ b/conf/rest/9.9.0/svm.yaml @@ -53,6 +53,7 @@ plugins: - new_status state online online `0` - ldap_signed ldap_session_security sign sign `0` - ldap_encrypted ldap_session_security seal seal `0` +# - ChangeLog export_options: instance_keys: diff --git a/conf/rest/9.9.0/volume.yaml b/conf/rest/9.9.0/volume.yaml index cdc5c3c9e..5fe0ecc70 100644 --- a/conf/rest/9.9.0/volume.yaml +++ b/conf/rest/9.9.0/volume.yaml @@ -17,6 +17,7 @@ counters: - ^state => state - ^style => style - ^type => type + - ^uuid => uuid - autosize.grow_threshold => autosize_grow_threshold_percent - autosize.maximum => autosize_maximum_size - snapshot_count @@ -128,6 +129,7 @@ plugins: # - volume `MDV_CRS_.+` # # Exclude Metadata volumes, Audit volumes have a “MDV_aud_” prefix # - volume `MDV_aud_.+` +# - ChangeLog export_options: instance_keys: diff --git a/conf/zapi/cdot/9.8.0/aggr.yaml b/conf/zapi/cdot/9.8.0/aggr.yaml index f8ae35c7b..3b7b3b0e5 100644 --- a/conf/zapi/cdot/9.8.0/aggr.yaml +++ b/conf/zapi/cdot/9.8.0/aggr.yaml @@ -5,7 +5,7 @@ object: aggr counters: aggr-attributes: - - ^^aggregate-uuid + - ^^aggregate-uuid => uuid - ^aggregate-name => aggr - aggr-inode-attributes: - files-private-used @@ -63,11 +63,11 @@ counters: plugins: - Aggregate - LabelAgent: - exclude_equals: - - root_aggr `true` - # metric label zapi_value rest_value `default_value` - value_to_num: - - new_status state online online `0` + exclude_equals: + - root_aggr `true` + # metric label zapi_value rest_value `default_value` + value_to_num: + - new_status state online online `0` export_options: instance_keys: diff --git a/conf/zapi/cdot/9.8.0/disk.yaml b/conf/zapi/cdot/9.8.0/disk.yaml index ca861e4c7..ef8dfe92b 100644 --- a/conf/zapi/cdot/9.8.0/disk.yaml +++ b/conf/zapi/cdot/9.8.0/disk.yaml @@ -6,7 +6,7 @@ object: disk counters: storage-disk-info: - ^^disk-name => disk - - ^^disk-uid + - ^^disk-uid => uuid - disk-inventory-info: - ^disk-type => type - ^is-shared => shared diff --git a/conf/zapi/cdot/9.8.0/lif.yaml b/conf/zapi/cdot/9.8.0/lif.yaml index c21229e08..46dde0b41 100644 --- a/conf/zapi/cdot/9.8.0/lif.yaml +++ b/conf/zapi/cdot/9.8.0/lif.yaml @@ -13,6 +13,7 @@ counters: - ^home-port => home_port - ^ipspace => ipspace - ^is-home => is_home + - ^lif-uuid => uuid - ^operational-status => status - ^subnet-name => subnet - data-protocols: diff --git a/conf/zapi/cdot/9.8.0/node.yaml b/conf/zapi/cdot/9.8.0/node.yaml index 0f34d442f..bee05a3a9 100644 --- a/conf/zapi/cdot/9.8.0/node.yaml +++ b/conf/zapi/cdot/9.8.0/node.yaml @@ -25,6 +25,7 @@ counters: - ^node-location => location - ^node-model => model - ^node-serial-number => serial + - ^node-uuid => uuid # - ^node-storage-configuration => storage_configuration - ^node-vendor => vendor - ^product-version => version @@ -46,6 +47,7 @@ plugins: - new_status healthy true up `0` join: - warnings `; ` failed_fan_message,failed_power_message,over_temperature +# - ChangeLog export_options: instance_keys: diff --git a/conf/zapi/cdot/9.8.0/shelf.yaml b/conf/zapi/cdot/9.8.0/shelf.yaml index 77d441f63..f8b92c8d7 100644 --- a/conf/zapi/cdot/9.8.0/shelf.yaml +++ b/conf/zapi/cdot/9.8.0/shelf.yaml @@ -6,7 +6,7 @@ object: shelf counters: storage-shelf-info: - - ^^shelf-uid + - ^^shelf-uid => uuid - ^module-type => module_type - ^op-status => op_status - ^serial-number => serial_number diff --git a/conf/zapi/cdot/9.8.0/svm.yaml b/conf/zapi/cdot/9.8.0/svm.yaml index a28b26c22..887fd3a7d 100644 --- a/conf/zapi/cdot/9.8.0/svm.yaml +++ b/conf/zapi/cdot/9.8.0/svm.yaml @@ -5,7 +5,7 @@ object: svm counters: vserver-info: - - ^^uuid => svm_uuid + - ^^uuid => uuid - ^anti-ransomware-default-volume-state => anti_ransomware_state - ^state => state - ^vserver-name => svm @@ -29,6 +29,7 @@ plugins: - new_status state online online `0` - ldap_signed ldap_session_security sign sign `0` - ldap_encrypted ldap_session_security seal seal `0` +# - ChangeLog export_options: instance_keys: @@ -56,4 +57,3 @@ export_options: - smb_signing_required - state - type - diff --git a/conf/zapi/cdot/9.8.0/volume.yaml b/conf/zapi/cdot/9.8.0/volume.yaml index 76c778928..1298ae3ab 100644 --- a/conf/zapi/cdot/9.8.0/volume.yaml +++ b/conf/zapi/cdot/9.8.0/volume.yaml @@ -12,7 +12,7 @@ counters: - grow-threshold-percent - maximum-size - volume-id-attributes: - - ^^instance-uuid => instance_uuid + - ^^instance-uuid => uuid - ^containing-aggregate-uuid => aggrUuid - ^junction-path => junction_path - ^name => volume @@ -103,6 +103,7 @@ plugins: - svm_root root_volume `false` `No` - node_root root_volume `true` `Yes` - svm_root root_volume `true` `Yes` +# - ChangeLog export_options: instance_keys: diff --git a/docs/plugins.md b/docs/plugins.md index 0ae2719d8..71e2bfd75 100644 --- a/docs/plugins.md +++ b/docs/plugins.md @@ -630,3 +630,5 @@ compute_metric: # "inode_files_total" and multiplying the result by 100. # inode_used_percent = inode_files_used / inode_files_total * 100 ``` + +# ChangeLog \ No newline at end of file diff --git a/grafana/dashboards/cmode/changelogMonitor.json b/grafana/dashboards/cmode/changelogMonitor.json new file mode 100644 index 000000000..67e78e71e --- /dev/null +++ b/grafana/dashboards/cmode/changelogMonitor.json @@ -0,0 +1,1290 @@ +{ + "__inputs": [ + { + "description": "", + "label": "Prometheus", + "name": "DS_PROMETHEUS", + "pluginId": "prometheus", + "pluginName": "Prometheus", + "type": "datasource" + } + ], + "__requires": [ + { + "id": "grafana", + "name": "Grafana", + "type": "grafana", + "version": "8.1.8" + }, + { + "id": "prometheus", + "name": "Prometheus", + "type": "datasource", + "version": "1.0.0" + }, + { + "id": "stat", + "name": "Stat", + "type": "panel", + "version": "" + }, + { + "id": "table", + "name": "Table", + "type": "panel", + "version": "" + }, + { + "id": "text", + "name": "Text", + "type": "panel", + "version": "" + } + ], + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "type": "dashboard" + } + ] + }, + "description": "The ONTAP Changelog Monitor, tracks configuration modifications in volumes, SVMs, and nodes, is deactivated by default. To leverage this feature, one must enable the ChangeLog plugin within the Volume, SVM, and Node Templates.", + "editable": true, + "gnetId": null, + "graphTooltip": 1, + "id": null, + "iteration": 1696405583422, + "links": [ + { + "asDropdown": true, + "icon": "external link", + "includeVars": true, + "keepTime": true, + "tags": [ + "cdot" + ], + "targetBlank": false, + "title": "Related Dashboards", + "tooltip": "", + "type": "dashboards", + "url": "" + } + ], + "panels": [ + { + "collapsed": false, + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 303, + "panels": [], + "title": "Important Information", + "type": "row" + }, + { + "datasource": "${DS_PROMETHEUS}", + "gridPos": { + "h": 2, + "w": 24, + "x": 0, + "y": 1 + }, + "id": 305, + "options": { + "content": "To use this dashboard, enable the ChangeLog plugin for node, svm, and volume templates. For more details, visit the [ChangeLog documentation](https://netapp.github.io/harvest/latest/plugins/#changelog).", + "mode": "markdown" + }, + "pluginVersion": "8.1.8", + "type": "text" + }, + { + "collapsed": true, + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 3 + }, + "id": 282, + "panels": [ + { + "datasource": "${DS_PROMETHEUS}", + "description": "This change type indicates that an existing ONTAP object has been updated.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "noValue": "0", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue", + "value": null + } + ] + }, + "unit": "locale" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 8, + "x": 0, + "y": 1 + }, + "id": 291, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "8.1.8", + "targets": [ + { + "exemplar": false, + "expr": "count by (cluster, datacenter, object) (last_over_time(change_log{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",object=~\"node\",op=\"update\"}[$__range]))", + "instant": true, + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "title": "Update", + "type": "stat" + }, + { + "datasource": "${DS_PROMETHEUS}", + "description": "This change type indicates that a new ONTAP object has been created.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "noValue": "0", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue", + "value": null + } + ] + }, + "unit": "locale" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 8, + "x": 8, + "y": 1 + }, + "id": 285, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "8.1.8", + "targets": [ + { + "exemplar": false, + "expr": "count by (cluster, datacenter, object) (last_over_time(change_log{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",object=~\"node\",op=\"create\"}[$__range]))", + "format": "time_series", + "instant": true, + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "title": "Create", + "type": "stat" + }, + { + "datasource": "${DS_PROMETHEUS}", + "description": "This change type indicates that an existing ONTAP object has been deleted.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "noValue": "0", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue", + "value": null + } + ] + }, + "unit": "locale" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 8, + "x": 16, + "y": 1 + }, + "id": 286, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "8.1.8", + "targets": [ + { + "exemplar": false, + "expr": "count by (cluster, datacenter, object) (last_over_time(change_log{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",object=~\"node\",op=\"delete\"}[$__range]))", + "instant": true, + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "title": "Delete", + "type": "stat" + }, + { + "datasource": "${DS_PROMETHEUS}", + "description": "`Poller Time:` The timestamp when Harvest Poller captured the change \n\n`Object:` The name of the ONTAP object that was changed (e.g., volume, svm, node) \n\n`OP:` The type of change that was made (e.g., create, update, delete) \n\n`Track:` Property of the object which was modified \n\n`New Value:` The new value of the object after the change was made \n\n`Old Value:` The previous value of the object before the change was made", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "displayMode": "auto", + "filterable": true + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "locale" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Time" + }, + "properties": [ + { + "id": "unit", + "value": "dateTimeAsIso" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Poller Time" + }, + "properties": [ + { + "id": "custom.align", + "value": "left" + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 6 + }, + "id": 288, + "options": { + "showHeader": true + }, + "pluginVersion": "8.1.8", + "targets": [ + { + "exemplar": false, + "expr": "last_over_time(change_log{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",object=~\"node\"}[$__range])", + "format": "table", + "instant": true, + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "title": "Node Changes ", + "transformations": [ + { + "id": "filterFieldsByName", + "options": { + "include": { + "names": [ + "cluster", + "datacenter", + "new_value", + "node", + "object", + "old_value", + "op", + "track", + "Value" + ] + } + } + }, + { + "id": "calculateField", + "options": { + "alias": "Time", + "binary": { + "left": "Value", + "operator": "*", + "reducer": "sum", + "right": "1000" + }, + "mode": "binary", + "reduce": { + "reducer": "sum" + }, + "replaceFields": false + } + }, + { + "id": "organize", + "options": { + "excludeByName": { + "Value": true + }, + "indexByName": { + "Time": 0, + "Value": 1, + "cluster": 3, + "datacenter": 2, + "new_value": 8, + "node": 5, + "object": 4, + "old_value": 9, + "op": 6, + "track": 7 + }, + "renameByName": { + "Time": "Poller Time", + "Value": "", + "cluster": "Cluster", + "datacenter": "Datacenter", + "new_value": "New Value", + "node": "Node", + "object": "Object", + "old_value": "Old Value", + "op": "OP", + "track": "Track" + } + } + }, + { + "id": "sortBy", + "options": { + "fields": {}, + "sort": [ + { + "desc": true, + "field": "Poller Time" + } + ] + } + } + ], + "type": "table" + } + ], + "title": "Node Changes", + "type": "row" + }, + { + "collapsed": true, + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 4 + }, + "id": 297, + "panels": [ + { + "datasource": "${DS_PROMETHEUS}", + "description": "This change type indicates that an existing ONTAP object has been updated.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "noValue": "0", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue", + "value": null + } + ] + }, + "unit": "locale" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 8, + "x": 0, + "y": 2 + }, + "id": 298, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "8.1.8", + "targets": [ + { + "exemplar": false, + "expr": "count by (cluster, datacenter, object) (last_over_time(change_log{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",object=~\"svm\",op=\"update\"}[$__range]))", + "instant": true, + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "title": "Update", + "type": "stat" + }, + { + "datasource": "${DS_PROMETHEUS}", + "description": "This change type indicates that a new ONTAP object has been created.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "noValue": "0", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue", + "value": null + } + ] + }, + "unit": "locale" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 8, + "x": 8, + "y": 2 + }, + "id": 292, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "8.1.8", + "targets": [ + { + "exemplar": false, + "expr": "count by (cluster, datacenter, object) (last_over_time(change_log{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",object=~\"svm\",op=\"create\"}[$__range]))", + "format": "time_series", + "instant": true, + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "title": "Create", + "type": "stat" + }, + { + "datasource": "${DS_PROMETHEUS}", + "description": "This change type indicates that an existing ONTAP object has been deleted.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "noValue": "0", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue", + "value": null + } + ] + }, + "unit": "locale" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 8, + "x": 16, + "y": 2 + }, + "id": 300, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "8.1.8", + "targets": [ + { + "exemplar": false, + "expr": "count by (cluster, datacenter, object) (last_over_time(change_log{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",object=~\"svm\",op=\"delete\"}[$__range]))", + "instant": true, + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "title": "Delete", + "type": "stat" + }, + { + "datasource": "${DS_PROMETHEUS}", + "description": "`Poller Time:` The timestamp when Harvest Poller captured the change \n\n`Object:` The name of the ONTAP object that was changed (e.g., volume, svm, node) \n\n`OP:` The type of change that was made (e.g., create, update, delete) \n\n`Track:` Property of the object which was modified \n\n`New Value:` The new value of the object after the change was made \n\n`Old Value:` The previous value of the object before the change was made", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "displayMode": "auto", + "filterable": true + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "locale" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Time" + }, + "properties": [ + { + "id": "unit", + "value": "dateTimeAsIso" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Poller Time" + }, + "properties": [ + { + "id": "custom.align", + "value": "left" + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 7 + }, + "id": 301, + "options": { + "showHeader": true, + "sortBy": [] + }, + "pluginVersion": "8.1.8", + "targets": [ + { + "exemplar": false, + "expr": "last_over_time(change_log{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",object=~\"svm\"}[$__range])", + "format": "table", + "instant": true, + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "title": "SVM Changes ", + "transformations": [ + { + "id": "filterFieldsByName", + "options": { + "include": { + "names": [ + "cluster", + "datacenter", + "new_value", + "node", + "object", + "old_value", + "op", + "track", + "Value", + "volume", + "style", + "svm" + ] + } + } + }, + { + "id": "calculateField", + "options": { + "alias": "Time", + "binary": { + "left": "Value", + "operator": "*", + "reducer": "sum", + "right": "1000" + }, + "mode": "binary", + "reduce": { + "reducer": "sum" + }, + "replaceFields": false + } + }, + { + "id": "organize", + "options": { + "excludeByName": { + "Value": true + }, + "indexByName": { + "Time": 0, + "Value": 1, + "cluster": 3, + "datacenter": 2, + "new_value": 8, + "object": 4, + "old_value": 9, + "op": 6, + "svm": 5, + "track": 7 + }, + "renameByName": { + "Time": "Poller Time", + "Value": "", + "cluster": "Cluster", + "datacenter": "Datacenter", + "new_value": "New Value", + "node": "Node", + "object": "Object", + "old_value": "Old Value", + "op": "OP", + "svm": "SVM", + "track": "Track" + } + } + }, + { + "id": "sortBy", + "options": { + "fields": {}, + "sort": [ + { + "desc": true, + "field": "Poller Time" + } + ] + } + } + ], + "type": "table" + } + ], + "title": "SVM Changes", + "type": "row" + }, + { + "collapsed": true, + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 5 + }, + "id": 290, + "panels": [ + { + "datasource": "${DS_PROMETHEUS}", + "description": "This change type indicates that an existing ONTAP object has been updated.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "noValue": "0", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue", + "value": null + } + ] + }, + "unit": "locale" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 8, + "x": 0, + "y": 3 + }, + "id": 284, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "8.1.8", + "targets": [ + { + "exemplar": false, + "expr": "count by (cluster, datacenter, object) (last_over_time(change_log{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",object=~\"volume\",op=\"update\"}[$__range]))", + "instant": true, + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "title": "Update", + "type": "stat" + }, + { + "datasource": "${DS_PROMETHEUS}", + "description": "This change type indicates that a new ONTAP object has been created.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "noValue": "0", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue", + "value": null + } + ] + }, + "unit": "locale" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 8, + "x": 8, + "y": 3 + }, + "id": 299, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "8.1.8", + "targets": [ + { + "exemplar": false, + "expr": "count by (cluster, datacenter, object) (last_over_time(change_log{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",object=~\"volume\",op=\"create\"}[$__range]))", + "format": "time_series", + "instant": true, + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "title": "Create", + "type": "stat" + }, + { + "datasource": "${DS_PROMETHEUS}", + "description": "This change type indicates that an existing ONTAP object has been deleted.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "noValue": "0", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue", + "value": null + } + ] + }, + "unit": "locale" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 8, + "x": 16, + "y": 3 + }, + "id": 293, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "8.1.8", + "targets": [ + { + "exemplar": false, + "expr": "count by (cluster, datacenter, object) (last_over_time(change_log{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",object=~\"volume\",op=\"delete\"}[$__range]))", + "instant": true, + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "title": "Delete", + "type": "stat" + }, + { + "datasource": "${DS_PROMETHEUS}", + "description": "`Poller Time:` The timestamp when Harvest Poller captured the change \n\n`Object:` The name of the ONTAP object that was changed (e.g., volume, svm, node) \n\n`OP:` The type of change that was made (e.g., create, update, delete) \n\n`Track:` Property of the object which was modified \n\n`New Value:` The new value of the object after the change was made \n\n`Old Value:` The previous value of the object before the change was made", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "displayMode": "auto", + "filterable": true + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "locale" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Time" + }, + "properties": [ + { + "id": "unit", + "value": "dateTimeAsIso" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Poller Time" + }, + "properties": [ + { + "id": "custom.align", + "value": "left" + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 8 + }, + "id": 295, + "options": { + "showHeader": true, + "sortBy": [] + }, + "pluginVersion": "8.1.8", + "targets": [ + { + "exemplar": false, + "expr": "last_over_time(change_log{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",object=~\"volume\"}[$__range])", + "format": "table", + "instant": true, + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "title": "Volume Changes ", + "transformations": [ + { + "id": "filterFieldsByName", + "options": { + "include": { + "names": [ + "cluster", + "datacenter", + "new_value", + "node", + "object", + "old_value", + "op", + "track", + "Value", + "volume", + "style", + "svm" + ] + } + } + }, + { + "id": "calculateField", + "options": { + "alias": "Time", + "binary": { + "left": "Value", + "operator": "*", + "reducer": "sum", + "right": "1000" + }, + "mode": "binary", + "reduce": { + "reducer": "sum" + }, + "replaceFields": false + } + }, + { + "id": "organize", + "options": { + "excludeByName": { + "Value": true + }, + "indexByName": { + "Time": 0, + "Value": 1, + "cluster": 3, + "datacenter": 2, + "new_value": 8, + "node": 11, + "object": 4, + "old_value": 9, + "op": 6, + "style": 12, + "svm": 10, + "track": 7, + "volume": 5 + }, + "renameByName": { + "Time": "Poller Time", + "Value": "", + "cluster": "Cluster", + "datacenter": "Datacenter", + "new_value": "New Value", + "node": "Node", + "object": "Object", + "old_value": "Old Value", + "op": "OP", + "style": "Style", + "svm": "SVM", + "track": "Track", + "volume": "Volume" + } + } + }, + { + "id": "sortBy", + "options": { + "fields": {}, + "sort": [ + { + "desc": true, + "field": "Poller Time" + } + ] + } + } + ], + "type": "table" + } + ], + "title": "Volume Changes", + "type": "row" + } + ], + "refresh": "", + "schemaVersion": 30, + "style": "dark", + "tags": [ + "harvest", + "ontap", + "cdot" + ], + "templating": { + "list": [ + { + "current": { + "selected": false, + "text": "Prometheus", + "value": "Prometheus" + }, + "description": null, + "error": null, + "hide": 2, + "includeAll": false, + "label": "Data Source", + "multi": false, + "name": "DS_PROMETHEUS", + "options": [], + "query": "prometheus", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + }, + { + "allValue": null, + "current": {}, + "datasource": "${DS_PROMETHEUS}", + "definition": "label_values(node_labels{system_type!=\"7mode\"},datacenter)", + "description": null, + "error": null, + "hide": 0, + "includeAll": false, + "label": "", + "multi": true, + "name": "Datacenter", + "options": [], + "query": { + "query": "label_values(node_labels{system_type!=\"7mode\"},datacenter)", + "refId": "Prometheus-Datacenter-Variable-Query" + }, + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": {}, + "datasource": "${DS_PROMETHEUS}", + "definition": "label_values(node_labels{system_type!=\"7mode\",datacenter=~\"$Datacenter\"},cluster)", + "description": null, + "error": null, + "hide": 0, + "includeAll": true, + "label": "", + "multi": true, + "name": "Cluster", + "options": [], + "query": { + "query": "label_values(node_labels{system_type!=\"7mode\",datacenter=~\"$Datacenter\"},cluster)", + "refId": "StandardVariableQuery" + }, + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-3h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ] + }, + "timezone": "", + "title": "ONTAP: Changelog Monitor", + "uid": "", + "version": 1 +} diff --git a/integration/test/dashboard_json_test.go b/integration/test/dashboard_json_test.go index bc4f7f856..c64cc36bb 100644 --- a/integration/test/dashboard_json_test.go +++ b/integration/test/dashboard_json_test.go @@ -90,6 +90,7 @@ var excludeCounters = []string{ "svm_read_total", "svm_vscan", "svm_write_total", + "change_log", "nfs_clients_", } diff --git a/pkg/matrix/instance.go b/pkg/matrix/instance.go index 78c57ba08..6464d7a4c 100644 --- a/pkg/matrix/instance.go +++ b/pkg/matrix/instance.go @@ -51,9 +51,40 @@ func (i *Instance) SetExportable(b bool) { i.exportable = b } -func (i *Instance) Clone(isExportable bool) *Instance { +func (i *Instance) Clone(isExportable bool, labels ...string) *Instance { clone := NewInstance(i.index) - clone.labels = maps.Clone(i.labels) + clone.labels = i.Copy(labels...) clone.exportable = isExportable return clone } + +func (i *Instance) Copy(labels ...string) map[string]string { + if len(labels) == 0 { + return maps.Clone(i.labels) + } + m := make(map[string]string, len(labels)) + for _, k := range labels { + m[k] = i.labels[k] + } + return m +} + +// CompareDiffs iterates through each key in compareKeys, checking if the receiver and prev have the same value for that key. +// When the values are different, return a new Map with the current and previous value +func (i *Instance) CompareDiffs(prev *Instance, compareKeys []string) (map[string]string, map[string]string) { + cur := make(map[string]string) + old := make(map[string]string) + + for _, compareKey := range compareKeys { + val1, ok1 := i.GetLabels()[compareKey] + if !ok1 { + continue + } + val2, ok2 := prev.GetLabels()[compareKey] + if !ok2 || val1 != val2 { + cur[compareKey] = val1 + old[compareKey] = val2 + } + } + return cur, old +} diff --git a/pkg/matrix/matrix.go b/pkg/matrix/matrix.go index af9a381ca..608f865c0 100644 --- a/pkg/matrix/matrix.go +++ b/pkg/matrix/matrix.go @@ -35,6 +35,7 @@ type With struct { Metrics bool Instances bool ExportInstances bool + Labels []string } func New(uuid, object string, identifier string) *Matrix { @@ -83,9 +84,9 @@ func (m *Matrix) Clone(with With) *Matrix { clone.instances = make(map[string]*Instance, len(m.GetInstances())) for key, instance := range m.GetInstances() { if with.ExportInstances { - clone.instances[key] = instance.Clone(instance.IsExportable()) + clone.instances[key] = instance.Clone(instance.IsExportable(), with.Labels...) } else { - clone.instances[key] = instance.Clone(false) + clone.instances[key] = instance.Clone(false, with.Labels...) } } } else {