From 34e6854cc3d817e47ef599d8f227ca2a36375498 Mon Sep 17 00:00:00 2001 From: hardikl Date: Fri, 21 Jul 2023 16:19:31 +0530 Subject: [PATCH] feat: ARW related fields and autoupdate+notification details --- cmd/collectors/rest/plugins/volume/volume.go | 99 +++++++++- conf/rest/9.12.0/ems_destination.yaml | 24 +++ conf/rest/9.12.0/support_auto_update.yaml | 15 ++ conf/rest/9.12.0/volume.yaml | 2 +- conf/rest/default.yaml | 2 + grafana/dashboards/cmode/security.json | 188 +++++++++++++++---- 6 files changed, 293 insertions(+), 37 deletions(-) create mode 100644 conf/rest/9.12.0/ems_destination.yaml create mode 100644 conf/rest/9.12.0/support_auto_update.yaml diff --git a/cmd/collectors/rest/plugins/volume/volume.go b/cmd/collectors/rest/plugins/volume/volume.go index 3ac3da398..63f8583e7 100644 --- a/cmd/collectors/rest/plugins/volume/volume.go +++ b/cmd/collectors/rest/plugins/volume/volume.go @@ -5,13 +5,16 @@ package volume import ( + "fmt" "github.com/netapp/harvest/v2/cmd/collectors" "github.com/netapp/harvest/v2/cmd/poller/plugin" "github.com/netapp/harvest/v2/cmd/tools/rest" "github.com/netapp/harvest/v2/pkg/conf" "github.com/netapp/harvest/v2/pkg/errs" "github.com/netapp/harvest/v2/pkg/matrix" + "github.com/netapp/harvest/v2/pkg/tree/node" "github.com/tidwall/gjson" + "regexp" "strconv" "strings" "time" @@ -22,6 +25,7 @@ type Volume struct { currentVal int client *rest.Client aggrsMap map[string]string // aggregate-uuid -> aggregate-name map + arw *matrix.Matrix } func New(p *plugin.AbstractPlugin) plugin.Plugin { @@ -55,6 +59,16 @@ func (my *Volume) Init() error { return err } + my.arw = matrix.New(my.Parent+".Volume", "volume_arw", "volume_arw") + exportOptions := node.NewS("export_options") + instanceKeys := exportOptions.NewChildS("instance_keys", "") + instanceKeys.NewChildS("", "ArwStatus") + my.arw.SetExportOptions(exportOptions) + _, err = my.arw.NewMetricFloat64("status", "status") + if err != nil { + my.Logger.Error().Stack().Err(err).Msg("add metric") + return err + } return nil } @@ -79,8 +93,11 @@ func (my *Volume) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, erro // update volume instance labels my.updateVolumeLabels(data) + // parse anti_ransomware_start_time, antiRansomwareState for all volumes and export at cluster level + my.handleARWProtection(data) + my.currentVal++ - return nil, nil + return []*matrix.Matrix{my.arw}, nil } func (my *Volume) updateVolumeLabels(data *matrix.Matrix) { @@ -96,6 +113,67 @@ func (my *Volume) updateVolumeLabels(data *matrix.Matrix) { } } +func (my *Volume) handleARWProtection(data *matrix.Matrix) { + var ( + arwInstance *matrix.Instance + err error + ) + + // Purge and reset data + my.arw.PurgeInstances() + my.arw.Reset() + + // Set all global labels + my.arw.SetGlobalLabels(data.GetGlobalLabels()) + + learningModeCount := 0 + learningCompleted := 0 + disabledCount := 0 + + for _, volume := range data.GetInstances() { + if arwState := volume.GetLabel("antiRansomwareState"); arwState != "" { + if arwState == "dry_run" || arwState == "enable_paused" { + if arwStartTime := volume.GetLabel("anti_ransomware_start_time"); arwStartTime != "" { + // If ARW startTime is more than 30 days old, which indicates that learning mode has been finished. + if (float64(time.Now().Unix()) - HandleTimestamp(arwStartTime)) > 2629743 { + learningCompleted++ + } + } + learningModeCount++ + } else if arwState == "disabled" { + disabledCount++ + } + } + } + + arwInstanceKey := data.GetGlobalLabels().Get("cluster") + data.GetGlobalLabels().Get("datacenter") + if arwInstance, err = my.arw.NewInstance(arwInstanceKey); err != nil { + my.Logger.Error().Stack().Err(err).Str("arwInstanceKey", arwInstanceKey).Msg("Failed to create arw instance") + return + } + + if disabledCount > 0 { + arwInstance.SetLabel("ArwStatus", "Not Monitoring") + } else if learningModeCount > 0 { + if learningCompleted > 0 { + arwInstance.SetLabel("ArwStatus", "Switch to Active Mode") + } else { + arwInstance.SetLabel("ArwStatus", "Learning Mode") + } + } else { + arwInstance.SetLabel("ArwStatus", "Active Mode") + } + + m := my.arw.GetMetric("status") + // populate numeric data + value := 1.0 + if err = m.SetValueFloat64(arwInstance, value); err != nil { + my.Logger.Error().Stack().Err(err).Float64("value", value).Msg("Failed to parse value") + } else { + my.Logger.Debug().Float64("value", value).Msg("added value") + } +} + func (my *Volume) getEncryptedDisks() ([]gjson.Result, error) { var ( result []gjson.Result @@ -126,3 +204,22 @@ func (my *Volume) updateAggrMap(disks []gjson.Result) { } } } + +// Example: timestamp: 2020-12-02T18:36:19-08:00 +var regexTimeStamp = regexp.MustCompile( + `[+-]?\d{4}(-[01]\d(-[0-3]\d(T[0-2]\d:[0-5]\d:?([0-5]\d(\.\d+)?)?[+-][0-2]\d:[0-5]\d?)?)?)?`) + +func HandleTimestamp(value string) float64 { + var timestamp time.Time + var err error + + if match := regexTimeStamp.MatchString(value); match { + // example: 2020-12-02T18:36:19-08:00 ==> 1606962979 + if timestamp, err = time.Parse(time.RFC3339, value); err != nil { + fmt.Printf("%v", err) + return 0 + } + return float64(timestamp.Unix()) + } + return 0 +} diff --git a/conf/rest/9.12.0/ems_destination.yaml b/conf/rest/9.12.0/ems_destination.yaml new file mode 100644 index 000000000..531bb9bae --- /dev/null +++ b/conf/rest/9.12.0/ems_destination.yaml @@ -0,0 +1,24 @@ + +name: EmsDestination +query: api/support/ems/destinations +object: ems_destination + +counters: + - ^^destination => destination + - ^^name => name + - ^^type => type + - ^certificate => certificate + - ^filter => filter + - ^syslog => syslog + - ^system_defined => system_defined + - filter: + - system_defined=false + +export_options: + instance_keys: + - destination + - name + - type + instance_labels: + - filter + - syslog diff --git a/conf/rest/9.12.0/support_auto_update.yaml b/conf/rest/9.12.0/support_auto_update.yaml new file mode 100644 index 000000000..0a74559ef --- /dev/null +++ b/conf/rest/9.12.0/support_auto_update.yaml @@ -0,0 +1,15 @@ + +name: SupportAutoUpdate +query: api/support/auto-update +object: support_auto_update + +counters: + - ^enabled => auto_update_enabled + +#export_options: +# include_all_labels: true +export_options: + instance_keys: + - auto_update_enabled + instance_labels: + - auto_update_enabled diff --git a/conf/rest/9.12.0/volume.yaml b/conf/rest/9.12.0/volume.yaml index f163fc3c7..31df04304 100644 --- a/conf/rest/9.12.0/volume.yaml +++ b/conf/rest/9.12.0/volume.yaml @@ -9,6 +9,7 @@ counters: - ^^svm.name => svm - ^aggregates.#.name => aggr - ^aggregates.#.uuid => aggrUuid # handled in plugin for flexgroup + - ^anti_ransomware.dry_run_start_time => anti_ransomware_start_time - ^anti_ransomware.state => antiRansomwareState - ^encryption.enabled => isEncrypted - ^is_svm_root => svm_root @@ -18,7 +19,6 @@ counters: - ^state => state - ^style => style - ^type => type - - anti_ransomware.dry_run_start_time(timestamp) => anti_ransomware_start_time - autosize.grow_threshold => autosize_grow_threshold_percent - autosize.maximum => autosize_maximum_size - snapshot_count diff --git a/conf/rest/default.yaml b/conf/rest/default.yaml index bb0722be6..1d17f30e0 100644 --- a/conf/rest/default.yaml +++ b/conf/rest/default.yaml @@ -13,6 +13,7 @@ objects: CloudTarget: cloud_target.yaml ClusterPeer: clusterpeer.yaml Disk: disk.yaml + EmsDestination: ems_destination.yaml # ExportRule: exports.yaml LIF: lif.yaml Health: health.yaml @@ -42,6 +43,7 @@ objects: Status: status.yaml Subsystem: subsystem.yaml Support: support.yaml + SupportAutoUpdate: support_auto_update.yaml SVM: svm.yaml Volume: volume.yaml VolumeAnalytics: volume_analytics.yaml diff --git a/grafana/dashboards/cmode/security.json b/grafana/dashboards/cmode/security.json index 2eacb6b2b..cb575ca59 100644 --- a/grafana/dashboards/cmode/security.json +++ b/grafana/dashboards/cmode/security.json @@ -2508,6 +2508,16 @@ } }, "type": "value" + }, + { + "options": { + "match": "null", + "result": { + "index": 2, + "text": "❌ Disabled" + } + }, + "type": "special" } ] }, @@ -3301,6 +3311,10 @@ "type": "special" } ] + }, + { + "id": "custom.width", + "value": 115 } ] }, @@ -3334,6 +3348,10 @@ "type": "range" } ] + }, + { + "id": "custom.width", + "value": 135 } ] }, @@ -3367,13 +3385,17 @@ "type": "range" } ] + }, + { + "id": "custom.width", + "value": 170 } ] }, { "matcher": { "id": "byName", - "options": "SVMs configured for ARW" + "options": "ARW Protection for SVMs" }, "properties": [ { @@ -3383,7 +3405,7 @@ "options": { "0": { "index": 0, - "text": "Yes" + "text": "Configured" } }, "type": "value" @@ -3393,7 +3415,7 @@ "from": 1, "result": { "index": 1, - "text": "❌ No" + "text": "❌ Not Configured" }, "to": 9999 }, @@ -3404,19 +3426,67 @@ "match": "null", "result": { "index": 2, - "text": "❌ No" + "text": "❌ Not Configured" + } + }, + "type": "special" + } + ] + }, + { + "id": "custom.width", + "value": 195 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "ARW Protection for Volumes" + }, + "properties": [ + { + "id": "mappings", + "value": [ + { + "options": { + "Learning Mode": { + "index": 3, + "text": "❌ Learning Mode" + }, + "Not Monitoring": { + "index": 1, + "text": "❌ Not Monitoring" + }, + "Switch to Active Mode": { + "index": 2, + "text": "❌ Switch to Active Mode" + } + }, + "type": "value" + }, + { + "options": { + "match": "null", + "result": { + "index": 0, + "text": "❌ Not Applicable" } }, "type": "special" } ] + }, + { + "id": "custom.width", + "value": 218 } ] }, { "matcher": { "id": "byName", - "options": "Volumes Monitored for ARW" + "options": "Notifications Configured" }, "properties": [ { @@ -3426,7 +3496,7 @@ "options": { "0": { "index": 0, - "text": "Yes" + "text": "❌ Not Configured" } }, "type": "value" @@ -3436,7 +3506,7 @@ "from": 1, "result": { "index": 1, - "text": "❌ No" + "text": "Configured" }, "to": 9999 }, @@ -3447,7 +3517,43 @@ "match": "null", "result": { "index": 2, - "text": "❌ No" + "text": "Not Applicable" + } + }, + "type": "special" + } + ] + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Automatic Updates Configured" + }, + "properties": [ + { + "id": "mappings", + "value": [ + { + "options": { + "false": { + "index": 0, + "text": "❌ Not Configured" + }, + "true": { + "index": 1, + "text": "Configured" + } + }, + "type": "value" + }, + { + "options": { + "match": "null", + "result": { + "index": 2, + "text": "Not Applicable" } }, "type": "special" @@ -3474,12 +3580,7 @@ "show": false }, "showHeader": true, - "sortBy": [ - { - "desc": false, - "displayName": "SVMs configured for ARW" - } - ] + "sortBy": [] }, "pluginVersion": "8.1.8", "targets": [ @@ -3624,7 +3725,7 @@ }, { "exemplar": false, - "expr": "group by (datacenter, cluster)(support_labels{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\", asup_enabled=\"true\", asup_https_configured!=\"https\"} or security_labels{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\", telnet_enabled=\"true\"} or security_labels{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\", rsh_enabled=\"true\"} or group by (datacenter, cluster)(security_account_labels{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\", hash_algorithm=\"md5\"}) > 0 or cluster_peer_labels{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\", encryption_state=\"\"} or cluster_peer_labels{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\", encryption_state=\"none\"} or group by (datacenter, cluster) (security_account_labels{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\", locked=\"false\"}) > 0 or security_login_labels{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\", scope=\"cluster\", banner=\"\"} or security_login_labels{datacenter=~\"$Datacenter\", cluster=~\"$Cluster\", svm=~\"$Cluster\"} or count by (datacenter, cluster) (ntpserver_labels{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\"}) < 1 or security_audit_destination_status{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\", protocol!=\"tcp_encrypted\"} or security_labels{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\", fips_enabled=\"false\"} or svm_labels{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\", type=\"admin\", ciphers=~\".*_cbc.*\"} or group by (datacenter, cluster) ((svm_labels{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\", cifs_protocol_enabled=\"true\"} or svm_labels{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\", nfs_protocol_enabled=\"true\"} and svm_labels{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\", audit_protocol_enabled=\"false\"}) or security_ssh_labels{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\", ciphers=~\".*_cbc.*\"} or security_login_labels{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\", scope=\"svm\", banner=\"\"} or security_login_labels{datacenter=~\"$Datacenter\", cluster=~\"$Cluster\", svm=~\"$SVM\"} or sum by (datacenter, cluster)(volume_labels{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",snapshot_policy=\"none\"}) > 0 or sum by (datacenter, cluster)(volume_labels{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",snapshot_autodelete=\"true\"}) > 0 or sum by (datacenter, cluster)(svm_labels{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",anti_ransomware_state=~\"|.*disabled\",type=\"data\"}) > 0 or sum by(datacenter, cluster)(volume_labels{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",antiRansomwareState=~\"|.*disabled\"}) > 0))", + "expr": "group by (datacenter, cluster)(support_labels{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\", asup_enabled=\"true\", asup_https_configured!=\"https\"} or security_labels{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\", telnet_enabled=\"true\"} or security_labels{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\", rsh_enabled=\"true\"} or group by (datacenter, cluster)(security_account_labels{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\", hash_algorithm=\"md5\"}) > 0 or cluster_peer_labels{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\", encryption_state=\"\"} or cluster_peer_labels{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\", encryption_state=\"none\"} or group by (datacenter, cluster) (security_account_labels{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\", locked=\"false\"}) > 0 or security_login_labels{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\", scope=\"cluster\", banner=\"\"} or security_login_labels{datacenter=~\"$Datacenter\", cluster=~\"$Cluster\", svm=~\"$Cluster\"} or count by (datacenter, cluster) (ntpserver_labels{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\"}) < 1 or security_audit_destination_status{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\", protocol!=\"tcp_encrypted\"} or security_labels{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\", fips_enabled=\"false\"} or svm_labels{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\", type=\"admin\", ciphers=~\".*_cbc.*\"} or group by (datacenter, cluster) ((svm_labels{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\", cifs_protocol_enabled=\"true\"} or svm_labels{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\", nfs_protocol_enabled=\"true\"} and svm_labels{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\", audit_protocol_enabled=\"false\"}) or security_ssh_labels{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\", ciphers=~\".*_cbc.*\"} or security_login_labels{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\", scope=\"svm\", banner=\"\"} or security_login_labels{datacenter=~\"$Datacenter\", cluster=~\"$Cluster\", svm=~\"$SVM\"} or sum by (datacenter, cluster)(volume_labels{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",snapshot_policy=\"none\"}) > 0 or sum by (datacenter, cluster)(volume_labels{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",snapshot_autodelete=\"true\"}) > 0 or sum by (datacenter, cluster)(svm_labels{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",anti_ransomware_state=~\"|.*disabled\",type=\"data\"}) > 0 or volume_arw_status{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",ArwStatus!=\"Active Mode\"} * on (instance) group_left() metadata_collector_instances{datacenter=~\"$Datacenter\",collector=\"Rest\", object=\"Volume\"} or count by (datacenter, cluster)(ems_destination_labels{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\"}) > 0 or support_auto_update_labels{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",auto_update_enabled=\"false\"}))", "format": "table", "hide": false, "instant": true, @@ -3664,7 +3765,7 @@ }, { "exemplar": false, - "expr": "sum by(datacenter, cluster)(volume_labels{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",antiRansomwareState=~\"|.*disabled\"})", + "expr": "volume_arw_status{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\"} * on (instance) group_left() metadata_collector_instances{datacenter=~\"$Datacenter\",collector=\"Rest\", object=\"Volume\"}", "format": "table", "hide": false, "instant": true, @@ -3674,13 +3775,23 @@ }, { "exemplar": false, - "expr": "count by (datacenter, cluster)((time() - (volume_labels{antiRansomwareState=~\"dry_run|enable_paused\"} * on(datacenter, cluster, svm, volume) group_right () volume_anti_ransomware_start_time{}))> 2629743)", + "expr": "count by (datacenter, cluster)(ems_destination_labels{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\"})", "format": "table", "hide": false, "instant": true, "interval": "", "legendFormat": "", "refId": "T" + }, + { + "exemplar": false, + "expr": "support_auto_update_labels{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\"}", + "format": "table", + "hide": false, + "instant": true, + "interval": "", + "legendFormat": "", + "refId": "U" } ], "title": "Cluster Compliance", @@ -3707,8 +3818,9 @@ "Value #H", "Value #Q", "Value #R", - "Value #S", - "Value #T" + "Value #T", + "auto_update_enabled", + "ArwStatus" ] } } @@ -3728,11 +3840,12 @@ "Value #D": false, "Value #E": false, "Value #G": false, - "Value #T": true, + "Value #S": true, + "Value #T": false, "__name__": true, "__name__ 1": true, "__name__ 2": true, - "datacenter": true, + "datacenter": false, "datacenter 1": true, "datacenter 2": true, "fips_enabled": false, @@ -3747,26 +3860,29 @@ "methods": true }, "indexByName": { + "ArwStatus": 6, "Value #H": 2, - "Value #I": 12, - "Value #K": 16, - "Value #L": 13, - "Value #M": 14, - "Value #N": 15, + "Value #I": 13, + "Value #K": 17, + "Value #L": 14, + "Value #M": 15, + "Value #N": 16, "Value #P": 0, "Value #Q": 3, "Value #R": 4, "Value #S": 5, - "asup_enabled": 9, - "certificateExpiryStatus": 6, - "certificateIssuerType": 17, + "asup_enabled": 10, + "certificateExpiryStatus": 7, + "certificateIssuerType": 18, "cluster": 1, - "fips_enabled": 7, - "locked": 10, - "rsh_enabled": 11, - "telnet_enabled": 8 + "datacenter": 19, + "fips_enabled": 8, + "locked": 11, + "rsh_enabled": 12, + "telnet_enabled": 9 }, "renameByName": { + "ArwStatus": "ARW Protection for Volumes", "Value #A": "Autosupport Https Transport", "Value #C": "Default Admin User", "Value #D": "MD5 in use", @@ -3780,12 +3896,14 @@ "Value #N": "Certificate Users", "Value #P": "Compliant", "Value #Q": "Snapshot Autodelete", - "Value #R": "SVMs configured for ARW", - "Value #S": "Volumes Monitored for ARW", + "Value #R": "ARW Protection for SVMs", + "Value #S": "Volumes Monitored for ARW Protection", + "Value #T": "Notifications Configured", "activediruser": "Active Directory Users", "asupEnabled": "", "asupHttpsConfigured": "", "asup_enabled": "Autosupport Https Transport", + "auto_update_enabled": "Automatic Updates Configured", "banner": "Login Banner", "certificateExpiryStatus": "Cluster Certificate Validity", "certificateIssuerType": "Certificate Issuer Type",