feat: Harvest should support recording and replaying HTTP requests

cmd/collectors/keyperf/keyperf.go

@@ -5,6 +5,7 @@ import (
 	"github.com/netapp/harvest/v2/cmd/collectors/rest"
 	"github.com/netapp/harvest/v2/cmd/poller/collector"
 	"github.com/netapp/harvest/v2/cmd/poller/plugin"
+	"github.com/netapp/harvest/v2/pkg/conf"
 	"github.com/netapp/harvest/v2/pkg/errs"
 	"github.com/netapp/harvest/v2/pkg/matrix"
 	"github.com/netapp/harvest/v2/pkg/slogx"
@@ -16,14 +17,14 @@ import (
 )
 
 const (
-	latencyIoReqd    = 10
-	numRecordsToSave = 60 // Number of records to save when using the recorder
+	latencyIoReqd = 10
 )
 
 type KeyPerf struct {
 	*rest.Rest    // provides: AbstractCollector, Client, Object, Query, TemplateFn, TemplateType
 	perfProp      *perfProp
-	pollDataCalls uint8
+	pollDataCalls int
+	recordsToSave int // Number of records to save when using the recorder
 }
 
 type counter struct {
@@ -91,6 +92,8 @@ func (kp *KeyPerf) Init(a *collector.AbstractCollector) error {
 
 	kp.buildCounters()
 
+	kp.recordsToSave = collector.RecordKeepLast(kp.Params, kp.Logger)
+
 	kp.Logger.Debug(
 		"initialized cache",
 		slog.Int("numMetrics", len(kp.Prop.Metrics)),
@@ -235,12 +238,21 @@ func (kp *KeyPerf) PollData() (map[string]*matrix.Matrix, error) {
 	}
 
 	kp.pollDataCalls++
-	if kp.pollDataCalls > numRecordsToSave {
+	if kp.pollDataCalls >= kp.recordsToSave {
 		kp.pollDataCalls = 0
 	}
 
-	headers := map[string]string{
-		"From": strconv.Itoa(int(kp.pollDataCalls)),
+	var headers map[string]string
+
+	poller, err := conf.PollerNamed(kp.Options.Poller)
+	if err != nil {
+		slog.Error("failed to find poller", slogx.Err(err), slog.String("poller", kp.Options.Poller))
+	}
+
+	if poller.IsRecording() {
+		headers = map[string]string{
+			"From": strconv.Itoa(kp.pollDataCalls),
+		}
 	}
 
 	perfRecords, err = kp.GetRestData(href, headers)

cmd/collectors/restperf/restperf.go

@@ -17,6 +17,7 @@ import (
 	"github.com/netapp/harvest/v2/cmd/poller/collector"
 	"github.com/netapp/harvest/v2/cmd/poller/plugin"
 	"github.com/netapp/harvest/v2/cmd/tools/rest"
+	"github.com/netapp/harvest/v2/pkg/conf"
 	"github.com/netapp/harvest/v2/pkg/errs"
 	"github.com/netapp/harvest/v2/pkg/matrix"
 	"github.com/netapp/harvest/v2/pkg/set"
@@ -42,7 +43,6 @@ const (
 	objWorkloadClass       = "user_defined|system_defined"
 	objWorkloadVolumeClass = "autovolume"
 	timestampMetricName    = "timestamp"
-	numRecordsToSave       = 60 // Number of records to save when using the recorder
 )
 
 var (
@@ -69,8 +69,9 @@ type RestPerf struct {
 	perfProp            *perfProp
 	archivedMetrics     map[string]*rest2.Metric // Keeps metric definitions that are not found in the counter schema. These metrics may be available in future ONTAP versions.
 	hasInstanceSchedule bool
-	pollInstanceCalls   uint8
-	pollDataCalls       uint8
+	pollInstanceCalls   int
+	pollDataCalls       int
+	recordsToSave       int // Number of records to save when using the recorder
 }
 
 type counter struct {
@@ -147,6 +148,8 @@ func (r *RestPerf) Init(a *collector.AbstractCollector) error {
 
 	r.InitSchedule()
 
+	r.recordsToSave = collector.RecordKeepLast(r.Params, r.Logger)
+
 	r.Logger.Debug(
 		"initialized cache",
 		slog.Int("numMetrics", len(r.Prop.Metrics)),
@@ -722,14 +725,25 @@ func (r *RestPerf) PollData() (map[string]*matrix.Matrix, error) {
 	}
 
 	r.pollDataCalls++
-	if r.pollDataCalls > numRecordsToSave {
+	if r.pollDataCalls >= r.recordsToSave {
 		r.pollDataCalls = 0
 	}
 
-	headers := map[string]string{
-		"From": strconv.Itoa(int(r.pollDataCalls)),
+	var headers map[string]string
+
+	poller, err := conf.PollerNamed(r.Options.Poller)
+	if err != nil {
+		slog.Error("failed to find poller", slogx.Err(err), slog.String("poller", r.Options.Poller))
 	}
+
+	if poller.IsRecording() {
+		headers = map[string]string{
+			"From": strconv.Itoa(r.pollDataCalls),
+		}
+	}
+
 	err = rest.FetchRestPerfData(r.Client, href, &perfRecords, headers)
+
 	if err != nil {
 		return nil, fmt.Errorf("failed to fetch href=%s %w", href, err)
 	}
@@ -1485,12 +1499,21 @@ func (r *RestPerf) PollInstance() (map[string]*matrix.Matrix, error) {
 	}
 
 	r.pollInstanceCalls++
-	if r.pollInstanceCalls > numRecordsToSave/3 {
+	if r.pollInstanceCalls > r.recordsToSave/3 {
 		r.pollInstanceCalls = 0
 	}
 
-	headers := map[string]string{
-		"From": strconv.Itoa(int(r.pollInstanceCalls)),
+	var headers map[string]string
+
+	poller, err := conf.PollerNamed(r.Options.Poller)
+	if err != nil {
+		slog.Error("failed to find poller", slogx.Err(err), slog.String("poller", r.Options.Poller))
+	}
+
+	if poller.IsRecording() {
+		headers = map[string]string{
+			"From": strconv.Itoa(r.pollInstanceCalls),
+		}
 	}
 
 	href := rest.NewHrefBuilder().

cmd/collectors/zapi/collector/zapi.go

@@ -19,23 +19,21 @@ import (
 	"github.com/netapp/harvest/v2/cmd/collectors/zapi/plugins/systemnode"
 	"github.com/netapp/harvest/v2/cmd/collectors/zapi/plugins/volume"
 	"github.com/netapp/harvest/v2/cmd/collectors/zapi/plugins/workload"
+	"github.com/netapp/harvest/v2/cmd/poller/collector"
 	"github.com/netapp/harvest/v2/cmd/poller/plugin"
+	client "github.com/netapp/harvest/v2/pkg/api/ontapi/zapi"
 	"github.com/netapp/harvest/v2/pkg/conf"
+	"github.com/netapp/harvest/v2/pkg/errs"
+	"github.com/netapp/harvest/v2/pkg/matrix"
 	"github.com/netapp/harvest/v2/pkg/set"
 	"github.com/netapp/harvest/v2/pkg/slogx"
+	"github.com/netapp/harvest/v2/pkg/tree/node"
 	"github.com/netapp/harvest/v2/pkg/util"
 	"log/slog"
 	"sort"
 	"strconv"
 	"strings"
 	"time"
-
-	"github.com/netapp/harvest/v2/cmd/poller/collector"
-	"github.com/netapp/harvest/v2/pkg/errs"
-	"github.com/netapp/harvest/v2/pkg/matrix"
-	"github.com/netapp/harvest/v2/pkg/tree/node"
-
-	client "github.com/netapp/harvest/v2/pkg/api/ontapi/zapi"
 )
 
 const BatchSize = "500"

cmd/collectors/zapiperf/zapiperf.go

@@ -40,6 +40,7 @@ import (
 	"github.com/netapp/harvest/v2/cmd/collectors/zapiperf/plugins/vscan"
 	"github.com/netapp/harvest/v2/cmd/poller/collector"
 	"github.com/netapp/harvest/v2/cmd/poller/plugin"
+	"github.com/netapp/harvest/v2/pkg/conf"
 	"github.com/netapp/harvest/v2/pkg/errs"
 	"github.com/netapp/harvest/v2/pkg/matrix"
 	"github.com/netapp/harvest/v2/pkg/set"
@@ -70,7 +71,6 @@ const (
 	objWorkloadVolumeClass  = "autovolume"
 	BILLION                 = 1_000_000_000
 	timestampMetricName     = "timestamp"
-	numRecordsToSave        = 60 // Number of records to save when using the recorder
 )
 
 var workloadDetailMetrics = []string{"resource_latency"}
@@ -90,8 +90,9 @@ type ZapiPerf struct {
 	keyName           string
 	keyNameIndex      int
 	testFilePath      string // Used only from unit test
-	pollDataCalls     uint8
-	pollInstanceCalls uint8
+	recordsToSave     int    // Number of records to save when using the recorder
+	pollDataCalls     int
+	pollInstanceCalls int
 }
 
 func init() {
@@ -127,6 +128,8 @@ func (z *ZapiPerf) Init(a *collector.AbstractCollector) error {
 
 	z.InitQOS()
 
+	z.recordsToSave = collector.RecordKeepLast(z.Params, z.Logger)
+
 	z.Logger.Debug("initialized")
 	return nil
 }
@@ -487,12 +490,21 @@ func (z *ZapiPerf) PollData() (map[string]*matrix.Matrix, error) {
 		}
 
 		z.pollDataCalls++
-		if z.pollDataCalls > numRecordsToSave {
+		if z.pollDataCalls >= z.recordsToSave {
 			z.pollDataCalls = 0
 		}
 
-		headers := map[string]string{
-			"From": strconv.Itoa(int(z.pollDataCalls)),
+		var headers map[string]string
+
+		poller, err := conf.PollerNamed(z.Options.Poller)
+		if err != nil {
+			slog.Error("failed to find poller", slogx.Err(err), slog.String("poller", z.Options.Poller))
+		}
+
+		if poller.IsRecording() {
+			headers = map[string]string{
+				"From": strconv.Itoa(z.pollDataCalls),
+			}
 		}
 
 		response, rd, pd, err := z.Client.InvokeWithTimers(z.testFilePath, headers)
@@ -1636,12 +1648,21 @@ func (z *ZapiPerf) PollInstance() (map[string]*matrix.Matrix, error) {
 		apiT = time.Now()
 
 		z.pollInstanceCalls++
-		if z.pollInstanceCalls > numRecordsToSave/3 {
+		if z.pollInstanceCalls >= z.recordsToSave/3 {
 			z.pollInstanceCalls = 0
 		}
 
-		headers := map[string]string{
-			"From": strconv.Itoa(int(z.pollInstanceCalls)),
+		var headers map[string]string
+
+		poller, err := conf.PollerNamed(z.Options.Poller)
+		if err != nil {
+			slog.Error("failed to find poller", slogx.Err(err), slog.String("poller", z.Options.Poller))
+		}
+
+		if poller.IsRecording() {
+			headers = map[string]string{
+				"From": strconv.Itoa(z.pollInstanceCalls),
+			}
 		}
 
 		responseData, err := z.Client.InvokeBatchRequest(request, batchTag, z.testFilePath, headers)

cmd/poller/collector/collector.go

@@ -73,7 +73,8 @@ type Collector interface {
 }
 
 const (
-	begin = "zBegin"
+	begin                = "zBegin"
+	DefaultRecordsToSave = 60
 )
 
 // Status defines the possible states of a collector

cmd/poller/collector/helpers.go

@@ -27,6 +27,7 @@ import (
 	"path/filepath"
 	"regexp"
 	"sort"
+	"strconv"
 	"strings"
 )
 
@@ -236,3 +237,29 @@ func GetBuiltinPlugin(name string, abc *plugin.AbstractPlugin) plugin.Plugin {
 
 	return nil
 }
+
+func RecordKeepLast(n *node.Node, logger *slog.Logger) int {
+	r := n.GetChildS("recorder")
+	if r == nil {
+		return DefaultRecordsToSave
+	}
+	if !r.HasChildS("path") {
+		return DefaultRecordsToSave
+	}
+
+	kl := r.GetChildContentS("keep_last")
+	if kl != "" {
+		keep, err := strconv.Atoi(kl)
+		if err != nil {
+			logger.Error(
+				"invalid keep_last value. Using default.",
+				slog.Int("default", DefaultRecordsToSave),
+				slog.String("value", kl),
+			)
+			return DefaultRecordsToSave
+		}
+		return keep
+	}
+
+	return DefaultRecordsToSave
+}

docs/configure-harvest-basic.md

@@ -205,10 +205,11 @@ to prevent sensitive information from being stored on disk.
 
 The `recorder` section in the `harvest.yml` file allows you to configure the HTTP recorder.
 
-| parameter | type                | description                                                                                                                                   |
-|-----------|---------------------|-----------------------------------------------------------------------------------------------------------------------------------------------|
-| `path`    | string **required** | Path to a directory. Recorded requests and responses will be stored here. Replaying will read the requests and responses from this directory. |
-| `mode`    | string **required** | `record` or `replay`                                                                                                                          |
+| parameter   | type                | description                                                                                                                                   | default |
+|-------------|---------------------|-----------------------------------------------------------------------------------------------------------------------------------------------|--------:|
+| `path`      | string **required** | Path to a directory. Recorded requests and responses will be stored here. Replaying will read the requests and responses from this directory. |         |
+| `mode`      | string **required** | `record` or `replay`                                                                                                                          |         |
+| `keep_last` | optional, int       | When mode is `record`, the number of records to keep before overwriting                                                                       |      60 |
 
 # Authentication
 

pkg/auth/auth.go

@@ -498,7 +498,7 @@ func (c *Credentials) Transport(request *http.Request, poller *conf.Poller) (htt
 		}
 	}
 
-	if poller.Recorder.Path == "" {
+	if !poller.IsRecording() {
 		return transport, nil
 	}
 

pkg/conf/conf.go

@@ -504,8 +504,9 @@ type ExportDef struct {
 }
 
 type Recorder struct {
-	Path string `yaml:"path,omitempty"`
-	Mode string `yaml:"mode,omitempty"` // record or replay
+	Path     string `yaml:"path,omitempty"`
+	Mode     string `yaml:"mode,omitempty"`      // record or replay
+	KeepLast string `yaml:"keep_last,omitempty"` // number of records to keep before overwriting
 }
 
 func (e *ExportDef) UnmarshalYAML(n *yaml.Node) error {
@@ -584,6 +585,10 @@ func (p *Poller) Union(defaults *Poller) {
 	p.CredentialsScript.Path = pCredentialsScript
 }
 
+func (p *Poller) IsRecording() bool {
+	return p.Recorder.Path != ""
+}
+
 // ZapiPoller creates a poller out of a node, this is a bridge between the node and struct-based code
 // Used by ZAPI based code
 func ZapiPoller(n *node.Node) *Poller {