From 3748f0678585a540003b346ecce8b925e836516b Mon Sep 17 00:00:00 2001 From: Karim Radhouani Date: Mon, 25 Mar 2024 13:05:13 -0700 Subject: [PATCH 1/6] add processor command --- pkg/app/get.go | 1 - pkg/app/processor.go | 167 ++++++++++++++++ pkg/cmd/processor/processor.go | 22 +++ pkg/cmd/root.go | 2 + pkg/config/config.go | 6 + pkg/formatters/event.go | 12 +- pkg/formatters/event_strings/event_strings.go | 5 +- .../prometheus_output/prometheus_common.go | 172 ++++++++++++++++- .../prometheus_output/prometheus_cache.go | 2 +- .../prometheus_output/prometheus_output.go | 180 +----------------- 10 files changed, 389 insertions(+), 180 deletions(-) create mode 100644 pkg/app/processor.go create mode 100644 pkg/cmd/processor/processor.go diff --git a/pkg/app/get.go b/pkg/app/get.go index 8e69e58b..8429ab1d 100644 --- a/pkg/app/get.go +++ b/pkg/app/get.go @@ -134,7 +134,6 @@ func (a *App) getRequest(ctx context.Context, tc *types.TargetConfig, req *gnmi. response, err := a.ClientGet(ctx, tc, xreq) if err != nil { - a.logError(fmt.Errorf("target %q get request failed: %v", tc.Name, err)) return nil, err } return response, nil diff --git a/pkg/app/processor.go b/pkg/app/processor.go new file mode 100644 index 00000000..379f7e60 --- /dev/null +++ b/pkg/app/processor.go @@ -0,0 +1,167 @@ +package app + +import ( + "bytes" + "encoding/json" + "errors" + "fmt" + "time" + + "github.com/AlekSi/pointer" + "github.com/openconfig/gnmic/pkg/config" + "github.com/openconfig/gnmic/pkg/file" + "github.com/openconfig/gnmic/pkg/formatters" + promcom "github.com/openconfig/gnmic/pkg/outputs/prometheus_output" + dto "github.com/prometheus/client_model/go" + "github.com/prometheus/common/expfmt" + "github.com/spf13/cobra" +) + +func (a *App) ProcessorPreRunE(cmd *cobra.Command, args []string) error { + a.Config.SetLocalFlagsFromFile(cmd) + + err := a.initPluginManager() + if err != nil { + return err + } + return nil +} + +func (a *App) ProcessorRunE(cmd *cobra.Command, args []string) error { + actionsConfig, err := a.Config.GetActions() + if err != nil { + return fmt.Errorf("failed reading actions config: %v", err) + } + pConfig, err := a.Config.GetEventProcessors() + if err != nil { + return fmt.Errorf("failed reading event processors config: %v", err) + } + tcs, err := a.Config.GetTargets() + if err != nil { + if !errors.Is(err, config.ErrNoTargetsFound) { + return err + } + } + // initialize processors + evps, err := formatters.MakeEventProcessors( + a.Logger, + a.Config.LocalFlags.ProcessorName, + pConfig, + tcs, + actionsConfig, + ) + if err != nil { + return err + } + // read input file + inputBytes, err := file.ReadFile(cmd.Context(), a.Config.LocalFlags.ProcessorInput) + if err != nil { + return err + } + evInput := make([][]*formatters.EventMsg, 0) + msgs := bytes.Split(inputBytes, []byte(a.Config.LocalFlags.ProcessorInputDelimiter)) + for i, bg := range msgs { + if len(bg) == 0 { + continue + } + mevs := make([]map[string]any, 0) + err = json.Unmarshal(bg, &mevs) + if err != nil { + return fmt.Errorf("failed json Unmarshal at msg index %d: %s: %v", i, bg, err) + } + + evs := make([]*formatters.EventMsg, 0, len(mevs)) + for _, mev := range mevs { + ev, err := formatters.EventFromMap(mev) + if err != nil { + return err + } + evs = append(evs, ev) + } + evInput = append(evInput, evs) + } + rrevs := make([][]*formatters.EventMsg, 0, len(evInput)) + for _, evs := range evInput { + revs := evs + for _, p := range evps { + revs = p.Apply(revs...) + } + rrevs = append(rrevs, revs) + } + + if len(a.Config.LocalFlags.ProcessorOutput) != 0 { + b, err := a.promFormat(rrevs, a.Config.LocalFlags.ProcessorOutput) + if err != nil { + return err + } + fmt.Println(string(b)) + return nil + } + + numEvOut := len(rrevs) + for i, rev := range rrevs { + b, err := json.MarshalIndent(rev, "", " ") + if err != nil { + return err + } + fmt.Println(string(b)) + if i == numEvOut-1 { + break + } + } + return nil +} + +func (a *App) InitProcessorFlags(cmd *cobra.Command) { + cmd.ResetFlags() + + cmd.Flags().StringVarP(&a.Config.LocalFlags.ProcessorInput, "input", "", "", "processors input") + cmd.MarkFlagRequired("input") + cmd.Flags().StringVarP(&a.Config.LocalFlags.ProcessorInputDelimiter, "delimiter", "", "\n", "processors input delimiter") + cmd.Flags().StringSliceVarP(&a.Config.LocalFlags.ProcessorName, "name", "", nil, "list of processors to apply to the input") + cmd.MarkFlagRequired("name") + cmd.Flags().StringSliceVarP(&a.Config.LocalFlags.ProcessorPrometheusOpts, "prom-opts", "", nil, "list of prometheus output options") + cmd.Flags().StringVarP(&a.Config.LocalFlags.ProcessorOutput, "output", "", "", "output name") +} + +func (a *App) promFormat(rrevs [][]*formatters.EventMsg, outName string) ([]byte, error) { + // read output config + outputPath := "outputs/" + outName + outputConfig := a.Config.FileConfig.GetStringMap(outputPath) + if outputConfig == nil { + return nil, fmt.Errorf("unknown output name: %s", outName) + } + + mb := &promcom.MetricBuilder{ + Prefix: a.Config.FileConfig.GetString(outputPath + "/metric-prefix"), + AppendSubscriptionName: a.Config.FileConfig.GetBool(outputPath + "/append-subscription-name"), + StringsAsLabels: a.Config.FileConfig.GetBool(outputPath + "/strings-as-labels"), + OverrideTimestamps: a.Config.FileConfig.GetBool(outputPath + "/override-timestamps"), + ExportTimestamps: a.Config.FileConfig.GetBool(outputPath + "/export-timestamps"), + } + + b := new(bytes.Buffer) + now := time.Now() + for _, revs := range rrevs { + for _, ev := range revs { + pms := mb.MetricsFromEvent(ev, now) + for _, pm := range pms { + m := &dto.Metric{} + err := pm.Write(m) + if err != nil { + return nil, err + } + _, err = expfmt.MetricFamilyToText(b, &dto.MetricFamily{ + Name: pointer.ToString(pm.Name), + Help: pointer.ToString("gNMIc generated metric"), + Type: dto.MetricType_UNTYPED.Enum(), + Metric: []*dto.Metric{m}, + }) + if err != nil { + return nil, err + } + } + } + } + return b.Bytes(), nil +} diff --git a/pkg/cmd/processor/processor.go b/pkg/cmd/processor/processor.go new file mode 100644 index 00000000..7ed81303 --- /dev/null +++ b/pkg/cmd/processor/processor.go @@ -0,0 +1,22 @@ +package processor + +import ( + "github.com/openconfig/gnmic/pkg/app" + "github.com/spf13/cobra" +) + +// processorCmd represents the processor command +func New(gApp *app.App) *cobra.Command { + cmd := &cobra.Command{ + Use: "processor", + Short: "apply a list of processors", + PreRunE: gApp.ProcessorPreRunE, + RunE: gApp.ProcessorRunE, + PostRun: func(cmd *cobra.Command, args []string) { + gApp.CleanupPlugins() + }, + SilenceUsage: true, + } + gApp.InitProcessorFlags(cmd) + return cmd +} diff --git a/pkg/cmd/root.go b/pkg/cmd/root.go index 34d43d99..00433a29 100644 --- a/pkg/cmd/root.go +++ b/pkg/cmd/root.go @@ -26,6 +26,7 @@ import ( "github.com/openconfig/gnmic/pkg/cmd/getset" "github.com/openconfig/gnmic/pkg/cmd/listener" "github.com/openconfig/gnmic/pkg/cmd/path" + "github.com/openconfig/gnmic/pkg/cmd/processor" "github.com/openconfig/gnmic/pkg/cmd/proxy" "github.com/openconfig/gnmic/pkg/cmd/set" "github.com/openconfig/gnmic/pkg/cmd/subscribe" @@ -77,6 +78,7 @@ func newRootCmd() *cobra.Command { gApp.RootCmd.AddCommand(subscribe.New(gApp)) gApp.RootCmd.AddCommand(version.New(gApp)) gApp.RootCmd.AddCommand(proxy.New(gApp)) + gApp.RootCmd.AddCommand(processor.New(gApp)) return gApp.RootCmd } diff --git a/pkg/config/config.go b/pkg/config/config.go index a79f083a..1d05b84b 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -254,6 +254,12 @@ type LocalFlags struct { DiffSetToNotifsFull bool `mapstructure:"diff-set-to-notifs-full,omitempty" json:"diff-set-to-notifs-full,omitempty" yaml:"diff-set-to-notifs-full,omitempty"` // TunnelServerSubscribe bool + // Processor + ProcessorInput string + ProcessorInputDelimiter string + ProcessorName []string + ProcessorPrometheusOpts []string + ProcessorOutput string } func New() *Config { diff --git a/pkg/formatters/event.go b/pkg/formatters/event.go index f6df18d8..7f600cbf 100644 --- a/pkg/formatters/event.go +++ b/pkg/formatters/event.go @@ -312,9 +312,15 @@ func EventFromMap(m map[string]interface{}) (*EventMsg, error) { if v, ok := m["timestamp"]; ok { i := num64(v) if i == nil { - return nil, fmt.Errorf("could not convert map to event message, timestamp it not an int64") + return nil, fmt.Errorf("could not convert map to event message, timestamp is not an int64: %T", v) } - e.Timestamp = i.(int64) + switch i := i.(type) { + case int64: + e.Timestamp = i + case uint64: + e.Timestamp = int64(i) + } + } if v, ok := m["tags"]; ok { switch v := v.(type) { @@ -383,6 +389,8 @@ func num64(n interface{}) interface{} { return uint64(n) case uint64: return uint64(n) + case float64: + return uint64(n) } return nil } diff --git a/pkg/formatters/event_strings/event_strings.go b/pkg/formatters/event_strings/event_strings.go index c5a7284f..e17b3b9a 100644 --- a/pkg/formatters/event_strings/event_strings.go +++ b/pkg/formatters/event_strings/event_strings.go @@ -226,9 +226,10 @@ func (s *stringsp) applyTagTransformations(e *formatters.EventMsg, k, v string) k, vi = t.apply(k, v) if vs, ok := vi.(string); ok { e.Tags[k] = vs - } else { - s.logger.Printf("failed to assert %v type as string", vi) + v = vs // change the original value in case it's used in the next transform + continue } + s.logger.Printf("failed to assert %v type as string", vi) } } } diff --git a/pkg/outputs/prometheus_output/prometheus_common.go b/pkg/outputs/prometheus_output/prometheus_common.go index c08ed84f..2dae2dd1 100644 --- a/pkg/outputs/prometheus_output/prometheus_common.go +++ b/pkg/outputs/prometheus_output/prometheus_common.go @@ -10,32 +10,158 @@ package prometheus_output import ( "errors" + "fmt" + "hash/fnv" "math" "path/filepath" "regexp" + "sort" "strconv" "strings" "time" "github.com/openconfig/gnmi/proto/gnmi" + "github.com/openconfig/gnmic/pkg/formatters" + "github.com/prometheus/client_golang/prometheus" + dto "github.com/prometheus/client_model/go" "github.com/prometheus/prometheus/model/labels" "github.com/prometheus/prometheus/prompb" - - "github.com/openconfig/gnmic/pkg/formatters" ) const ( - metricNameRegex = "[^a-zA-Z0-9_]+" + metricNameRegex = "[^a-zA-Z0-9_]+" + defaultMetricHelp = "gNMIc generated metric" ) var ( MetricNameRegex = regexp.MustCompile(metricNameRegex) ) +type PromMetric struct { + Name string + Time *time.Time + // AddedAt is used to expire metrics if the time field is not initialized + // this happens when ExportTimestamp == false + AddedAt time.Time + + labels []prompb.Label + value float64 +} + +// Metric +func (p *PromMetric) CalculateKey() uint64 { + h := fnv.New64a() + h.Write([]byte(p.Name)) + if len(p.labels) > 0 { + h.Write([]byte(":")) + sort.Slice(p.labels, func(i, j int) bool { + return p.labels[i].Name < p.labels[j].Name + }) + for _, label := range p.labels { + h.Write([]byte(label.Name)) + h.Write([]byte(":")) + h.Write([]byte(label.Value)) + h.Write([]byte(":")) + } + } + return h.Sum64() +} + +func (p *PromMetric) String() string { + if p == nil { + return "" + } + sb := strings.Builder{} + sb.WriteString("name=") + sb.WriteString(p.Name) + sb.WriteString(",") + numLabels := len(p.labels) + if numLabels > 0 { + sb.WriteString("labels=[") + for i, lb := range p.labels { + sb.WriteString(lb.Name) + sb.WriteString("=") + sb.WriteString(lb.Value) + if i < numLabels-1 { + sb.WriteString(",") + } + } + sb.WriteString("],") + } + sb.WriteString(fmt.Sprintf("value=%f,", p.value)) + sb.WriteString("time=") + if p.Time != nil { + sb.WriteString(p.Time.String()) + } else { + sb.WriteString("nil") + } + sb.WriteString(",addedAt=") + sb.WriteString(p.AddedAt.String()) + return sb.String() +} + +// Desc implements prometheus.Metric +func (p *PromMetric) Desc() *prometheus.Desc { + labelNames := make([]string, 0, len(p.labels)) + for _, label := range p.labels { + labelNames = append(labelNames, label.Name) + } + + return prometheus.NewDesc(p.Name, defaultMetricHelp, labelNames, nil) +} + +// Write implements prometheus.Metric +func (p *PromMetric) Write(out *dto.Metric) error { + out.Untyped = &dto.Untyped{ + Value: &p.value, + } + out.Label = make([]*dto.LabelPair, 0, len(p.labels)) + for i := range p.labels { + out.Label = append(out.Label, &dto.LabelPair{Name: &p.labels[i].Name, Value: &p.labels[i].Value}) + } + if p.Time == nil { + return nil + } + timestamp := p.Time.UnixNano() / 1000000 + out.TimestampMs = ×tamp + return nil +} + +func (mb *MetricBuilder) MetricsFromEvent(ev *formatters.EventMsg, now time.Time) []*PromMetric { + pms := make([]*PromMetric, 0, len(ev.Values)) + labels := mb.GetLabels(ev) + for vName, val := range ev.Values { + v, err := getFloat(val) + if err != nil { + if !mb.StringsAsLabels { + continue + } + v = 1.0 + } + pm := &PromMetric{ + Name: mb.MetricName(ev.Name, vName), + labels: labels, + value: v, + AddedAt: now, + } + if mb.OverrideTimestamps && mb.ExportTimestamps { + ev.Timestamp = now.UnixNano() + } + if mb.ExportTimestamps { + tm := time.Unix(0, ev.Timestamp) + pm.Time = &tm + } + pms = append(pms, pm) + } + return pms +} + type MetricBuilder struct { Prefix string AppendSubscriptionName bool StringsAsLabels bool + OverrideTimestamps bool + ExportTimestamps bool } func (m *MetricBuilder) GetLabels(ev *formatters.EventMsg) []prompb.Label { @@ -168,3 +294,43 @@ func (m *MetricBuilder) TimeSeriesFromEvent(ev *formatters.EventMsg) []*NamedTim } return promTS } + +func getFloat(v interface{}) (float64, error) { + switch i := v.(type) { + case float64: + return float64(i), nil + case float32: + return float64(i), nil + case int64: + return float64(i), nil + case int32: + return float64(i), nil + case int16: + return float64(i), nil + case int8: + return float64(i), nil + case uint64: + return float64(i), nil + case uint32: + return float64(i), nil + case uint16: + return float64(i), nil + case uint8: + return float64(i), nil + case int: + return float64(i), nil + case uint: + return float64(i), nil + case string: + f, err := strconv.ParseFloat(i, 64) + if err != nil { + return math.NaN(), err + } + return f, err + //lint:ignore SA1019 still need DecimalVal for backward compatibility + case *gnmi.Decimal64: + return float64(i.Digits) / math.Pow10(int(i.Precision)), nil + default: + return math.NaN(), errors.New("getFloat: unknown value is of incompatible type") + } +} diff --git a/pkg/outputs/prometheus_output/prometheus_output/prometheus_cache.go b/pkg/outputs/prometheus_output/prometheus_output/prometheus_cache.go index 7ec84024..0cc73d30 100644 --- a/pkg/outputs/prometheus_output/prometheus_output/prometheus_cache.go +++ b/pkg/outputs/prometheus_output/prometheus_output/prometheus_cache.go @@ -66,7 +66,7 @@ func (p *prometheusOutput) collectFromCache(ch chan<- prometheus.Metric) { defer cancel() now := time.Now() for _, ev := range events { - for _, pm := range p.metricsFromEvent(ev, now) { + for _, pm := range p.mb.MetricsFromEvent(ev, now) { select { case <-ctx.Done(): p.logger.Printf("collection context terminated: %v", ctx.Err()) diff --git a/pkg/outputs/prometheus_output/prometheus_output/prometheus_output.go b/pkg/outputs/prometheus_output/prometheus_output/prometheus_output.go index c530b6bb..7d282129 100644 --- a/pkg/outputs/prometheus_output/prometheus_output/prometheus_output.go +++ b/pkg/outputs/prometheus_output/prometheus_output/prometheus_output.go @@ -12,15 +12,11 @@ import ( "context" "crypto/tls" "encoding/json" - "errors" "fmt" - "hash/fnv" "io" "log" - "math" "net" "net/http" - "sort" "strconv" "strings" "sync" @@ -33,8 +29,6 @@ import ( "github.com/openconfig/gnmi/proto/gnmi" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promhttp" - dto "github.com/prometheus/client_model/go" - "github.com/prometheus/prometheus/prompb" "google.golang.org/protobuf/proto" "github.com/openconfig/gnmic/pkg/api/types" @@ -59,16 +53,6 @@ const ( defaultNumWorkers = 1 ) -type promMetric struct { - name string - labels []prompb.Label - time *time.Time - value float64 - // addedAt is used to expire metrics if the time field is not initialized - // this happens when ExportTimestamp == false - addedAt time.Time -} - func init() { outputs.Register(outputType, func() outputs.Output { return &prometheusOutput{ @@ -76,7 +60,7 @@ func init() { eventChan: make(chan *formatters.EventMsg), msgChan: make(chan *outputs.ProtoMsg), wg: new(sync.WaitGroup), - entries: make(map[uint64]*promMetric), + entries: make(map[uint64]*promcom.PromMetric), logger: log.New(io.Discard, loggingPrefix, utils.DefaultLoggingFlags), } }) @@ -91,7 +75,7 @@ type prometheusOutput struct { wg *sync.WaitGroup server *http.Server sync.Mutex - entries map[uint64]*promMetric + entries map[uint64]*promcom.PromMetric mb *promcom.MetricBuilder evps []formatters.EventProcessor @@ -196,6 +180,8 @@ func (p *prometheusOutput) Init(ctx context.Context, name string, cfg map[string Prefix: p.cfg.MetricPrefix, AppendSubscriptionName: p.cfg.AppendSubscriptionName, StringsAsLabels: p.cfg.StringsAsLabels, + OverrideTimestamps: p.cfg.OverrideTimestamps, + ExportTimestamps: p.cfg.ExportTimestamps, } if p.cfg.CacheConfig != nil { @@ -424,13 +410,13 @@ func (p *prometheusOutput) workerHandleEvent(ev *formatters.EventMsg) { } p.Lock() defer p.Unlock() - for _, pm := range p.metricsFromEvent(ev, time.Now()) { - key := pm.calculateKey() + for _, pm := range p.mb.MetricsFromEvent(ev, time.Now()) { + key := pm.CalculateKey() e, ok := p.entries[key] // if the entry key is not present add it to the map. // if present add it only if the entry timestamp is newer than the // existing one. - if !ok || pm.time == nil || (ok && pm.time != nil && e.time.Before(*pm.time)) { + if !ok || pm.Time == nil || (ok && pm.Time != nil && e.Time.Before(*pm.Time)) { p.entries[key] = pm if p.cfg.Debug { p.logger.Printf("saved key=%d, metric: %+v", key, pm) @@ -446,12 +432,12 @@ func (p *prometheusOutput) expireMetrics() { expiry := time.Now().Add(-p.cfg.Expiration) for k, e := range p.entries { if p.cfg.ExportTimestamps { - if e.time.Before(expiry) { + if e.Time.Before(expiry) { delete(p.entries, k) } continue } - if e.addedAt.Before(expiry) { + if e.AddedAt.Before(expiry) { delete(p.entries, k) } } @@ -550,125 +536,6 @@ func (p *prometheusOutput) setDefaults() error { return nil } -// Metric -func (p *promMetric) calculateKey() uint64 { - h := fnv.New64a() - h.Write([]byte(p.name)) - if len(p.labels) > 0 { - h.Write([]byte(":")) - sort.Slice(p.labels, func(i, j int) bool { - return p.labels[i].Name < p.labels[j].Name - }) - for _, label := range p.labels { - h.Write([]byte(label.Name)) - h.Write([]byte(":")) - h.Write([]byte(label.Value)) - h.Write([]byte(":")) - } - } - return h.Sum64() -} - -func (p *promMetric) String() string { - if p == nil { - return "" - } - sb := strings.Builder{} - sb.WriteString("name=") - sb.WriteString(p.name) - sb.WriteString(",") - numLabels := len(p.labels) - if numLabels > 0 { - sb.WriteString("labels=[") - for i, lb := range p.labels { - sb.WriteString(lb.Name) - sb.WriteString("=") - sb.WriteString(lb.Value) - if i < numLabels-1 { - sb.WriteString(",") - } - } - sb.WriteString("],") - } - sb.WriteString(fmt.Sprintf("value=%f,", p.value)) - sb.WriteString("time=") - if p.time != nil { - sb.WriteString(p.time.String()) - } else { - sb.WriteString("nil") - } - sb.WriteString(",addedAt=") - sb.WriteString(p.addedAt.String()) - return sb.String() -} - -// Desc implements prometheus.Metric -func (p *promMetric) Desc() *prometheus.Desc { - labelNames := make([]string, 0, len(p.labels)) - for _, label := range p.labels { - labelNames = append(labelNames, label.Name) - } - - return prometheus.NewDesc(p.name, defaultMetricHelp, labelNames, nil) -} - -// Write implements prometheus.Metric -func (p *promMetric) Write(out *dto.Metric) error { - out.Untyped = &dto.Untyped{ - Value: &p.value, - } - out.Label = make([]*dto.LabelPair, 0, len(p.labels)) - for i := range p.labels { - out.Label = append(out.Label, &dto.LabelPair{Name: &p.labels[i].Name, Value: &p.labels[i].Value}) - } - if p.time == nil { - return nil - } - timestamp := p.time.UnixNano() / 1000000 - out.TimestampMs = ×tamp - return nil -} - -func getFloat(v interface{}) (float64, error) { - switch i := v.(type) { - case float64: - return float64(i), nil - case float32: - return float64(i), nil - case int64: - return float64(i), nil - case int32: - return float64(i), nil - case int16: - return float64(i), nil - case int8: - return float64(i), nil - case uint64: - return float64(i), nil - case uint32: - return float64(i), nil - case uint16: - return float64(i), nil - case uint8: - return float64(i), nil - case int: - return float64(i), nil - case uint: - return float64(i), nil - case string: - f, err := strconv.ParseFloat(i, 64) - if err != nil { - return math.NaN(), err - } - return f, err - //lint:ignore SA1019 still need DecimalVal for backward compatibility - case *gnmi.Decimal64: - return float64(i.Digits) / math.Pow10(int(i.Precision)), nil - default: - return math.NaN(), errors.New("getFloat: unknown value is of incompatible type") - } -} - func (p *prometheusOutput) SetName(name string) { if p.cfg.Name == "" { p.cfg.Name = name @@ -693,32 +560,3 @@ func (p *prometheusOutput) SetClusterName(name string) { } func (p *prometheusOutput) SetTargetsConfig(map[string]*types.TargetConfig) {} - -func (p *prometheusOutput) metricsFromEvent(ev *formatters.EventMsg, now time.Time) []*promMetric { - pms := make([]*promMetric, 0, len(ev.Values)) - labels := p.mb.GetLabels(ev) - for vName, val := range ev.Values { - v, err := getFloat(val) - if err != nil { - if !p.cfg.StringsAsLabels { - continue - } - v = 1.0 - } - pm := &promMetric{ - name: p.mb.MetricName(ev.Name, vName), - labels: labels, - value: v, - addedAt: now, - } - if p.cfg.OverrideTimestamps && p.cfg.ExportTimestamps { - ev.Timestamp = now.UnixNano() - } - if p.cfg.ExportTimestamps { - tm := time.Unix(0, ev.Timestamp) - pm.time = &tm - } - pms = append(pms, pm) - } - return pms -} From f89f9260375e9dd8453c71cadf53c9fe27014296 Mon Sep 17 00:00:00 2001 From: Adam Libresco Date: Sat, 30 Mar 2024 05:23:02 -0400 Subject: [PATCH 2/6] Sort prometheus labels in accordance with the prometheus spec --- .../prometheus_output/prometheus_common.go | 9 ++++++ .../prometheus_common_test.go | 29 +++++++++++++++++++ 2 files changed, 38 insertions(+) diff --git a/pkg/outputs/prometheus_output/prometheus_common.go b/pkg/outputs/prometheus_output/prometheus_common.go index 2dae2dd1..ab92d86b 100644 --- a/pkg/outputs/prometheus_output/prometheus_common.go +++ b/pkg/outputs/prometheus_output/prometheus_common.go @@ -9,6 +9,7 @@ package prometheus_output import ( + "cmp" "errors" "fmt" "hash/fnv" @@ -16,6 +17,7 @@ import ( "path/filepath" "regexp" "sort" + "slices" "strconv" "strings" "time" @@ -278,6 +280,13 @@ func (m *MetricBuilder) TimeSeriesFromEvent(ev *formatters.EventMsg) []*NamedTim Name: labels.MetricName, Value: tsName, }) + + // The prometheus spec requires label names to be sorted + // https://prometheus.io/docs/concepts/remote_write_spec/ + slices.SortFunc(tsLabelsWithName, func(a prompb.Label, b prompb.Label) int { + return cmp.Compare(a.Name, b.Name) + }) + nts := &NamedTimeSeries{ Name: tsName, TS: &prompb.TimeSeries{ diff --git a/pkg/outputs/prometheus_output/prometheus_common_test.go b/pkg/outputs/prometheus_output/prometheus_common_test.go index 63cde927..d3cd8ac0 100644 --- a/pkg/outputs/prometheus_output/prometheus_common_test.go +++ b/pkg/outputs/prometheus_output/prometheus_common_test.go @@ -9,10 +9,13 @@ package prometheus_output import ( + "cmp" + "slices" "testing" "github.com/openconfig/gnmic/pkg/formatters" "github.com/prometheus/prometheus/model/labels" + "github.com/prometheus/prometheus/prompb" ) var metricNameSet = map[string]struct { @@ -97,6 +100,32 @@ func TestTimeSeriesFromEvent(t *testing.T) { } } +func TestTimeSeriesLabelsSorted(t *testing.T) { + metricBuilder := &MetricBuilder{StringsAsLabels: true} + event := &formatters.EventMsg{ + Name: "eventName", + Timestamp: 12345, + Tags: map[string]string{ + "tagName": "tagVal", + }, + Values: map[string]interface{}{ + "strName1": "strVal1", + "strName2": "strVal2", + "intName1": 1, + "intName2": 2, + }, + Deletes: []string{}, + } + for _, nts := range metricBuilder.TimeSeriesFromEvent(event) { + areLabelsSorted := slices.IsSortedFunc(nts.TS.Labels, func(a prompb.Label, b prompb.Label) int { + return cmp.Compare(a.Name, b.Name) + }) + if !areLabelsSorted { + t.Errorf("labels names are not sorted, got '%v'", nts.TS.Labels) + } + } +} + func TestMetricName(t *testing.T) { for name, tc := range metricNameSet { t.Run(name, func(t *testing.T) { From d3b888a4549fc760d35d53a0252e2ff76ed8fbd1 Mon Sep 17 00:00:00 2001 From: Adam Libresco Date: Thu, 18 Apr 2024 15:08:49 -0400 Subject: [PATCH 3/6] Add a config option for kafka producer flush frequency --- pkg/outputs/kafka_output/kafka_output.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pkg/outputs/kafka_output/kafka_output.go b/pkg/outputs/kafka_output/kafka_output.go index 040ea5da..d105daba 100644 --- a/pkg/outputs/kafka_output/kafka_output.go +++ b/pkg/outputs/kafka_output/kafka_output.go @@ -85,6 +85,7 @@ type config struct { MaxRetry int `mapstructure:"max-retry,omitempty"` Timeout time.Duration `mapstructure:"timeout,omitempty"` RecoveryWaitTime time.Duration `mapstructure:"recovery-wait-time,omitempty"` + FlushFrequency time.Duration `mapstructure:"flush-frequency,omitempty"` SyncProducer bool `mapstructure:"sync-producer,omitempty"` RequiredAcks string `mapstructure:"required-acks,omitempty"` Format string `mapstructure:"format,omitempty"` @@ -546,6 +547,7 @@ func (k *kafkaOutput) createConfig() (*sarama.Config, error) { cfg.Producer.Retry.Max = k.cfg.MaxRetry cfg.Producer.Return.Successes = true cfg.Producer.Timeout = k.cfg.Timeout + cfg.Producer.Flush.Frequency = k.cfg.FlushFrequency switch k.cfg.RequiredAcks { case requiredAcksNoResponse: case requiredAcksWaitForLocal: From 3965db22c9c071c9c09e92fb80bee385c0cf3904 Mon Sep 17 00:00:00 2001 From: Karim Radhouani Date: Mon, 25 Mar 2024 13:05:13 -0700 Subject: [PATCH 4/6] add processor command --- pkg/outputs/prometheus_output/prometheus_common.go | 1 + 1 file changed, 1 insertion(+) diff --git a/pkg/outputs/prometheus_output/prometheus_common.go b/pkg/outputs/prometheus_output/prometheus_common.go index ab92d86b..15d68b7e 100644 --- a/pkg/outputs/prometheus_output/prometheus_common.go +++ b/pkg/outputs/prometheus_output/prometheus_common.go @@ -18,6 +18,7 @@ import ( "regexp" "sort" "slices" + "sort" "strconv" "strings" "time" From 1c73da63be64e9aaa3fa3ede79a65af2300c0cc5 Mon Sep 17 00:00:00 2001 From: Karim Radhouani Date: Mon, 22 Apr 2024 16:44:07 -0700 Subject: [PATCH 5/6] cleanup and add docs --- docs/cmd/processor.md | 364 ++++++++++++++++++ pkg/app/processor.go | 6 +- pkg/cmd/processor/processor.go | 1 + pkg/config/config.go | 37 +- .../prometheus_output/prometheus_common.go | 2 +- 5 files changed, 388 insertions(+), 22 deletions(-) create mode 100644 docs/cmd/processor.md diff --git a/docs/cmd/processor.md b/docs/cmd/processor.md new file mode 100644 index 00000000..5c50df94 --- /dev/null +++ b/docs/cmd/processor.md @@ -0,0 +1,364 @@ +### Description + +The `[processor | proc]` command allows running a set of event processor offline given an input of event messages. + +If expects a file input (`--input`) containing a list of event messages and one or more processor(s) name(s) (`--name`) defined in the main config file. +This command will read the input file, validate the configured processors, apply them on the input event messages and output the result. + +### Usage + +`gnmic [global-flags] processor [local-flags]` + +### Local Flags + +The processor command supports the following local flags: + +#### name + +The `[--name]` flag sets the list of processors names to apply to the input. + +#### input + +The `[--input]` flag is used to specify the path to a file containing a list of event messages (`stdin` can be specified by giving the `-` value). + +#### delimiter + +The `[--delimiter]` flag is used to set the delimiter string between event messages in the input file, defaults to `\n`. + +#### output + +The `[--output]` flag references an output name configured in the main config file. The command will out format the resulting messages according to the output config. This is mainly for outputs with `type: prometheus` + +### Example + +Config File + +```yaml +outputs: + out1: + type: prometheus + metric-prefix: "gnmic" + strings-as-labels: true + +processors: + proc0: + event-strings: + value-names: + - "^_" + transforms: + + # processor name + proc1: + # processor type + event-strings: + value-names: + - ".*" + transforms: + # strings function name + - path-base: + apply-on: "name" + proc2: + event-strings: + tag-names: + - "interface_name" + - "subscription-name" + - "source" + transforms: + # strings function name + - to-upper: + apply-on: "value" + - to-upper: + apply-on: "name" + proc3: + # processor type + event-drop: + condition: ".values | length == 0" +``` + +input File: + +```json +[ + { + "name": "sub1", + "timestamp": 1710890476202665500, + "tags": { + "interface_name": "mgmt0", + "source": "clab-traps-srl1", + "subscription-name": "sub1" + }, + "values": { + "/interface/statistics/in-packets": 351770 + } + }, + { + "name": "sub1", + "timestamp": 1710890476202665500, + "tags": { + "interface_name": "mgmt0", + "source": "clab-traps-srl1", + "subscription-name": "sub1" + }, + "values": { + "/interface/statistics/in-octets": 35284165 + } + }, + { + "name": "sub1", + "timestamp": 1710890476202665500, + "tags": { + "interface_name": "mgmt0", + "source": "clab-traps-srl1", + "subscription-name": "sub1" + }, + "values": { + "/interface/statistics/in-unicast-packets": 338985 + } + }, + { + "name": "sub1", + "timestamp": 1710890476202665500, + "tags": { + "interface_name": "mgmt0", + "source": "clab-traps-srl1", + "subscription-name": "sub1" + }, + "values": { + "/interface/statistics/in-broadcast-packets": 1218 + } + }, + { + "name": "sub1", + "timestamp": 1710890476202665500, + "tags": { + "interface_name": "mgmt0", + "source": "clab-traps-srl1", + "subscription-name": "sub1" + }, + "values": { + "/interface/statistics/in-multicast-packets": 5062 + } + }, + { + "name": "sub1", + "timestamp": 1710890476202665500, + "tags": { + "interface_name": "mgmt0", + "source": "clab-traps-srl1", + "subscription-name": "sub1" + }, + "values": { + "/interface/statistics/in-discarded-packets": 6377 + } + }, + { + "name": "sub1", + "timestamp": 1710890476202665500, + "tags": { + "interface_name": "mgmt0", + "source": "clab-traps-srl1", + "subscription-name": "sub1" + }, + "values": { + "/interface/statistics/in-error-packets": 128 + } + }, + { + "name": "sub1", + "timestamp": 1710890476202665500, + "tags": { + "interface_name": "mgmt0", + "source": "clab-traps-srl1", + "subscription-name": "sub1" + }, + "values": { + "/interface/statistics/in-fcs-error-packets": 0 + } + }, + { + "name": "sub1", + "timestamp": 1710890476202665500, + "tags": { + "interface_name": "mgmt0", + "source": "clab-traps-srl1", + "subscription-name": "sub1" + }, + "values": { + "/interface/statistics/out-packets": 568218 + } + }, + { + "name": "sub1", + "timestamp": 1710890476202665500, + "tags": { + "interface_name": "mgmt0", + "source": "clab-traps-srl1", + "subscription-name": "sub1" + }, + "values": { + "/interface/statistics/out-octets": 219527024 + } + }, + { + "name": "sub1", + "timestamp": 1710890476202665500, + "tags": { + "interface_name": "mgmt0", + "source": "clab-traps-srl1", + "subscription-name": "sub1" + }, + "values": { + "/interface/statistics/out-mirror-octets": 0 + } + }, + { + "name": "sub1", + "timestamp": 1710890476202665500, + "tags": { + "interface_name": "mgmt0", + "source": "clab-traps-srl1", + "subscription-name": "sub1" + }, + "values": { + "/interface/statistics/out-unicast-packets": 567532 + } + }, + { + "name": "sub1", + "timestamp": 1710890476202665500, + "tags": { + "interface_name": "mgmt0", + "source": "clab-traps-srl1", + "subscription-name": "sub1" + }, + "values": { + "/interface/statistics/out-broadcast-packets": 6 + } + }, + { + "name": "sub1", + "timestamp": 1710890476202665500, + "tags": { + "interface_name": "mgmt0", + "source": "clab-traps-srl1", + "subscription-name": "sub1" + }, + "values": { + "/interface/statistics/out-multicast-packets": 680 + } + }, + { + "name": "sub1", + "timestamp": 1710890476202665500, + "tags": { + "interface_name": "mgmt0", + "source": "clab-traps-srl1", + "subscription-name": "sub1" + }, + "values": { + "/interface/statistics/out-discarded-packets": 0 + } + }, + { + "name": "sub1", + "timestamp": 1710890476202665500, + "tags": { + "interface_name": "mgmt0", + "source": "clab-traps-srl1", + "subscription-name": "sub1" + }, + "values": { + "/interface/statistics/out-error-packets": 0 + } + }, + { + "name": "sub1", + "timestamp": 1710890476202665500, + "tags": { + "interface_name": "mgmt0", + "source": "clab-traps-srl1", + "subscription-name": "sub1" + }, + "values": { + "/interface/statistics/out-mirror-packets": 0 + } + }, + { + "name": "sub1", + "timestamp": 1710890476202665500, + "tags": { + "interface_name": "mgmt0", + "source": "clab-traps-srl1", + "subscription-name": "sub1" + }, + "values": { + "/interface/statistics/carrier-transitions": 1 + } + } +] +``` + +Command: + +```shell +gnmic processor --input /path/to/event_msg.txt --name proc1,proc2,proc3 --output out1 +``` + +Output: + +```text +# HELP gnmic_in_packets gNMIc generated metric +# TYPE gnmic_in_packets untyped +gnmic_in_packets{interface_name="mgmt0",source="clab-traps-srl1",subscription_name="sub1"} 351770 +# HELP gnmic_in_octets gNMIc generated metric +# TYPE gnmic_in_octets untyped +gnmic_in_octets{subscription_name="sub1",interface_name="mgmt0",source="clab-traps-srl1"} 3.5284165e+07 +# HELP gnmic_in_unicast_packets gNMIc generated metric +# TYPE gnmic_in_unicast_packets untyped +gnmic_in_unicast_packets{subscription_name="sub1",interface_name="mgmt0",source="clab-traps-srl1"} 338985 +# HELP gnmic_in_broadcast_packets gNMIc generated metric +# TYPE gnmic_in_broadcast_packets untyped +gnmic_in_broadcast_packets{interface_name="mgmt0",source="clab-traps-srl1",subscription_name="sub1"} 1218 +# HELP gnmic_in_multicast_packets gNMIc generated metric +# TYPE gnmic_in_multicast_packets untyped +gnmic_in_multicast_packets{interface_name="mgmt0",source="clab-traps-srl1",subscription_name="sub1"} 5062 +# HELP gnmic_in_discarded_packets gNMIc generated metric +# TYPE gnmic_in_discarded_packets untyped +gnmic_in_discarded_packets{interface_name="mgmt0",source="clab-traps-srl1",subscription_name="sub1"} 6377 +# HELP gnmic_in_error_packets gNMIc generated metric +# TYPE gnmic_in_error_packets untyped +gnmic_in_error_packets{interface_name="mgmt0",source="clab-traps-srl1",subscription_name="sub1"} 128 +# HELP gnmic_in_fcs_error_packets gNMIc generated metric +# TYPE gnmic_in_fcs_error_packets untyped +gnmic_in_fcs_error_packets{interface_name="mgmt0",source="clab-traps-srl1",subscription_name="sub1"} 0 +# HELP gnmic_out_packets gNMIc generated metric +# TYPE gnmic_out_packets untyped +gnmic_out_packets{interface_name="mgmt0",source="clab-traps-srl1",subscription_name="sub1"} 568218 +# HELP gnmic_out_octets gNMIc generated metric +# TYPE gnmic_out_octets untyped +gnmic_out_octets{interface_name="mgmt0",source="clab-traps-srl1",subscription_name="sub1"} 2.19527024e+08 +# HELP gnmic_out_mirror_octets gNMIc generated metric +# TYPE gnmic_out_mirror_octets untyped +gnmic_out_mirror_octets{interface_name="mgmt0",source="clab-traps-srl1",subscription_name="sub1"} 0 +# HELP gnmic_out_unicast_packets gNMIc generated metric +# TYPE gnmic_out_unicast_packets untyped +gnmic_out_unicast_packets{subscription_name="sub1",interface_name="mgmt0",source="clab-traps-srl1"} 567532 +# HELP gnmic_out_broadcast_packets gNMIc generated metric +# TYPE gnmic_out_broadcast_packets untyped +gnmic_out_broadcast_packets{interface_name="mgmt0",source="clab-traps-srl1",subscription_name="sub1"} 6 +# HELP gnmic_out_multicast_packets gNMIc generated metric +# TYPE gnmic_out_multicast_packets untyped +gnmic_out_multicast_packets{source="clab-traps-srl1",subscription_name="sub1",interface_name="mgmt0"} 680 +# HELP gnmic_out_discarded_packets gNMIc generated metric +# TYPE gnmic_out_discarded_packets untyped +gnmic_out_discarded_packets{interface_name="mgmt0",source="clab-traps-srl1",subscription_name="sub1"} 0 +# HELP gnmic_out_error_packets gNMIc generated metric +# TYPE gnmic_out_error_packets untyped +gnmic_out_error_packets{interface_name="mgmt0",source="clab-traps-srl1",subscription_name="sub1"} 0 +# HELP gnmic_out_mirror_packets gNMIc generated metric +# TYPE gnmic_out_mirror_packets untyped +gnmic_out_mirror_packets{interface_name="mgmt0",source="clab-traps-srl1",subscription_name="sub1"} 0 +# HELP gnmic_carrier_transitions gNMIc generated metric +# TYPE gnmic_carrier_transitions untyped +gnmic_carrier_transitions{subscription_name="sub1",interface_name="mgmt0",source="clab-traps-srl1"} 1 +``` \ No newline at end of file diff --git a/pkg/app/processor.go b/pkg/app/processor.go index 379f7e60..8b072f99 100644 --- a/pkg/app/processor.go +++ b/pkg/app/processor.go @@ -120,7 +120,6 @@ func (a *App) InitProcessorFlags(cmd *cobra.Command) { cmd.Flags().StringVarP(&a.Config.LocalFlags.ProcessorInputDelimiter, "delimiter", "", "\n", "processors input delimiter") cmd.Flags().StringSliceVarP(&a.Config.LocalFlags.ProcessorName, "name", "", nil, "list of processors to apply to the input") cmd.MarkFlagRequired("name") - cmd.Flags().StringSliceVarP(&a.Config.LocalFlags.ProcessorPrometheusOpts, "prom-opts", "", nil, "list of prometheus output options") cmd.Flags().StringVarP(&a.Config.LocalFlags.ProcessorOutput, "output", "", "", "output name") } @@ -131,7 +130,10 @@ func (a *App) promFormat(rrevs [][]*formatters.EventMsg, outName string) ([]byte if outputConfig == nil { return nil, fmt.Errorf("unknown output name: %s", outName) } - + outType := a.Config.FileConfig.GetString(outputPath + "/type") + if outType != "prometheus" && outType != "remote_write" { + return nil, fmt.Errorf("output %q must be of type 'prometheus' or 'remote_write'", outName) + } mb := &promcom.MetricBuilder{ Prefix: a.Config.FileConfig.GetString(outputPath + "/metric-prefix"), AppendSubscriptionName: a.Config.FileConfig.GetBool(outputPath + "/append-subscription-name"), diff --git a/pkg/cmd/processor/processor.go b/pkg/cmd/processor/processor.go index 7ed81303..bb6181df 100644 --- a/pkg/cmd/processor/processor.go +++ b/pkg/cmd/processor/processor.go @@ -9,6 +9,7 @@ import ( func New(gApp *app.App) *cobra.Command { cmd := &cobra.Command{ Use: "processor", + Aliases: []string{"proc"}, Short: "apply a list of processors", PreRunE: gApp.ProcessorPreRunE, RunE: gApp.ProcessorRunE, diff --git a/pkg/config/config.go b/pkg/config/config.go index 1d05b84b..d4b2fd8a 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -168,7 +168,7 @@ type LocalFlags struct { SubscribeStreamMode string `mapstructure:"subscribe-stream_mode,omitempty" json:"subscribe-stream-mode,omitempty" yaml:"subscribe-stream-mode,omitempty"` SubscribeSampleInterval time.Duration `mapstructure:"subscribe-sample-interval,omitempty" json:"subscribe-sample-interval,omitempty" yaml:"subscribe-sample-interval,omitempty"` SubscribeSuppressRedundant bool `mapstructure:"subscribe-suppress-redundant,omitempty" json:"subscribe-suppress-redundant,omitempty" yaml:"subscribe-suppress-redundant,omitempty"` - SubscribeHeartbeatInterval time.Duration `mapstructure:"subscribe-heartbear-interval,omitempty" json:"subscribe-heartbear-interval,omitempty" yaml:"subscribe-heartbear-interval,omitempty"` + SubscribeHeartbeatInterval time.Duration `mapstructure:"subscribe-heartbeat-interval,omitempty" json:"subscribe-heartbeat-interval,omitempty" yaml:"subscribe-heartbeat-interval,omitempty"` SubscribeModel []string `mapstructure:"subscribe-model,omitempty" json:"subscribe-model,omitempty" yaml:"subscribe-model,omitempty"` SubscribeQuiet bool `mapstructure:"subscribe-quiet,omitempty" json:"subscribe-quiet,omitempty" yaml:"subscribe-quiet,omitempty"` SubscribeTarget string `mapstructure:"subscribe-target,omitempty" json:"subscribe-target,omitempty" yaml:"subscribe-target,omitempty"` @@ -207,23 +207,23 @@ type LocalFlags struct { // VersionUpgrade UpgradeUsePkg bool `mapstructure:"upgrade-use-pkg" json:"upgrade-use-pkg,omitempty" yaml:"upgrade-use-pkg,omitempty"` // GetSet - GetSetPrefix string `mapstructure:"getset-prefix,omitempty" json:"getset-prefix,omitempty" yaml:"getset-prefix,omitempty"` - GetSetGet string `mapstructure:"getset-get,omitempty" json:"getset-get,omitempty" yaml:"getset-get,omitempty"` - GetSetModel []string - GetSetTarget string `mapstructure:"getset-target,omitempty" json:"getset-target,omitempty" yaml:"getset-target,omitempty"` - GetSetType string `mapstructure:"getset-type,omitempty" json:"getset-type,omitempty" yaml:"getset-type,omitempty"` - GetSetCondition string `mapstructure:"getset-condition,omitempty" json:"getset-condition,omitempty" yaml:"getset-condition,omitempty"` - GetSetUpdate string `mapstructure:"getset-update,omitempty" json:"getset-update,omitempty" yaml:"getset-update,omitempty"` - GetSetReplace string `mapstructure:"getset-replace,omitempty" json:"getset-replace,omitempty" yaml:"getset-replace,omitempty"` - GetSetDelete string `mapstructure:"getset-delete,omitempty" json:"getset-delete,omitempty" yaml:"getset-delete,omitempty"` - GetSetValue string `mapstructure:"getset-value,omitempty" json:"getset-value,omitempty" yaml:"getset-value,omitempty"` + GetSetPrefix string `mapstructure:"getset-prefix,omitempty" json:"getset-prefix,omitempty" yaml:"getset-prefix,omitempty"` + GetSetGet string `mapstructure:"getset-get,omitempty" json:"getset-get,omitempty" yaml:"getset-get,omitempty"` + GetSetModel []string `mapstructure:"get-set-model,omitempty" yaml:"get-set-model,omitempty" json:"get-set-model,omitempty"` + GetSetTarget string `mapstructure:"getset-target,omitempty" json:"getset-target,omitempty" yaml:"getset-target,omitempty"` + GetSetType string `mapstructure:"getset-type,omitempty" json:"getset-type,omitempty" yaml:"getset-type,omitempty"` + GetSetCondition string `mapstructure:"getset-condition,omitempty" json:"getset-condition,omitempty" yaml:"getset-condition,omitempty"` + GetSetUpdate string `mapstructure:"getset-update,omitempty" json:"getset-update,omitempty" yaml:"getset-update,omitempty"` + GetSetReplace string `mapstructure:"getset-replace,omitempty" json:"getset-replace,omitempty" yaml:"getset-replace,omitempty"` + GetSetDelete string `mapstructure:"getset-delete,omitempty" json:"getset-delete,omitempty" yaml:"getset-delete,omitempty"` + GetSetValue string `mapstructure:"getset-value,omitempty" json:"getset-value,omitempty" yaml:"getset-value,omitempty"` // Generate GenerateOutput string `mapstructure:"generate-output,omitempty" json:"generate-output,omitempty" yaml:"generate-output,omitempty"` GenerateJSON bool `mapstructure:"generate-json,omitempty" json:"generate-json,omitempty" yaml:"generate-json,omitempty"` GenerateConfigOnly bool `mapstructure:"generate-config-only,omitempty" json:"generate-config-only,omitempty" yaml:"generate-config-only,omitempty"` GeneratePath string `mapstructure:"generate-path,omitempty" json:"generate-path,omitempty" yaml:"generate-path,omitempty"` - GenerateCamelCase bool `mapstructure:"generate-camel-case,omitempty" json:"generate-camel-case" yaml:"generate-camel-case,omitempty"` - GenerateSnakeCase bool `mapstructure:"generate-snake-case,omitempty" json:"generate-snake-case" yaml:"generate-snake-case,omitempty"` + GenerateCamelCase bool `mapstructure:"generate-camel-case,omitempty" json:"generate-camel-case,omitempty" yaml:"generate-camel-case,omitempty"` + GenerateSnakeCase bool `mapstructure:"generate-snake-case,omitempty" json:"generate-snake-case,omitempty" yaml:"generate-snake-case,omitempty"` // Generate Set Request GenerateSetRequestUpdatePath []string `mapstructure:"generate-update-path,omitempty" json:"generate-update-path,omitempty" yaml:"generate-update-path,omitempty"` GenerateSetRequestReplacePath []string `mapstructure:"generate-replace-path,omitempty" json:"generate-replace-path,omitempty" yaml:"generate-replace-path,omitempty"` @@ -253,13 +253,12 @@ type LocalFlags struct { DiffSetToNotifsResponse string `mapstructure:"diff-set-to-notifs-response,omitempty" json:"diff-set-to-notifs-response,omitempty" yaml:"diff-set-to-notifs-response,omitempty"` DiffSetToNotifsFull bool `mapstructure:"diff-set-to-notifs-full,omitempty" json:"diff-set-to-notifs-full,omitempty" yaml:"diff-set-to-notifs-full,omitempty"` // - TunnelServerSubscribe bool + TunnelServerSubscribe bool `mapstructure:"tunnel-server-subscribe,omitempty" yaml:"tunnel-server-subscribe,omitempty" json:"tunnel-server-subscribe,omitempty"` // Processor - ProcessorInput string - ProcessorInputDelimiter string - ProcessorName []string - ProcessorPrometheusOpts []string - ProcessorOutput string + ProcessorInput string `mapstructure:"processor-input,omitempty" yaml:"processor-input,omitempty" json:"processor-input,omitempty"` + ProcessorInputDelimiter string `mapstructure:"processor-input-delimiter,omitempty" yaml:"processor-input-delimiter,omitempty" json:"processor-input-delimiter,omitempty"` + ProcessorName []string `mapstructure:"processor-name,omitempty" yaml:"processor-name,omitempty" json:"processor-name,omitempty"` + ProcessorOutput string `mapstructure:"processor-output,omitempty" yaml:"processor-output,omitempty" json:"processor-output,omitempty"` } func New() *Config { diff --git a/pkg/outputs/prometheus_output/prometheus_common.go b/pkg/outputs/prometheus_output/prometheus_common.go index 97f132ce..82ed59f8 100644 --- a/pkg/outputs/prometheus_output/prometheus_common.go +++ b/pkg/outputs/prometheus_output/prometheus_common.go @@ -16,7 +16,7 @@ import ( "math" "path/filepath" "regexp" - "slices" + "slices" "sort" "strconv" "strings" From 571e9f2e09868274f114d0edf09e40397d4a0b33 Mon Sep 17 00:00:00 2001 From: Karim Radhouani Date: Mon, 22 Apr 2024 18:21:45 -0700 Subject: [PATCH 6/6] fix typos --- docs/cmd/processor.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/cmd/processor.md b/docs/cmd/processor.md index 5c50df94..9b2ecd80 100644 --- a/docs/cmd/processor.md +++ b/docs/cmd/processor.md @@ -3,7 +3,7 @@ The `[processor | proc]` command allows running a set of event processor offline given an input of event messages. If expects a file input (`--input`) containing a list of event messages and one or more processor(s) name(s) (`--name`) defined in the main config file. -This command will read the input file, validate the configured processors, apply them on the input event messages and output the result. +This command will read the input file, validate the configured processors, apply them on the input event messages and print out the result. ### Usage @@ -301,7 +301,7 @@ input File: Command: ```shell -gnmic processor --input /path/to/event_msg.txt --name proc1,proc2,proc3 --output out1 +gnmic processor --input /path/to/event_msg.txt --delimiter "\n###" --name proc1,proc2,proc3 --output out1 ``` Output: @@ -361,4 +361,4 @@ gnmic_out_mirror_packets{interface_name="mgmt0",source="clab-traps-srl1",subscri # HELP gnmic_carrier_transitions gNMIc generated metric # TYPE gnmic_carrier_transitions untyped gnmic_carrier_transitions{subscription_name="sub1",interface_name="mgmt0",source="clab-traps-srl1"} 1 -``` \ No newline at end of file +```