Write latency statistics go under a histogram key (#42514)

* Write latency goes under histogram * Update fields.yml to include new exposed beats stats * Add't updates for integ tests --------- Co-authored-by: Julien Lind <[email protected]>
elastic · Jan 31, 2025 · 0ff52eb · 0ff52eb
1 parent 174c08d
commit 0ff52eb
Show file tree

Hide file tree

Showing 4 changed files with 216 additions and 9 deletions.
diff --git a/metricbeat/docs/fields.asciidoc b/metricbeat/docs/fields.asciidoc
@@ -10184,16 +10184,113 @@ type: long
 
 --
 
+
 *`beat.stats.libbeat.pipeline.queue.acked`*::
 +
 --
+Number of acknowledged events
+
+
+type: long
+
+--
+
+*`beat.stats.libbeat.pipeline.queue.added.bytes`*::
++
+--
+Number of bytes added to the queue
+
+
+type: long
+
+--
+
+*`beat.stats.libbeat.pipeline.queue.added.events`*::
++
+--
+Number of events added to the queue
+
+
+type: long
+
+--
+
+*`beat.stats.libbeat.pipeline.queue.consumed.bytes`*::
++
+--
+Number of bytes consumed from the queue
+
+
 type: long
 
 --
 
+*`beat.stats.libbeat.pipeline.queue.consumed.events`*::
++
+--
+Number of events consumed from the queue
+
+
+type: long
+
+--
+
+*`beat.stats.libbeat.pipeline.queue.filled.bytes`*::
++
+--
+Number of bytes filled in the queue
+
+
+type: long
+
+--
+
+*`beat.stats.libbeat.pipeline.queue.filled.events`*::
++
+--
+Number of events filled in the queue
+
+
+type: long
+
+--
+
+*`beat.stats.libbeat.pipeline.queue.filled.pct`*::
++
+--
+Percentage of the queue filled
+
+
+type: float
+
+--
+
 *`beat.stats.libbeat.pipeline.queue.max_events`*::
 +
 --
+Maximum number of events allowed in the queue
+
+
+type: long
+
+--
+
+*`beat.stats.libbeat.pipeline.queue.removed.bytes`*::
++
+--
+Number of bytes removed from the queue
+
+
+type: long
+
+--
+
+*`beat.stats.libbeat.pipeline.queue.removed.events`*::
++
+--
+Number of events removed from the queue
+
+
 type: long
 
 --
@@ -10437,6 +10534,43 @@ type: long
 
 
 
+*`beat.stats.libbeat.output.write.latency.histogram.count`*::
++
+--
+type: long
+
+--
+
+*`beat.stats.libbeat.output.write.latency.histogram.max`*::
++
+--
+type: float
+
+--
+
+*`beat.stats.libbeat.output.write.latency.histogram.median`*::
++
+--
+type: long
+
+--
+
+*`beat.stats.libbeat.output.write.latency.histogram.p95`*::
++
+--
+type: float
+
+--
+
+*`beat.stats.libbeat.output.write.latency.histogram.p99`*::
++
+--
+type: float
+
+--
+
+
+
 
 *`beat.stats.output.elasticsearch.bulk_requests.available`*::
 +

diff --git a/metricbeat/module/beat/fields.go b/metricbeat/module/beat/fields.go
diff --git a/metricbeat/module/beat/stats/_meta/fields.yml b/metricbeat/module/beat/stats/_meta/fields.yml
@@ -561,10 +561,64 @@
           fields:
             - name: clients
               type: long
-            - name: queue.acked
-              type: long
-            - name: queue.max_events
-              type: long
+            - name: queue
+              type: group
+              fields:
+                - name: acked
+                  type: long
+                  metric_type: counter
+                  description: >
+                    Number of acknowledged events
+                - name: added.bytes
+                  type: long
+                  metric_type: counter
+                  description: >
+                    Number of bytes added to the queue
+                - name: added.events
+                  type: long
+                  metric_type: counter
+                  description: >
+                    Number of events added to the queue
+                - name: consumed.bytes
+                  type: long
+                  metric_type: counter
+                  description: >
+                    Number of bytes consumed from the queue
+                - name: consumed.events
+                  type: long
+                  metric_type: counter
+                  description: >
+                    Number of events consumed from the queue
+                - name: filled.bytes
+                  type: long
+                  metric_type: gauge
+                  description: >
+                    Number of bytes filled in the queue
+                - name: filled.events
+                  type: long
+                  metric_type: gauge
+                  description: >
+                    Number of events filled in the queue
+                - name: filled.pct
+                  type: float
+                  metric_type: gauge
+                  description: >
+                    Percentage of the queue filled
+                - name: max_events
+                  type: long
+                  metric_type: gauge
+                  description: >
+                    Maximum number of events allowed in the queue
+                - name: removed.bytes
+                  type: long
+                  metric_type: counter
+                  description: >
+                    Number of bytes removed from the queue
+                - name: removed.events
+                  type: long
+                  metric_type: counter
+                  description: >
+                    Number of events removed from the queue
             - name: events
               type: group
               fields:
@@ -665,6 +719,22 @@
                   type: long
                   description: >
                     Number of write errors
+                - name: latency
+                  type: group
+                  fields:
+                    - name: histogram
+                      type: group
+                      fields:
+                        - name: count
+                          type: long
+                        - name: max
+                          type: float
+                        - name: median
+                          type: long
+                        - name: p95
+                          type: float
+                        - name: p99
+                          type: float
     - name: output
       type: group
       fields:

diff --git a/metricbeat/module/beat/stats/data.go b/metricbeat/module/beat/stats/data.go
@@ -72,10 +72,13 @@ var (
 					"bytes":  c.Int("bytes"),
 					"errors": c.Int("errors"),
 					"latency": c.Dict("latency", s.Schema{
-						"count":  c.Int("count"),
-						"max":    c.Int("max"),
-						"median": c.Float("median"),
-						"p99":    c.Float("p99"),
+						"histogram": c.Dict("histogram", s.Schema{
+							"count":  c.Int("count"),
+							"max":    c.Int("max"),
+							"median": c.Float("median"),
+							"p95":    c.Float("p95"),
+							"p99":    c.Float("p99"),
+						}),
 					}),
 				}),
 			}),