Backport missing winlog changes: (#42360)

- Pipeline improvements
- Winlog input docs
- Metadata store tracking of newest events

filebeat/docs/inputs/input-winlog.asciidoc

@@ -452,8 +452,5 @@ performance and reduce CPU usage. *{vista_and_newer}*
   api: wineventlog-experimental
 --------------------------------------------------------------------------------
 
-There are a few notable differences in the events:
-
-* Events that contained data under `winlog.user_data` will now have it under
-  `winlog.event_data`.
-* Setting `include_xml: true` has no effect.
+* If `include_xml` is `true` the performance will be the same as the default API,
+as performance improvements are lost when parsing the XML.

winlogbeat/sys/wineventlog/metadata_store.go

@@ -51,8 +51,11 @@ type PublisherMetadataStore struct {
 
 	winevent.WinMeta
 
+	// Keeps track of the latest metadata available for each event.
+	EventsNewest map[uint16]*EventMetadata
 	// Event ID to event metadata (message and event data param names).
-	Events map[uint32]*EventMetadata
+	// Keeps track of all available versions for each event.
+	EventsByVersion map[uint32]*EventMetadata
 	// Event ID to map of fingerprints to event metadata. The fingerprint value
 	// is hash of the event data parameters count and types.
 	EventFingerprints map[uint32]map[uint64]*EventMetadata
@@ -103,7 +106,8 @@ func NewEmptyPublisherMetadataStore(provider string, log *logp.Logger) *Publishe
 			Levels:   map[uint8]string{},
 			Tasks:    map[uint16]string{},
 		},
-		Events:            map[uint32]*EventMetadata{},
+		EventsNewest:      map[uint16]*EventMetadata{},
+		EventsByVersion:   map[uint32]*EventMetadata{},
 		EventFingerprints: map[uint32]map[uint64]*EventMetadata{},
 		MessagesByID:      map[uint32]string{},
 		log:               log.With("publisher", provider, "empty", true),
@@ -183,15 +187,17 @@ func (s *PublisherMetadataStore) initEvents() error {
 	}
 	defer itr.Close()
 
-	s.Events = map[uint32]*EventMetadata{}
+	s.EventsNewest = map[uint16]*EventMetadata{}
+	s.EventsByVersion = map[uint32]*EventMetadata{}
 	for itr.Next() {
 		evt, err := newEventMetadataFromPublisherMetadata(itr, s.Metadata)
 		if err != nil {
 			s.log.Warnw("Failed to read event metadata from publisher. Continuing to next event.",
 				"error", err)
 			continue
 		}
-		s.Events[getEventCombinedID(evt.EventID, evt.Version)] = evt
+		s.EventsNewest[evt.EventID] = evt
+		s.EventsByVersion[getEventCombinedID(evt.EventID, evt.Version)] = evt
 	}
 	return itr.Err()
 }
@@ -235,7 +241,12 @@ func (s *PublisherMetadataStore) getEventMetadata(eventID uint16, version uint8,
 	// metadata then we just associate the fingerprint with a pointer to the
 	// providers metadata for the event ID.
 
-	defaultEM := s.Events[combinedID]
+	defaultEM, found := s.EventsByVersion[combinedID]
+	if !found {
+		// if we do not have a specific metadata for this event version
+		// we fallback to get the newest available one
+		defaultEM = s.EventsNewest[eventID]
+	}
 
 	// Use XML to get the parameters names.
 	em, err := newEventMetadataFromEventHandle(s.Metadata, eventHandle)

winlogbeat/sys/wineventlog/metadata_store_test.go

@@ -40,7 +40,8 @@ func TestPublisherMetadataStore(t *testing.T) {
 	}
 	defer s.Close()
 
-	assert.NotEmpty(t, s.Events)
+	assert.NotEmpty(t, s.EventsByVersion)
+	assert.NotEmpty(t, s.EventsNewest)
 	assert.Empty(t, s.EventFingerprints)
 
 	t.Run("event_metadata_from_handle", func(t *testing.T) {