Merge branch 'main' of github.com:stefans-elastic/beats into iis-errors

.github/workflows/bump-elastic-stack-snapshot.yml

@@ -32,7 +32,7 @@ jobs:
       - uses: actions/checkout@v4
 
       - name: Install Updatecli in the runner
-        uses: updatecli/updatecli-action@704a64517239e0993c5e3bf6749a063b8f950d9f # v0.76.1
+        uses: updatecli/updatecli-action@4aca518a70708e38063453d8de9c551af7f48ac3 # v0.76.1
 
       - name: Run Updatecli in Apply mode
         run: updatecli --experimental apply --config .github/workflows/updatecli.d/bump-elastic-stack-snapshot.yml --values .github/workflows/updatecli.d/values.d/scm.yml
@@ -41,7 +41,7 @@ jobs:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
 
       - if: ${{ failure()  }}
-        uses: slackapi/slack-github-action@37ebaef184d7626c5f204ab8d3baff4262dd30f0 # v1.27.0
+        uses: slackapi/slack-github-action@485a9d42d3a73031f12ec201c457e2162c45d02d # v2.0.0
         with:
           channel-id: '#ingest-notifications'
           payload: |

.github/workflows/bump-golang.yml

@@ -23,15 +23,15 @@ jobs:
       - uses: actions/checkout@v4
 
       - name: Install Updatecli in the runner
-        uses: updatecli/updatecli-action@704a64517239e0993c5e3bf6749a063b8f950d9f # v0.76.1
+        uses: updatecli/updatecli-action@4aca518a70708e38063453d8de9c551af7f48ac3 # v0.76.1
 
       - name: Run Updatecli in Apply mode
         run: updatecli --experimental apply --config .github/workflows/updatecli.d/${{ matrix.file }} --values .github/workflows/updatecli.d/values.d/scm.yml
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
 
       - if: ${{ failure()  }}
-        uses: slackapi/slack-github-action@37ebaef184d7626c5f204ab8d3baff4262dd30f0 # v1.27.0
+        uses: slackapi/slack-github-action@485a9d42d3a73031f12ec201c457e2162c45d02d # v2.0.0
         with:
           channel-id: '#ingest-notifications'
           payload: |

.github/workflows/check-default.yml

@@ -22,7 +22,7 @@ jobs:
         go-version-file: .go-version
     #  when using ubuntu-latest, python 3.10 is not the default version.
     - name: Fix Code is not compatible with Python 3.12
-      uses: actions/setup-python@v4
+      uses: actions/setup-python@v5
       with:
         python-version: '3.10'
     - name: Run check-default

.github/workflows/check-docs.yml

@@ -30,7 +30,7 @@ jobs:
       run: sudo apt-get install -y librpm-dev
     #  when using ubuntu-latest, python 3.10 is not the default version.
     - name: Fix Code is not compatible with Python 3.12
-      uses: actions/setup-python@v4
+      uses: actions/setup-python@v5
       with:
         python-version: '3.10'
     - name: Run check

.github/workflows/mergify-labels-copier.yml

@@ -0,0 +1,23 @@
+name: mergify backport labels copier
+
+on:
+  pull_request:
+    types:
+      - opened
+
+permissions:
+  contents: read
+
+jobs:
+  mergify-backport-labels-copier:
+    runs-on: ubuntu-latest
+    if: startsWith(github.head_ref, 'mergify/bp/')
+    permissions:
+      # Add GH labels
+      pull-requests: write
+      # See https://github.com/cli/cli/issues/6274
+      repository-projects: read
+    steps:
+      - uses: elastic/oblt-actions/mergify/labels-copier@v1
+        with:
+          excluded-labels-regex: "^backport-*"

.github/workflows/notify-stalled-snapshots.yml

@@ -55,7 +55,7 @@ jobs:
 
       - if: ${{ contains(steps.search.outputs.found, 'true') }}
         name: Report obsoleted branches (slack)
-        uses: slackapi/slack-github-action@37ebaef184d7626c5f204ab8d3baff4262dd30f0 # v1.27.0
+        uses: slackapi/slack-github-action@485a9d42d3a73031f12ec201c457e2162c45d02d # v2.0.0
         with:
           channel-id: '#ingest-notifications'
           payload:  ":red_circle: Elastic Stack version for the `${{ matrix.branch }}` branch has not been updated for a while (`> 7 days`). Review the (<${{ env.URL_QUERY }}|open PRs>)"
@@ -64,7 +64,7 @@ jobs:
 
       - if: ${{ contains(steps.search.outputs.found, 'true') }}
         name: Report obsoleted branches (email)
-        uses: dawidd6/action-send-mail@2cea9617b09d79a095af21254fbcb7ae95903dde
+        uses: dawidd6/action-send-mail@611879133a9569642c41be66f4a323286e9b8a3b
         with:
           server_address: ${{ secrets.MAIL_SERVER }}
           username: ${{ secrets.MAIL_USERNAME }}

.github/workflows/updatecli-compose.yml

@@ -38,7 +38,7 @@ jobs:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
 
       - if: ${{ failure()  }}
-        uses: slackapi/slack-github-action@37ebaef184d7626c5f204ab8d3baff4262dd30f0 # v1.27.0
+        uses: slackapi/slack-github-action@485a9d42d3a73031f12ec201c457e2162c45d02d # v2.0.0
         with:
           channel-id: '#ingest-notifications'
           payload: |

CHANGELOG.next.asciidoc

@@ -248,6 +248,8 @@ https://github.com/elastic/beats/compare/v8.8.1\...main[Check the HEAD diff]
 
 *Packetbeat*
 
+- Properly marshal nested structs in ECS fields, fixing issues with mixed cases in field names {pull}42116[42116]
+
 
 *Winlogbeat*
 
@@ -443,6 +445,7 @@ https://github.com/elastic/beats/compare/v8.8.1\...main[Check the HEAD diff]
 - Collect .NET CLR (IIS) Memory, Exceptions and LocksAndThreads metrics {pull}41929[41929]
 - Added `tier_preference`, `creation_date` and `version` fields to the `elasticsearch.index` metricset. {pull}41944[41944]
 - Add `use_performance_counters` to collect CPU metrics using performance counters on Windows for `system/cpu` and `system/core` {pull}41965[41965]
+- Preserve queries for debugging when `merge_results: true` in SQL module {pull}42271[42271]
 
 *Metricbeat*
 - Add benchmark module {pull}41801[41801]

NOTICE.txt

@@ -12488,11 +12488,11 @@ SOFTWARE
 
 --------------------------------------------------------------------------------
 Dependency : github.com/elastic/elastic-agent-libs
-Version: v0.18.0
+Version: v0.18.1
 Licence type (autodetected): Apache-2.0
 --------------------------------------------------------------------------------
 
-Contents of probable licence file $GOMODCACHE/github.com/elastic/[email protected].0/LICENSE:
+Contents of probable licence file $GOMODCACHE/github.com/elastic/[email protected].1/LICENSE:
 
                                  Apache License
                            Version 2.0, January 2004
@@ -13121,11 +13121,11 @@ Contents of probable licence file $GOMODCACHE/github.com/elastic/[email protected]
 
 --------------------------------------------------------------------------------
 Dependency : github.com/elastic/go-elasticsearch/v8
-Version: v8.14.0
+Version: v8.17.0
 Licence type (autodetected): Apache-2.0
 --------------------------------------------------------------------------------
 
-Contents of probable licence file $GOMODCACHE/github.com/elastic/go-elasticsearch/v8@v8.14.0/LICENSE:
+Contents of probable licence file $GOMODCACHE/github.com/elastic/go-elasticsearch/v8@v8.17.0/LICENSE:
 
                                  Apache License
                            Version 2.0, January 2004

auditbeat/auditbeat.reference.yml

@@ -1573,7 +1573,7 @@ logging.files:
   #path: /var/log/auditbeat
 
   # The name of the files where the logs are written to.
-  #name: auditbeat-event-data
+  #name: auditbeat-events-data
 
   # Configure log file size limit. If the limit is reached, log file will be
   # automatically rotated.

filebeat/filebeat.reference.yml

@@ -2728,7 +2728,7 @@ logging.files:
   #path: /var/log/filebeat
 
   # The name of the files where the logs are written to.
-  #name: filebeat-event-data
+  #name: filebeat-events-data
 
   # Configure log file size limit. If the limit is reached, log file will be
   # automatically rotated.

go.mod

@@ -177,9 +177,9 @@ require (
 	github.com/elastic/bayeux v1.0.5
 	github.com/elastic/ebpfevents v0.6.0
 	github.com/elastic/elastic-agent-autodiscover v0.9.0
-	github.com/elastic/elastic-agent-libs v0.18.0
+	github.com/elastic/elastic-agent-libs v0.18.1
 	github.com/elastic/elastic-agent-system-metrics v0.11.7
-	github.com/elastic/go-elasticsearch/v8 v8.14.0
+	github.com/elastic/go-elasticsearch/v8 v8.17.0
 	github.com/elastic/go-quark v0.2.0
 	github.com/elastic/go-sfdc v0.0.0-20241010131323-8e176480d727
 	github.com/elastic/mito v1.16.0

go.sum

@@ -342,8 +342,8 @@ github.com/elastic/elastic-agent-autodiscover v0.9.0 h1:+iWIKh0u3e8I+CJa3FfWe9h0
 github.com/elastic/elastic-agent-autodiscover v0.9.0/go.mod h1:5iUxLHhVdaGSWYTveSwfJEY4RqPXTG13LPiFoxcpFd4=
 github.com/elastic/elastic-agent-client/v7 v7.15.0 h1:nDB7v8TBoNuD6IIzC3z7Q0y+7bMgXoT2DsHfolO2CHE=
 github.com/elastic/elastic-agent-client/v7 v7.15.0/go.mod h1:6h+f9QdIr3GO2ODC0Y8+aEXRwzbA5W4eV4dd/67z7nI=
-github.com/elastic/elastic-agent-libs v0.18.0 h1:PKG1StgHu2MfOwOryGuAVgNZlZXyvVSDw3SvLUfel+w=
-github.com/elastic/elastic-agent-libs v0.18.0/go.mod h1:5CR02awPrBr+tfmjBBK+JI+dMmHNQjpVY24J0wjbC7M=
+github.com/elastic/elastic-agent-libs v0.18.1 h1:dE6jf/D9bP8eRMQsV7KKpKV/G8zQzwMFBTj1w4e716c=
+github.com/elastic/elastic-agent-libs v0.18.1/go.mod h1:rWdyrrAFzZwgNNi41Tsqhlt2c2GdXWhCEwcsnqISJ2U=
 github.com/elastic/elastic-agent-system-metrics v0.11.7 h1:1xm2okCM0eQZ4jivZgUFSlt6HAn/nPgKB/Fj8eLG6mY=
 github.com/elastic/elastic-agent-system-metrics v0.11.7/go.mod h1:nzkrGajQA29YNcfP62gfzhxX9an3/xdQ3RmfQNw9YTI=
 github.com/elastic/elastic-transport-go/v8 v8.6.0 h1:Y2S/FBjx1LlCv5m6pWAF2kDJAHoSjSRSJCApolgfthA=
@@ -358,8 +358,8 @@ github.com/elastic/go-docappender/v2 v2.3.0 h1:Vr+l36jM+sE/LHp0JFxSIbHlWTSk8CpBb
 github.com/elastic/go-docappender/v2 v2.3.0/go.mod h1:VNWgXUE9HX1G6W6ON8dOs/KdH8aCxXir/fxxcfrnov4=
 github.com/elastic/go-elasticsearch/v7 v7.17.10 h1:TCQ8i4PmIJuBunvBS6bwT2ybzVFxxUhhltAs3Gyu1yo=
 github.com/elastic/go-elasticsearch/v7 v7.17.10/go.mod h1:OJ4wdbtDNk5g503kvlHLyErCgQwwzmDtaFC4XyOxXA4=
-github.com/elastic/go-elasticsearch/v8 v8.14.0 h1:1ywU8WFReLLcxE1WJqii3hTtbPUE2hc38ZK/j4mMFow=
-github.com/elastic/go-elasticsearch/v8 v8.14.0/go.mod h1:WRvnlGkSuZyp83M2U8El/LGXpCjYLrvlkSgkAH4O5I4=
+github.com/elastic/go-elasticsearch/v8 v8.17.0 h1:e9cWksE/Fr7urDRmGPGp47Nsp4/mvNOrU8As1l2HQQ0=
+github.com/elastic/go-elasticsearch/v8 v8.17.0/go.mod h1:lGMlgKIbYoRvay3xWBeKahAiJOgmFDsjZC39nmO3H64=
 github.com/elastic/go-libaudit/v2 v2.6.1 h1:eN7tobGizmB+OJpCuG7gvPX7Nxni//H47uvMDXlMrI0=
 github.com/elastic/go-libaudit/v2 v2.6.1/go.mod h1:8205nkf2oSrXFlO4H5j8/cyVMoSF3Y7jt+FjgS4ubQU=
 github.com/elastic/go-licenser v0.4.2 h1:bPbGm8bUd8rxzSswFOqvQh1dAkKGkgAmrPxbUi+Y9+A=

heartbeat/heartbeat.reference.yml

@@ -1660,7 +1660,7 @@ logging.files:
   #path: /var/log/heartbeat
 
   # The name of the files where the logs are written to.
-  #name: heartbeat-event-data
+  #name: heartbeat-events-data
 
   # Configure log file size limit. If the limit is reached, log file will be
   # automatically rotated.

libbeat/_meta/config/logging.reference.yml.tmpl

@@ -92,7 +92,7 @@ logging.files:
   #path: /var/log/{{.BeatName}}
 
   # The name of the files where the logs are written to.
-  #name: {{.BeatName}}-event-data
+  #name: {{.BeatName}}-events-data
 
   # Configure log file size limit. If the limit is reached, log file will be
   # automatically rotated.

libbeat/docs/loggingconfig.asciidoc

@@ -320,7 +320,7 @@ the <<directory-layout>> section for details.
 [float]
 ==== `logging.event_data.files.name`
 
-The name of the file that logs are written to. The default is '{beatname_lc}'-event-data.
+The name of the file that logs are written to. The default is '{beatname_lc}'-events-data.
 
 [float]
 ==== `logging.event_data.files.rotateeverybytes`

libbeat/docs/shared-beats-attributes.asciidoc

@@ -2,6 +2,7 @@
 :beats-ref-all: https://www.elastic.co/guide/en/beats/libbeat
 :dashboards: https://artifacts.elastic.co/downloads/beats/beats-dashboards/beats-dashboards-{version}.zip
 :dockerimage: docker.elastic.co/beats/{beatname_lc}:{version}
+:dockerimage-wolfi: docker.elastic.co/beats/{beatname_lc}-wolfi:{version}
 :dockerconfig: https://raw.githubusercontent.com/elastic/beats/{branch}/deploy/docker/{beatname_lc}.docker.yml
 :downloads: https://artifacts.elastic.co/downloads/beats
 :libbeat-processors-dir: {beats-root}/libbeat/processors

libbeat/docs/shared-docker.asciidoc

@@ -35,6 +35,16 @@ https://www.docker.elastic.co[www.docker.elastic.co].
 
 ifndef::apm-server[]
 
+As another option, you can use the hardened link:https://wolfi.dev/[Wolfi] image.
+Using Wolfi images requires Docker version 20.10.10 or higher.
+For details about why the Wolfi images have been introduced, refer to our article 
+link:https://www.elastic.co/blog/reducing-cves-in-elastic-container-images[Reducing CVEs in Elastic container images].
+
+[source,terminal,subs="attributes"]
+----
+docker pull {dockerimage-wolfi}
+----
+
 ==== Optional: Verify the image
 
 You can use the https://docs.sigstore.dev/cosign/installation/[Cosign application] to verify the {beatname_uc} Docker image signature.

libbeat/publisher/pipeline/controller_test.go

@@ -231,6 +231,8 @@ func TestQueueProducerBlocksUntilOutputIsSet(t *testing.T) {
 		}()
 	}
 	allStarted := waitUntilTrue(time.Second, func() bool {
+		controller.queueLock.Lock()
+		defer controller.queueLock.Unlock()
 		return len(controller.pendingRequests) == producerCount
 	})
 	assert.True(t, allStarted, "All queueProducer requests should be saved as pending requests by outputController")

metricbeat/metricbeat.reference.yml

@@ -2497,7 +2497,7 @@ logging.files:
   #path: /var/log/metricbeat
 
   # The name of the files where the logs are written to.
-  #name: metricbeat-event-data
+  #name: metricbeat-events-data
 
   # Configure log file size limit. If the limit is reached, log file will be
   # automatically rotated.

packetbeat/packetbeat.reference.yml

@@ -2039,7 +2039,7 @@ logging.files:
   #path: /var/log/packetbeat
 
   # The name of the files where the logs are written to.
-  #name: packetbeat-event-data
+  #name: packetbeat-events-data
 
   # Configure log file size limit. If the limit is reached, log file will be
   # automatically rotated.

packetbeat/pb/event.go

@@ -416,8 +416,23 @@ func marshalStruct(m mapstr.M, key string, val reflect.Value) error {
 	}
 
 	typ := val.Type()
+	// pre-emptively handle time
+	if reflect.TypeOf(time.Time{}) == typ {
+		_, err := m.Put(key, val.Interface())
+		if err != nil {
+			return fmt.Errorf("error creating time value: %w", err)
+		}
+		return nil
+	}
+
+	// NumField() will panic if we don't have a struct
+	if val.Type().Kind() != reflect.Struct {
+		return fmt.Errorf("value must be a struct or a pointer to a struct, but got %v at key %s", val.Type(), key)
+	}
+
 	for i := 0; i < typ.NumField(); i++ {
 		structField := typ.Field(i)
+
 		tag := getTag(structField)
 		if tag == "" {
 			continue
@@ -431,7 +446,7 @@ func marshalStruct(m mapstr.M, key string, val reflect.Value) error {
 				case "inline":
 					inline = true
 				default:
-					return fmt.Errorf("Unsupported flag %q in tag %q of type %s", flag, tag, typ)
+					return fmt.Errorf("unsupported flag %q in tag %q of type %s", flag, tag, typ)
 				}
 			}
 			tag = tags[0]
@@ -446,6 +461,13 @@ func marshalStruct(m mapstr.M, key string, val reflect.Value) error {
 			if err := marshalStruct(m, key, fieldValue); err != nil {
 				return err
 			}
+			// look for a struct or pointer to a struct
+			// that reflect.Ptr check is needed so Elem() doesn't panic
+		} else if (structField.Type.Kind() == reflect.Ptr && fieldValue.Elem().Kind() == reflect.Struct) ||
+			structField.Type.Kind() == reflect.Struct {
+			if err := marshalStruct(m, key+"."+tag, fieldValue); err != nil {
+				return err
+			}
 		} else {
 			if _, err := m.Put(key+"."+tag, fieldValue.Interface()); err != nil {
 				return err