[winlogbeat] Default to raw api (#42275)

* Default to raw api * Install sysmon event manifests * Re-generate golden files * Remove unused renderer option * Fix tests * Fix tests * Remove field from docs * Fix doc default value * Make script compatible with older ps * Print error * Fix web request * Clean tests and Fallback to use latest available metadata if non found * Fix security pipeline for new rendered events * Regenerate ingest golden files * Fix test for win2016 * fix github action * Update winlogbeat/sys/wineventlog/metadata_store_test.go Co-authored-by: Leszek Kubik <[email protected]> * simplify sysmon script * Add retries and simplify call --------- Co-authored-by: Leszek Kubik <[email protected]>
elastic · Jan 20, 2025 · 5ead8ab · 5ead8ab
1 parent 7fd2d46
commit 5ead8ab
Show file tree

Hide file tree

Showing 244 changed files with 4,516 additions and 3,777 deletions.
diff --git a/.buildkite/scripts/install_sysmon.ps1 b/.buildkite/scripts/install_sysmon.ps1
@@ -0,0 +1,56 @@
+$downloadUrl = "https://live.sysinternals.com/Sysmon64.exe"
+$tempFolder = "$env:TEMP\SysmonDownload"
+$sysmonPath = "$tempFolder\Sysmon64.exe"
+
+if (!(Test-Path $tempFolder)) {
+    New-Item -ItemType Directory -Path $tempFolder
+}
+
+$ProgressPreference = 'SilentlyContinue'
+function ParseErrorForResponseBody($Error) {
+    if ($PSVersionTable.PSVersion.Major -lt 6) {
+        if ($Error.Exception.Response) {  
+            $Reader = New-Object System.IO.StreamReader($Error.Exception.Response.GetResponseStream())
+            $Reader.BaseStream.Position = 0
+            $Reader.DiscardBufferedData()
+            $ResponseBody = $Reader.ReadToEnd()
+            if ($ResponseBody.StartsWith('{')) {
+                $ResponseBody = $ResponseBody | ConvertFrom-Json
+            }
+            return $ResponseBody
+        }
+    }
+    else {
+        return $Error.ErrorDetails.Message
+    }
+}
+
+$attempts=1
+do
+{
+    $attempts++
+    try {
+        [Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12
+        $result = Invoke-WebRequest -Uri $downloadUrl -OutFile $sysmonPath -UseBasicParsing
+        break
+    }
+    catch {
+        $resp = ParseErrorForResponseBody($_)
+        Write-Host "$resp"
+        if ($attempts -gt 5) {
+            exit 1
+        }
+    }
+} while ($attempts -le 5)
+
+Write-Host "Sysmon64.exe downloaded successfully."
+
+if ($sysmonPath) {
+    Start-Process -FilePath $sysmonPath -ArgumentList "-m" -Wait
+
+    Write-Host "Sysmon event manifest installation completed."
+} else {
+    Write-Host "Sysmon executable not found in the downloaded archive."
+}
+
+Remove-Item -Path $tempFolder -Force -Recurse
diff --git a/.buildkite/x-pack/pipeline.xpack.winlogbeat.yml b/.buildkite/x-pack/pipeline.xpack.winlogbeat.yml
@@ -65,6 +65,7 @@ steps:
       - label: ":windows: x-pack/winlogbeat Win 2019 Unit Tests"
         key: "mandatory-win-2019-unit-tests"
         command: |
+          .buildkite/scripts/install_sysmon.ps1
           Set-Location -Path x-pack/winlogbeat
           mage build unitTest
         retry:
@@ -91,6 +92,7 @@ steps:
 
       - label: ":windows: x-pack/winlogbeat: Win 2016 Unit Tests"
         command: |
+          .buildkite/scripts/install_sysmon.ps1
           Set-Location -Path x-pack/winlogbeat
           mage build unitTest
         key: "mandatory-win-2016-unit-tests"
@@ -118,6 +120,7 @@ steps:
 
       - label: ":windows: x-pack/winlogbeat: Win 2022 Unit Tests"
         command: |
+          .buildkite/scripts/install_sysmon.ps1
           Set-Location -Path x-pack/winlogbeat
           mage build unitTest
         key: "mandatory-win-2022-unit-tests"
@@ -150,6 +153,7 @@ steps:
     steps:
       - label: ":windows: x-pack/winlogbeat: Win 10 Unit Tests"
         command: |
+          .buildkite/scripts/install_sysmon.ps1
           Set-Location -Path x-pack/winlogbeat
           mage build unitTest
         key: "extended-win-10-unit-tests"
@@ -177,6 +181,7 @@ steps:
 
       - label: ":windows: x-pack/winlogbeat: Win 11 Unit Tests"
         command: |
+          .buildkite/scripts/install_sysmon.ps1
           Set-Location -Path x-pack/winlogbeat
           mage build unitTest
         key: "extended-win-11-unit-tests"
@@ -204,6 +209,7 @@ steps:
 
       - label: ":windows: x-pack/winlogbeat: Win 2019 Unit Tests"
         command: |
+          .buildkite/scripts/install_sysmon.ps1
           Set-Location -Path x-pack/winlogbeat
           mage build unitTest
         key: "extended-win-2019-unit-tests"

diff --git a/.github/workflows/check-dev-tools.yml b/.github/workflows/check-dev-tools.yml
@@ -21,6 +21,10 @@ jobs:
     - uses: actions/setup-go@v5
       with:
         go-version-file: .go-version
+    - name: Fix Code is not compatible with Python 3.12
+      uses: actions/setup-python@v5
+      with:
+        python-version: '3.10'
     - name: Run check/update
       run: |
         go install github.com/magefile/mage

diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc
@@ -92,6 +92,7 @@ https://github.com/elastic/beats/compare/v8.8.1\...main[Check the HEAD diff]
 *Winlogbeat*
 
 - Add "event.category" and "event.type" to Sysmon module for EventIDs 8, 9, 19, 20, 27, 28, 255 {pull}35193[35193]
+- Default to use raw api and delete older xml implementation. {pull}42275[42275]
 
 *Functionbeat*
 

diff --git a/dev-tools/ecs-migration.yml b/dev-tools/ecs-migration.yml
@@ -1972,12 +1972,6 @@
 
 # renames to match Windows Event Log naming
 
-- from: type
-  to: winlog.api
-  alias: true
-  beat: winlogbeat
-  rename: false
-
 - from: log_name
   to: winlog.channel
   alias: true

diff --git a/filebeat/docs/inputs/input-winlog.asciidoc b/filebeat/docs/inputs/input-winlog.asciidoc
@@ -40,7 +40,7 @@ Here is a sample configuration:
 ==== `batch_read_size`
 
 The maximum number of event log records to read from the Windows API in a single
-batch. The default batch size is 100. Most Windows versions return an error if
+batch. The default batch size is 512. Most Windows versions return an error if
 the value is larger than 1024. *{vista_and_newer}*
 
 {beatname_uc} starts a goroutine (a lightweight thread) to read from each
@@ -181,40 +181,6 @@ IDs to include (e.g. 4700-4800), and single event IDs to exclude (e.g. -4735).
   event_id: 4624, 4625, 4700-4800, -4735
 --------------------------------------------------------------------------------
 
-[WARNING]
-=======================================
-If you specify more than 22 query conditions (event IDs or event ID ranges), some
-versions of Windows will prevent {beatname_uc} from reading the event log due to
-limits in the query system. If this occurs a similar warning as shown below will
-be logged by {beatname_uc}, and it will continue processing data from other event
-logs.
-
-`WARN EventLog[Application] Open() error. No events will be read from this
-source. The specified query is invalid.`
-
-In some cases, the limit may be lower than 22 conditions. For instance, using a
-mixture of ranges and single event IDs, along with an additional parameter such
-as `ignore older`, results in a limit of 21 conditions.
-
-If you have more than 22 conditions, you can workaround this Windows limitation
-by using a drop_event[drop-event] processor to do the filtering after
-{beatname_uc} has received the events from Windows. The filter shown below is
-equivalent to `event_id: 903, 1024, 4624` but can be expanded beyond 22
-event IDs.
-
-[source,yaml]
---------------------------------------------------------------------------------
-- type: winlog
-  name: Security
-  processors:
-    - drop_event.when.not.or:
-      - equals.winlog.event_id: 903
-      - equals.winlog.event_id: 1024
-      - equals.winlog.event_id: 4624
---------------------------------------------------------------------------------
-
-=======================================
-
 [float]
 ==== `language`
 
@@ -350,6 +316,9 @@ Example:
   include_xml: true
 --------------------------------------------------------------------------------
 
+* This can have a significant impact on performance that can vary depending
+on your system specs.
+
 [float]
 ==== `tags`
 
@@ -434,26 +403,3 @@ stopped. *{vista_and_newer}*
 
 Setting `no_more_events` to `stop` is useful when reading from archived event
 log files where you want to read the whole file then exit. 
-
-[float]
-==== `api`
-
-This selects the event log reader implementation that is used to read events
-from the Windows APIs. You should only set this option when testing experimental
-features. When the value is set to `wineventlog-experimental` {beatname_uc} will
-replace the default event log reader with the **experimental** implementation.
-We are evaluating this implementation to see if it can provide increased
-performance and reduce CPU usage. *{vista_and_newer}*
-
-[source,yaml]
---------------------------------------------------------------------------------
-- type: winlog
-  name: ForwardedEvents
-  api: wineventlog-experimental
---------------------------------------------------------------------------------
-
-There are a few notable differences in the events:
-
-* Events that contained data under `winlog.user_data` will now have it under
-  `winlog.event_data`.
-* Setting `include_xml: true` has no effect.
diff --git a/winlogbeat/_meta/fields.common.yml b/winlogbeat/_meta/fields.common.yml
@@ -23,13 +23,6 @@
         All fields specific to the Windows Event Log are defined here.
       fields:
 
-        - name: api
-          required: true
-          description: >
-            The event log API type used to read the record. The possible values are
-            "wineventlog" for the Windows Event Log XML reader or "wineventlog-raw" for its
-            more performant implementation.
-
         - name: activity_id
           type: keyword
           required: false
@@ -499,10 +492,6 @@
     version of Winlogbeat. These are added to the index template when
     `migration.6_to_7.enable: true` is set in the configuration.
   fields:
-    - name: type
-      type: alias
-      path: winlog.api
-      migration: true
 
     - name: activity_id
       type: alias

diff --git a/winlogbeat/docs/fields.asciidoc b/winlogbeat/docs/fields.asciidoc
@@ -15372,15 +15372,6 @@ Field aliases based on Winlogbeat 6.x that point to the fields for this version
 
 
 
-*`type`*::
-+
---
-type: alias
-
-alias to: winlog.api
-
---
-
 *`activity_id`*::
 +
 --
@@ -16279,16 +16270,6 @@ All fields specific to the Windows Event Log are defined here.
 
 
 
-*`winlog.api`*::
-+
---
-The event log API type used to read the record. The possible values are "wineventlog" for the Windows Event Log XML reader or "wineventlog-raw" for its more performant implementation.
-
-
-required: True
-
---
-
 *`winlog.activity_id`*::
 +
 --

diff --git a/winlogbeat/docs/winlogbeat-options.asciidoc b/winlogbeat/docs/winlogbeat-options.asciidoc
@@ -98,7 +98,7 @@ winlogbeat.event_logs:
 ==== `event_logs.batch_read_size`
 
 The maximum number of event log records to read from the Windows API in a single
-batch. The default batch size is 100. Most Windows versions return an error if
+batch. The default batch size is 512. Most Windows versions return an error if
 the value is larger than 1024. *{vista_and_newer}*
 
 {beatname_uc} starts a goroutine (a lightweight thread) to read from each
@@ -376,6 +376,9 @@ winlogbeat.event_logs:
     include_xml: true
 --------------------------------------------------------------------------------
 
+* This can have a significant impact on performance that can vary depending
+on your system specs.
+
 [float]
 ==== `event_logs.tags`
 
@@ -462,24 +465,6 @@ Setting `no_more_events` to `stop` is useful when reading from archived event
 log files where you want to read the whole file then exit. There's a complete
 example of how to read from an `.evtx` file in the <<reading-from-evtx,FAQ>>.
 
-[float]
-==== `event_logs.api`
-
-This selects the event log reader implementation that is used to read events
-from the Windows APIs. When the value is set to `wineventlog-raw` Winlogbeat will
-replace the default XML event log reader with a more performant implementation.
-*{vista_and_newer}*
-
-[source,yaml]
---------------------------------------------------------------------------------
-winlogbeat.event_logs:
-  - name: ForwardedEvents
-    api: wineventlog-raw
---------------------------------------------------------------------------------
-
-* If `include_xml` is `true` the performance will be the same as the default API,
-as performance improvements are lost when parsing the XML.
-
 
 [float]
 ==== `overwrite_pipelines`

diff --git a/winlogbeat/eventlog/bench_test.go b/winlogbeat/eventlog/bench_test.go
@@ -58,21 +58,17 @@ func TestBenchmarkRead(t *testing.T) {
 		safeWriteEvent(t, writer, uint32(rand.Int63()%1000), strconv.Itoa(i)+" "+randomSentence(256))
 	}
 
-	for _, api := range []string{winEventLogAPIName, winEventLogExpAPIName} {
-		t.Run("api="+api, func(t *testing.T) {
-			for _, includexml := range []bool{true, false} {
-				for _, batchSize := range []int{10, 100, 500, 1000} {
-					t.Run(fmt.Sprintf("include_xml=%v/batch_size=%d", includexml, batchSize), func(t *testing.T) {
-						result := testing.Benchmark(benchmarkEventLog(api, includexml, batchSize))
-						outputBenchmarkResults(t, result)
-					})
-				}
-			}
-		})
+	for _, includexml := range []bool{true, false} {
+		for _, batchSize := range []int{10, 100, 500, 1000} {
+			t.Run(fmt.Sprintf("include_xml=%v/batch_size=%d", includexml, batchSize), func(t *testing.T) {
+				result := testing.Benchmark(benchmarkEventLog(includexml, batchSize))
+				outputBenchmarkResults(t, result)
+			})
+		}
 	}
 }
 
-func benchmarkEventLog(api string, includexml bool, batchSize int) func(b *testing.B) {
+func benchmarkEventLog(includexml bool, batchSize int) func(b *testing.B) {
 	return func(b *testing.B) {
 		conf := mapstr.M{
 			"name":            providerName,
@@ -81,7 +77,7 @@ func benchmarkEventLog(api string, includexml bool, batchSize int) func(b *testi
 			"include_xml":     includexml,
 		}
 
-		log := openLog(b, api, nil, conf)
+		log := openLog(b, nil, conf)
 		defer log.Close()
 
 		events := 0