Merge remote-tracking branch 'upstream/main' into winlogbeat-default-api

.github/workflows/check-default.yml

@@ -20,6 +20,11 @@ jobs:
     - uses: actions/setup-go@v5
       with:
         go-version-file: .go-version
+    #  when using ubuntu-latest, python 3.10 is not the default version.
+    - name: Fix Code is not compatible with Python 3.12
+      uses: actions/setup-python@v4
+      with:
+        python-version: '3.10'
     - name: Run check-default
       run: |
         go install github.com/magefile/mage

.github/workflows/check-docs.yml

@@ -28,6 +28,11 @@ jobs:
       run: sudo apt-get install -y libsystemd-dev
     - name: Install librpm-dev
       run: sudo apt-get install -y librpm-dev
+    #  when using ubuntu-latest, python 3.10 is not the default version.
+    - name: Fix Code is not compatible with Python 3.12
+      uses: actions/setup-python@v4
+      with:
+        python-version: '3.10'
     - name: Run check
       run: |
         make check

CHANGELOG.next.asciidoc

@@ -17,7 +17,9 @@ https://github.com/elastic/beats/compare/v8.8.1\...main[Check the HEAD diff]
 - Drop support for Debian 10 and upgrade statically linked glibc from 2.28 to 2.31 {pull}41402[41402]
 - Fix metrics not being ingested, due to "Limit of total fields [10000] has been exceeded while adding new fields [...]". The total fields limit has been increased to 12500. No significant performance impact on Elasticsearch is anticipated. {pull}41640[41640]
 - Set default kafka version to 2.1.0 in kafka output and filebeat. {pull}41662[41662]
+- Replace default Ubuntu-based images with UBI-minimal-based ones {pull}42150[42150]
 - Fix templates and docs to use correct `--` version of command line arguments. {issue}42038[42038] {pull}42060[42060]
+- removed support for a single `-` to precede multi-letter command line arguments.  Use `--` instead. {issue}42117[42117] {pull}42209[42209] 
 
 *Auditbeat*
 
@@ -384,6 +386,9 @@ https://github.com/elastic/beats/compare/v8.8.1\...main[Check the HEAD diff]
 - Rate limiting operability improvements in the Okta provider of the Entity Analytics input. {issue}40106[40106] {pull}41977[41977]
 - Added default values in the streaming input for websocket retries and put a cap on retry wait time to be lesser than equal to the maximum defined wait time. {pull}42012[42012]
 - Rate limiting fault tolerance improvements in the Okta provider of the Entity Analytics input. {issue}40106[40106] {pull}42094[42094]
+- Added OAuth2 support with auto token refresh for websocket streaming input. {issue}41989[41989] {pull}42212[42212]
+- Added infinite & blanket retry options to websockets and improved logging and retry logic. {pull}42225[42225]
+- Introduce ignore older and start timestamp filters for AWS S3 input. {pull}41804[41804]
 
 *Auditbeat*
 

auditbeat/main_test.go

@@ -24,7 +24,6 @@ import (
 	"testing"
 
 	"github.com/elastic/beats/v7/auditbeat/cmd"
-	"github.com/elastic/beats/v7/libbeat/cfgfile"
 	"github.com/elastic/beats/v7/libbeat/tests/system/template"
 )
 
@@ -35,14 +34,11 @@ func init() {
 	systemTest = flag.Bool("systemTest", false, "Set to true when running system tests")
 
 	cmd.RootCmd.PersistentFlags().AddGoFlag(flag.CommandLine.Lookup("systemTest"))
-	cfgfile.AddAllowedBackwardsCompatibleFlag("systemTest")
 	cmd.RootCmd.PersistentFlags().AddGoFlag(flag.CommandLine.Lookup("test.coverprofile"))
-	cfgfile.AddAllowedBackwardsCompatibleFlag("test.coverprofile")
 }
 
 // Test started when the test binary is started. Only calls main.
 func TestSystem(*testing.T) {
-	cfgfile.ConvertFlagsForBackwardsCompatibility()
 	if *systemTest {
 		main()
 	}

auditbeat/tests/system/test_show_command.py

@@ -57,7 +57,7 @@ def test_show_auditd_rules(self):
                 }
             }]
         )
-        proc = self.start_beat(extra_args=['-strict.perms=false'])
+        proc = self.start_beat(extra_args=['--strict.perms=false'])
         # auditbeat adds an extra rule to ignore itself
         self.wait_log_contains('Successfully added {0} of {0} audit rules.'.format(len(rules) + 1),
                                max_timeout=30)

dev-tools/cmd/dashboards/export_dashboards.go

@@ -46,7 +46,7 @@ func main() {
 	insecure := flag.Bool("insecure", false, "Disable TLS verification.")
 	spaceID := flag.String("space-id", "", "Space ID")
 	dashboard := flag.String("dashboard", "", "Dashboard ID")
-	fileOutput := flag.String("output", "", "Output NDJSON file, when exporting dashboards for Beats, please use -folder instead")
+	fileOutput := flag.String("output", "", "Output NDJSON file, when exporting dashboards for Beats, please use --folder instead")
 	folderOutput := flag.String("folder", "", "Output folder to save all assets to more human friendly JSON format")
 	ymlFile := flag.String("yml", "", "Path to the module.yml file containing the dashboards")
 	flag.BoolVar(&indexPattern, "indexPattern", false, "include index-pattern in output")
@@ -56,7 +56,7 @@ func main() {
 	log.SetFlags(0)
 
 	if len(*fileOutput) > 0 {
-		log.Fatalf("-output is configured, please use -folder flag instead to get the expected formatting of assets")
+		log.Fatalf("--output is configured, please use --folder flag instead to get the expected formatting of assets")
 	}
 
 	u, err := url.Parse(*kibanaURL)
@@ -92,10 +92,10 @@ func main() {
 
 	if len(*ymlFile) == 0 && len(*dashboard) == 0 {
 		flag.Usage()
-		log.Fatalf("Please specify a dashboard ID (-dashboard) or a manifest file (-yml)")
+		log.Fatalf("Please specify a dashboard ID (--dashboard) or a manifest file (--yml)")
 	}
 	if len(*folderOutput) == 0 {
-		log.Fatalf("Please specify a target folder using -folder flag")
+		log.Fatalf("Please specify a target folder using --folder flag")
 	}
 
 	if len(*ymlFile) > 0 {

dev-tools/mage/dashboard.go

@@ -49,8 +49,8 @@ func ExportDashboard() error {
 	folder := CWD("module", module)
 
 	args := []string{
-		"-folder", folder,
-		"-dashboard", id,
+		"--folder", folder,
+		"--dashboard", id,
 	}
 	if kibanaURL := EnvOr("KIBANA_URL", ""); kibanaURL != "" {
 		args = append(args, "-kibana", kibanaURL)

dev-tools/mage/gotool/go.go

@@ -163,7 +163,7 @@ func HasTests(pkg string) (bool, error) {
 }
 
 func (goTest) WithCoverage(to string) ArgOpt {
-	return combine(flagArg("-cover", ""), flagArgIf("-test.coverprofile", to))
+	return combine(flagArg("-cover", ""), flagArgIf("--test.coverprofile", to))
 }
 func (goTest) Short(b bool) ArgOpt        { return flagBoolIf("-test.short", b) }
 func (goTest) Use(bin string) ArgOpt      { return extraArgIf("use", bin) }
@@ -224,10 +224,6 @@ func runVGo(cmd string, args *Args) error {
 	}, cmd, args)
 }
 
-func runGo(cmd string, args *Args) error {
-	return execGoWith(sh.RunWith, cmd, args)
-}
-
 func execGoWith(
 	fn func(map[string]string, string, ...string) error,
 	cmd string, args *Args,

dev-tools/packaging/packages.yml

@@ -159,7 +159,7 @@ shared:
   - &docker_spec
     <<: *binary_spec
     extra_vars:
-      from: '--platform=linux/amd64 ubuntu:24.04'
+      from: '--platform=linux/amd64 docker.elastic.co/ubi9/ubi-minimal'
       buildFrom: '--platform=linux/amd64 cgr.dev/chainguard/wolfi-base'
       user: '{{ .BeatName }}'
       linux_capabilities: ''
@@ -172,7 +172,7 @@ shared:
   - &docker_arm_spec
     <<: *docker_spec
     extra_vars:
-      from: '--platform=linux/arm64 ubuntu:24.04'
+      from: '--platform=linux/arm64 docker.elastic.co/ubi9/ubi-minimal'
       buildFrom: '--platform=linux/arm64 cgr.dev/chainguard/wolfi-base'
 
   - &docker_ubi_spec

dev-tools/packaging/templates/docker/Dockerfile.tmpl

@@ -49,25 +49,14 @@ RUN for iter in {1..10}; do \
 {{- if contains .from "ubi-minimal" }}
 RUN for iter in {1..10}; do \
         microdnf -y update && \
-        microdnf -y install findutils shadow-utils && \
+        microdnf -y install findutils shadow-utils ca-certificates \
+        gawk libcap xz tar && \
         microdnf clean all && \
         exit_code=0 && break || exit_code=$? && echo "microdnf error: retry $iter in 10s" && sleep 10; \
     done; \
     (exit $exit_code)
 {{- end }}
 
-{{- if contains .from "ubuntu" }}
-RUN touch /var/mail/ubuntu && chown ubuntu /var/mail/ubuntu && userdel -r ubuntu
-
-RUN for iter in {1..10}; do \
-        apt-get update -y && \
-        DEBIAN_FRONTEND=noninteractive apt-get install --no-install-recommends --yes ca-certificates curl gawk libcap2-bin xz-utils && \
-        apt-get clean all && \
-        exit_code=0 && break || exit_code=$? && echo "apt-get error: retry $iter in 10s" && sleep 10; \
-    done; \
-    (exit $exit_code)
-{{- end }}
-
 LABEL \
   org.label-schema.build-date="{{ date }}" \
   org.label-schema.schema-version="1.0" \
@@ -166,7 +155,7 @@ USER {{ .user }}
 RUN (npm i -g --loglevel verbose --production --engine-strict  @elastic/synthetics@stack_release || sh -c 'tail -n +1 ${NPM_CONFIG_PREFIX}/_logs/* && exit 1')
 {{- end }}
 
-{{- if (and (eq .BeatName "heartbeat") (contains .from "ubuntu")) }}
+{{- if (and (eq .BeatName "heartbeat") (contains .from "ubi-minimal")) }}
 USER root
 ENV NODE_PATH={{ $beatHome }}/.node
 RUN echo \
@@ -217,12 +206,23 @@ USER root
 # We don't use npx playwright install-deps because that could pull a newer version
 # Install additional fonts as well
 RUN for iter in {1..10}; do \
-    apt-get update -y && \
-    $NODE_PATH/node/lib/node_modules/@elastic/synthetics/node_modules/.bin/playwright install-deps chromium && \
-    DEBIAN_FRONTEND=noninteractive apt-get install --no-install-recommends --yes \
-    fonts-noto \
-    fonts-noto-cjk && \
-    exit_code=0 && break || exit_code=$? && echo "apt-get error: retry $iter in 10s" && sleep 10; \
+    microdnf -y update && \
+    microdnf -y install fontconfig freetype cairo glib2 gtk3 pango xorg-x11-fonts-misc xorg-x11-fonts-Type1 \
+    at-spi2-atk atk at-spi2-core alsa-lib cups-libs dbus-libs libdrm mesa-libEGL mesa-libgbm nspr nss libX11 \
+    libX11-xcb libxcb libXcomposite libXdamage libXext libXfixes libXrandr libxkbcommon libxshmfence glib2 \
+    dbus-glib libicu mesa-libGL unzip -y && \
+    mkdir -p /usr/share/fonts/google-noto && \
+    curl -LO https://noto-website-2.storage.googleapis.com/pkgs/NotoSansCJKjp-hinted.zip && \
+    unzip NotoSansCJKjp-hinted.zip -d /usr/share/fonts/google-noto && \
+    rm -f NotoSansCJKjp-hinted.zip && \
+    microdnf -y remove unzip && \
+    curl -LO https://github.com/googlefonts/noto-fonts/raw/main/hinted/ttf/NotoSans/NotoSans-Regular.ttf && \
+    mv NotoSans-Regular.ttf /usr/share/fonts/google-noto && \
+    curl -LO https://github.com/googlefonts/noto-emoji/raw/main/fonts/NotoColorEmoji.ttf && \
+    mv NotoColorEmoji.ttf /usr/share/fonts/google-noto && \
+    fc-cache -fv && \
+    microdnf clean all && \
+    exit_code=0 && break || exit_code=$? && echo "microdnf error: retry $iter in 10s" && sleep 10; \
 done; \
 (exit $exit_code)
 {{- end }}

docs/devguide/modules-dev-guide.asciidoc

@@ -100,7 +100,7 @@ Then run `export_dashboards` like this:
 ----
 $ cd dev-tools/cmd/dashboards
 $ make # if export_dashboard is not built yet
-$ ./export_dashboards -yml '../../../filebeat/module/{module}/module.yml'
+$ ./export_dashboards --yml '../../../filebeat/module/{module}/module.yml'
 ----
 
 New Filebeat modules might not be compatible with Kibana 5.x. To export dashboards

docs/devguide/newdashboards.asciidoc

@@ -253,7 +253,7 @@ MODULE=redis ID=AV4REOpp5NkDleZmzKkE mage exportDashboard
 
 [source,shell]
 ---------------
-./filebeat export dashboard -id 7fea2930-478e-11e7-b1f0-cb29bac6bf8b -folder module/redis
+./filebeat export dashboard --id 7fea2930-478e-11e7-b1f0-cb29bac6bf8b --folder module/redis
 ---------------
 
 This generates an appropriate folder under module/redis for the dashboard, separating assets into dashboards, searches, vizualizations, etc.
@@ -289,12 +289,12 @@ By passing the yml file to the `export_dashboards.go` script or to the Beat, you
 
 [source,shell]
 -------------------
-go run dev-tools/cmd/dashboards/export_dashboards.go -yml filebeat/module/system/module.yml -folder dashboards
+go run dev-tools/cmd/dashboards/export_dashboards.go --yml filebeat/module/system/module.yml --folder dashboards
 -------------------
 
 [source,shell]
 -------------------
-./filebeat export dashboard -yml filebeat/module/system/module.yml
+./filebeat export dashboard --yml filebeat/module/system/module.yml
 -------------------
 
 

filebeat/beater/filebeat.go

@@ -166,7 +166,7 @@ func newBeater(b *beat.Beat, plugins PluginFactory, rawConfig *conf.C) (beat.Bea
 	}
 
 	if *once && config.ConfigInput.Enabled() && config.ConfigModules.Enabled() {
-		return nil, fmt.Errorf("input configs and -once cannot be used together")
+		return nil, fmt.Errorf("input configs and --once cannot be used together")
 	}
 
 	if config.IsInputEnabled("stdin") && len(enabledInputs) > 1 {

filebeat/cmd/generate.go

@@ -26,7 +26,6 @@ import (
 	"github.com/elastic/beats/v7/filebeat/generator/fields"
 	"github.com/elastic/beats/v7/filebeat/generator/fileset"
 	"github.com/elastic/beats/v7/filebeat/generator/module"
-	"github.com/elastic/beats/v7/libbeat/cfgfile"
 	"github.com/elastic/beats/v7/libbeat/common/cli"
 	"github.com/elastic/elastic-agent-libs/paths"
 )
@@ -64,9 +63,7 @@ func genGenerateModuleCmd() *cobra.Command {
 	}
 
 	genModuleCmd.Flags().String("modules-path", defaultHomePath, "Path to modules directory")
-	cfgfile.AddAllowedBackwardsCompatibleFlag("modules-path")
 	genModuleCmd.Flags().String("es-beats", defaultHomePath, "Path to Elastic Beats")
-	cfgfile.AddAllowedBackwardsCompatibleFlag("es-beats")
 
 	return genModuleCmd
 }
@@ -91,9 +88,7 @@ func genGenerateFilesetCmd() *cobra.Command {
 	}
 
 	genFilesetCmd.Flags().String("modules-path", defaultHomePath, "Path to modules directory")
-	cfgfile.AddAllowedBackwardsCompatibleFlag("modules-path")
 	genFilesetCmd.Flags().String("es-beats", defaultHomePath, "Path to Elastic Beats")
-	cfgfile.AddAllowedBackwardsCompatibleFlag("es-beats")
 
 	return genFilesetCmd
 }
@@ -118,9 +113,7 @@ func genGenerateFieldsCmd() *cobra.Command {
 	}
 
 	genFieldsCmd.Flags().String("es-beats", defaultHomePath, "Path to Elastic Beats")
-	cfgfile.AddAllowedBackwardsCompatibleFlag("es-beats")
 	genFieldsCmd.Flags().Bool("without-documentation", false, "Do not add description fields")
-	cfgfile.AddAllowedBackwardsCompatibleFlag("without-documentation")
 
 	return genFieldsCmd
 }

filebeat/cmd/root.go

@@ -26,7 +26,6 @@ import (
 	"github.com/elastic/beats/v7/filebeat/fileset"
 	"github.com/elastic/beats/v7/filebeat/include"
 	"github.com/elastic/beats/v7/filebeat/input"
-	"github.com/elastic/beats/v7/libbeat/cfgfile"
 	"github.com/elastic/beats/v7/libbeat/cmd"
 	"github.com/elastic/beats/v7/libbeat/cmd/instance"
 
@@ -50,9 +49,7 @@ func FilebeatSettings(moduleNameSpace string) instance.Settings {
 	}
 	runFlags := pflag.NewFlagSet(Name, pflag.ExitOnError)
 	runFlags.AddGoFlag(flag.CommandLine.Lookup("once"))
-	cfgfile.AddAllowedBackwardsCompatibleFlag("once")
 	runFlags.AddGoFlag(flag.CommandLine.Lookup("modules"))
-	cfgfile.AddAllowedBackwardsCompatibleFlag("modules")
 	return instance.Settings{
 		RunFlags:      runFlags,
 		Name:          Name,
@@ -69,10 +66,8 @@ func FilebeatSettings(moduleNameSpace string) instance.Settings {
 func Filebeat(inputs beater.PluginFactory, settings instance.Settings) *cmd.BeatsRootCmd {
 	command := cmd.GenRootCmdWithSettings(beater.New(inputs), settings)
 	command.PersistentFlags().AddGoFlag(flag.CommandLine.Lookup("M"))
-	cfgfile.AddAllowedBackwardsCompatibleFlag("M")
 	command.TestCmd.Flags().AddGoFlag(flag.CommandLine.Lookup("modules"))
 	command.SetupCmd.Flags().AddGoFlag(flag.CommandLine.Lookup("modules"))
-	cfgfile.AddAllowedBackwardsCompatibleFlag("modules")
 	command.AddCommand(cmd.GenModulesCmd(Name, "", buildModulesManager))
 	command.AddCommand(genGenerateCmd())
 	return command

filebeat/main_test.go

@@ -26,7 +26,6 @@ import (
 
 	fbcmd "github.com/elastic/beats/v7/filebeat/cmd"
 	inputs "github.com/elastic/beats/v7/filebeat/input/default-inputs"
-	"github.com/elastic/beats/v7/libbeat/cfgfile"
 	cmd "github.com/elastic/beats/v7/libbeat/cmd"
 	"github.com/elastic/beats/v7/libbeat/tests/system/template"
 )
@@ -41,14 +40,11 @@ func init() {
 	systemTest = flag.Bool("systemTest", false, "Set to true when running system tests")
 	fbCommand = fbcmd.Filebeat(inputs.Init, fbcmd.FilebeatSettings(""))
 	fbCommand.PersistentFlags().AddGoFlag(flag.CommandLine.Lookup("systemTest"))
-	cfgfile.AddAllowedBackwardsCompatibleFlag("systemTest")
 	fbCommand.PersistentFlags().AddGoFlag(flag.CommandLine.Lookup("test.coverprofile"))
-	cfgfile.AddAllowedBackwardsCompatibleFlag("test.coverprofile")
 }
 
 // Test started when the test binary is started. Only calls main.
 func TestSystem(t *testing.T) {
-	cfgfile.ConvertFlagsForBackwardsCompatibility()
 	if *systemTest {
 		if err := fbCommand.Execute(); err != nil {
 			os.Exit(1)