Merge branch 'main' into dependabot/go_modules/main/windows-67a2ff958b

Author: Linu-Elias

NOTICE.txt

@@ -10879,11 +10879,11 @@ SOFTWARE
 
 --------------------------------------------------------------------------------
 Dependency : github.com/elastic/elastic-agent-libs
-Version: v0.24.0
+Version: v0.24.1
 Licence type (autodetected): Apache-2.0
 --------------------------------------------------------------------------------
 
-Contents of probable licence file $GOMODCACHE/github.com/elastic/[email protected].0/LICENSE:
+Contents of probable licence file $GOMODCACHE/github.com/elastic/[email protected].1/LICENSE:
 
                                  Apache License
                            Version 2.0, January 2004

auditbeat/module/auditd/audit_linux.go

@@ -20,16 +20,17 @@ package auditd
 import (
 	"errors"
 	"fmt"
+	"maps"
 	"os"
 	"runtime"
+	"slices"
 	"strconv"
 	"strings"
 	"sync"
 	"syscall"
 	"time"
 
 	"github.com/elastic/beats/v7/auditbeat/ab"
-	"github.com/elastic/beats/v7/libbeat/common"
 	"github.com/elastic/beats/v7/metricbeat/mb"
 	"github.com/elastic/beats/v7/metricbeat/mb/parse"
 	"github.com/elastic/elastic-agent-libs/logp"
@@ -629,13 +630,13 @@ func buildMetricbeatEvent(msgs []*auparse.AuditMessage, config Config) mb.Event
 	normalizeEventFields(auditEvent, out.RootFields)
 
 	// User set for related.user
-	var userSet common.StringSet
+	var userSet map[string]struct{}
 	if config.ResolveIDs {
-		userSet = make(common.StringSet)
+		userSet = make(map[string]struct{})
 	}
 
 	// Copy user.*/group.* fields from event
-	setECSEntity := func(key string, ent aucoalesce.ECSEntityData, root mapstr.M, set common.StringSet) {
+	setECSEntity := func(key string, ent aucoalesce.ECSEntityData, root mapstr.M, set map[string]struct{}) {
 		if ent.ID == "" && ent.Name == "" {
 			return
 		}
@@ -652,7 +653,7 @@ func buildMetricbeatEvent(msgs []*auparse.AuditMessage, config Config) mb.Event
 		if ent.Name != "" {
 			_, _ = root.Put(nameField, ent.Name)
 			if set != nil {
-				set.Add(ent.Name)
+				set[ent.Name] = struct{}{}
 			}
 		} else {
 			_ = root.Delete(nameField)
@@ -665,10 +666,9 @@ func buildMetricbeatEvent(msgs []*auparse.AuditMessage, config Config) mb.Event
 	setECSEntity("user.changes", auditEvent.ECS.User.Changes, out.RootFields, userSet)
 	setECSEntity("group", auditEvent.ECS.Group, out.RootFields, nil)
 
-	if userSet != nil {
-		if userSet.Count() != 0 {
-			_, _ = out.RootFields.Put("related.user", userSet.ToSlice())
-		}
+	if len(userSet) != 0 {
+		relatedUser := slices.Compact(slices.Sorted(maps.Keys(userSet)))
+		_, _ = out.RootFields.Put("related.user", relatedUser)
 	}
 	getStringField := func(key string, m mapstr.M) (str string) {
 		if asIf, _ := m.GetValue(key); asIf != nil {

auditbeat/module/file_integrity/action.go

@@ -19,9 +19,8 @@ package file_integrity
 
 import (
 	"math/bits"
+	"slices"
 	"strings"
-
-	"github.com/elastic/beats/v7/libbeat/common"
 )
 
 // Action is a description of the changes described by an event.
@@ -128,7 +127,9 @@ func (action Action) ECSTypes() []string {
 			list = append(list, name)
 		}
 	}
-	return common.MakeStringSet(list...).ToSlice()
+	slices.Sort(list)
+	list = slices.Compact(list)
+	return list
 }
 
 // MarshalText marshals the Action to a textual representation of itself.
@@ -211,5 +212,7 @@ func (actions ActionArray) ECSTypes() []string {
 	for _, action := range actions {
 		list = append(list, action.ECSTypes()...)
 	}
-	return common.MakeStringSet(list...).ToSlice()
+	slices.Sort(list)
+	list = slices.Compact(list)
+	return list
 }

auditbeat/scripts/docs_collector.py

@@ -50,8 +50,9 @@ def collect(base_paths):
                     versions.append(f"{key} {value}")
                 applies_to = ", ".join(versions)
             elif "release" in fields[0]:
-                if fields[0]["release"] != "ga":
-                    applies_to = fields[0]["release"]
+                applies_to = fields[0]["release"]
+            else:
+                applies_to = "ga"
 
         module_file = """---
 mapped_pages:
@@ -167,6 +168,8 @@ def collect(base_paths):
     module_list_output = """---
 mapped_pages:
   - https://www.elastic.co/guide/en/beats/auditbeat/current/auditbeat-modules.html
+applies_to:
+  stack: ga
 ---
 
 % This file is generated! See scripts/docs_collector.py
@@ -181,7 +184,7 @@ def collect(base_paths):
         title = details["title"]
         applies_to = details["applies_to"]
         module_list_output += "* [{}](/reference/auditbeat/auditbeat-module-{}.md)".format(title, m)
-        if applies_to:
+        if applies_to and applies_to is not "ga":
             module_list_output += " {{applies_to}}`stack: {}`".format(applies_to)
         module_list_output += "\n"
 

changelog/fragments/1759439225-chore-fix-defer-usage-for-stopped-status-reporting.yaml

@@ -0,0 +1,17 @@
+kind: bug-fix
+summary: fix defer usage for stopped status reporting
+component: filebeat
+
+# AUTOMATED
+# OPTIONAL to manually add other PR URLs
+# PR URL: A link the PR that added the changeset.
+# If not present is automatically filled by the tooling finding the PR where this changelog fragment has been added.
+# NOTE: the tooling supports backports, so it's able to fill the original PR number instead of the backport PR number.
+# Please provide it if you are adding a fragment for a different PR.
+# pr: https://github.com/owner/repo/1234
+
+# AUTOMATED
+# OPTIONAL to manually add other issue URLs
+# Issue URL; optional; the GitHub issue related to this changeset (either closes or is part of).
+# If not present is automatically filled by the tooling with the issue linked to the PR number.
+# issue: https://github.com/owner/repo/1234

docs/reference/auditbeat/auditbeat-module-auditd.md

@@ -1,6 +1,8 @@
 ---
 mapped_pages:
   - https://www.elastic.co/guide/en/beats/auditbeat/current/auditbeat-module-auditd.html
+applies_to:
+  stack: ga
 ---
 
 % This file is generated! See scripts/docs_collector.py

docs/reference/auditbeat/auditbeat-module-file_integrity.md

@@ -1,6 +1,8 @@
 ---
 mapped_pages:
   - https://www.elastic.co/guide/en/beats/auditbeat/current/auditbeat-module-file_integrity.html
+applies_to:
+  stack: ga
 ---
 
 % This file is generated! See scripts/docs_collector.py

docs/reference/auditbeat/auditbeat-modules.md

@@ -1,6 +1,8 @@
 ---
 mapped_pages:
   - https://www.elastic.co/guide/en/beats/auditbeat/current/auditbeat-modules.html
+applies_to:
+  stack: ga
 ---
 
 % This file is generated! See scripts/docs_collector.py

docs/reference/auditbeat/kafka-output.md

@@ -99,7 +99,21 @@ To use `GSSAPI` mechanism to authenticate with Kerberos, you must leave this fie
 
 The Kafka topic used for produced events.
 
-You can set the topic dynamically by using a format string to access any event field. For example, this configuration uses a custom field, `fields.log_topic`, to set the topic for each event:
+You can set a static topic, for example `auditbeat`, or you can use a format string to set a topic dynamically based on one or more [Elastic Common Schema (ECS)](ecs://reference/index.md) fields. Available fields include:
+
+* `data_stream.type`
+* `data_stream.dataset`
+* `data_stream.namespace`
+* `@timestamp`
+* `event.dataset`
+
+For example:
+
+```yaml
+topic: '%{[data_stream.type]}-%{[data_stream.dataset]}-%{[data_stream.namespace]}'
+```
+
+You can also set a custom field. This is useful if you need to construct a more complex or structured topic name. For example, this configuration uses the `fields.log_topic` custom field to set the topic for each event:
 
 ```yaml
 topic: '%{[fields.log_topic]}'
@@ -109,6 +123,19 @@ topic: '%{[fields.log_topic]}'
 To learn how to add custom fields to events, see the [`fields`](/reference/auditbeat/configuration-general-options.md#libbeat-configuration-fields) option.
 ::::
 
+To set a dynamic topic value for outputting {{auditbeat}} data to Kafka, you can add the [`add_fields` processor](/reference/auditbeat/add-fields.md) to {{auditbeat}}'s input configuration settings.
+    
+For example, the following `add_fields` processor creates a dynamic topic value for the `fields.log_topic` field by combining multiple [ECS data stream fields](ecs://reference/ecs-data_stream.md):
+
+  ```yaml
+  - add_fields:
+      target: ''
+      fields:
+        log_topic: '%{[data_stream.type]}-%{[data_stream.dataset]}-%{[data_stream.namespace]}' <1>
+  ```
+  1. Depending on the values of the data stream fields, this generates topic names such as `logs-nginx.access-production` or `metrics-system.cpu-staging` as the value of the custom `log_topic` field.
+
+  For more information, refer to [Filter and enhance data with processors](/reference/auditbeat/filtering-enhancing-data.md).
 
 See the [`topics`](#topics-option-kafka) setting for other ways to set the topic dynamically.
 

docs/reference/filebeat/kafka-output.md

@@ -99,7 +99,21 @@ To use `GSSAPI` mechanism to authenticate with Kerberos, you must leave this fie
 
 The Kafka topic used for produced events.
 
-You can set the topic dynamically by using a format string to access any event field. For example, this configuration uses a custom field, `fields.log_topic`, to set the topic for each event:
+You can set a static topic, for example `filebeat`, or you can use a format string to set a topic dynamically based on one or more [Elastic Common Schema (ECS)](ecs://reference/index.md) fields. Available fields include:
+
+* `data_stream.type`
+* `data_stream.dataset`
+* `data_stream.namespace`
+* `@timestamp`
+* `event.dataset`
+
+For example:
+
+```yaml
+topic: '%{[data_stream.type]}-%{[data_stream.dataset]}-%{[data_stream.namespace]}'
+```
+
+You can also set a custom field. This is useful if you need to construct a more complex or structured topic name. For example, this configuration uses the `fields.log_topic` custom field to set the topic for each event:
 
 ```yaml
 topic: '%{[fields.log_topic]}'
@@ -109,6 +123,19 @@ topic: '%{[fields.log_topic]}'
 To learn how to add custom fields to events, see the [`fields`](/reference/filebeat/configuration-general-options.md#libbeat-configuration-fields) option.
 ::::
 
+To set a dynamic topic value for outputting {{filebeat}} data to Kafka, you can add the [`add_fields` processor](/reference/filebeat/add-fields.md) to {{filebeat}}'s input configuration settings.
+    
+For example, the following `add_fields` processor creates a dynamic topic value for the `fields.log_topic` field by combining multiple [ECS data stream fields](ecs://reference/ecs-data_stream.md):
+
+  ```yaml
+  - add_fields:
+      target: ''
+      fields:
+        log_topic: '%{[data_stream.type]}-%{[data_stream.dataset]}-%{[data_stream.namespace]}' <1>
+  ```
+  1. Depending on the values of the data stream fields, this generates topic names such as `logs-nginx.access-production` or `metrics-system.cpu-staging` as the value of the custom `log_topic` field.
+
+  For more information, refer to [Filter and enhance data with processors](/reference/filebeat/filtering-enhancing-data.md).
 
 See the [`topics`](#topics-option-kafka) setting for other ways to set the topic dynamically.