x-pack/filebeat/entityanalytics/provider/activedirectory: allow attri…

…bute selection (elastic#40662)

In cases where there are large numbers of users, the complete set of
results may be unreasonably large. This change allows a configuration to
be tuned to only include attributes that are required by the user.

CHANGELOG.next.asciidoc

@@ -294,6 +294,7 @@ https://github.com/elastic/beats/compare/v8.8.1\...main[Check the HEAD diff]
 - Document `winlog` input. {issue}40074[40074] {pull}40462[40462]
 - Added retry logic to websocket connections in the streaming input. {issue}40271[40271] {pull}40601[40601]
 - Disable event normalization for netflow input {pull}40635[40635]
+- Allow attribute selection in the Active Directory entity analytics provider. {issue}40482[40482] {pull}40662[40662]
 
 *Auditbeat*
 

x-pack/filebeat/docs/inputs/input-entity-analytics.asciidoc

@@ -190,6 +190,20 @@ The Active Directory Base Distinguished Name. Field is required.
 
 The client user name. Used for authentication. The user must have Active Directory read access. Field is required.
 
+[float]
+===== `user_attributes`
+
+The set of directory attributes to request from the Active Directory server when collecting user data.
+If not set, all user attributes are requested. If set, only listed attributes are requested, including
+`distinguishedDomain` and `whenChanged`. Note that the Active Directory attribute names are used.
+
+[float]
+===== `group_attributes`
+
+The set of directory attributes to request from the Active Directory server when collecting group data.
+If not set, all group attributes are requested. If set, only listed attributes are requested, including
+`distinguishedDomain` and `whenChanged`. Note that the Active Directory attribute names are used.
+
 [float]
 ===== `ad_paging_size`
 

x-pack/filebeat/input/entityanalytics/provider/activedirectory/activedirectory.go

@@ -25,7 +25,6 @@ import (
 	"github.com/elastic/elastic-agent-libs/config"
 	"github.com/elastic/elastic-agent-libs/logp"
 	"github.com/elastic/elastic-agent-libs/mapstr"
-	"github.com/elastic/elastic-agent-libs/transport/httpcommon"
 	"github.com/elastic/elastic-agent-libs/transport/tlscommon"
 	"github.com/elastic/go-concert/ctxtool"
 )
@@ -128,6 +127,9 @@ func (p *adInput) Run(inputCtx v2.Context, store *kvstore.Store, client beat.Cli
 	syncTimer := time.NewTimer(syncWaitTime)
 	updateTimer := time.NewTimer(updateWaitTime)
 
+	p.cfg.UserAttrs = withMandatory(p.cfg.UserAttrs, "distinguishedName", "whenChanged")
+	p.cfg.GrpAttrs = withMandatory(p.cfg.GrpAttrs, "distinguishedName", "whenChanged")
+
 	for {
 		select {
 		case <-inputCtx.Cancelation.Done():
@@ -169,13 +171,21 @@ func (p *adInput) Run(inputCtx v2.Context, store *kvstore.Store, client beat.Cli
 	}
 }
 
-// clientOption returns constructed client configuration options, including
-// setting up http+unix and http+npipe transports if requested.
-func clientOptions(keepalive httpcommon.WithKeepaliveSettings) []httpcommon.TransportOption {
-	return []httpcommon.TransportOption{
-		httpcommon.WithAPMHTTPInstrumentation(),
-		keepalive,
+// withMandatory adds the required attribute names to attr unless attr is empty.
+func withMandatory(attr []string, include ...string) []string {
+	if len(attr) == 0 {
+		return nil
+	}
+outer:
+	for _, m := range include {
+		for _, a := range attr {
+			if m == a {
+				continue outer
+			}
+		}
+		attr = append(attr, m)
 	}
+	return attr
 }
 
 // runFullSync performs a full synchronization. It will fetch user and group
@@ -316,7 +326,7 @@ func (p *adInput) doFetchUsers(ctx context.Context, state *stateStore, fullSync
 		since = state.whenChanged
 	}
 
-	entries, err := activedirectory.GetDetails(p.cfg.URL, p.cfg.User, p.cfg.Password, p.baseDN, since, p.cfg.PagingSize, nil, p.tlsConfig)
+	entries, err := activedirectory.GetDetails(p.cfg.URL, p.cfg.User, p.cfg.Password, p.baseDN, since, p.cfg.UserAttrs, p.cfg.GrpAttrs, p.cfg.PagingSize, nil, p.tlsConfig)
 	p.logger.Debugf("received %d users from API", len(entries))
 	if err != nil {
 		return nil, err

x-pack/filebeat/input/entityanalytics/provider/activedirectory/conf.go

@@ -31,6 +31,9 @@ type conf struct {
 	User     string `config:"ad_user" validate:"required"`
 	Password string `config:"ad_password" validate:"required"`
 
+	UserAttrs []string `config:"user_attributes"`
+	GrpAttrs  []string `config:"group_attributes"`
+
 	PagingSize uint32 `config:"ad_paging_size"`
 
 	// SyncInterval is the time between full

x-pack/filebeat/input/entityanalytics/provider/activedirectory/internal/activedirectory/activedirectory.go

@@ -39,7 +39,7 @@ type Entry struct {
 // only records with whenChanged since that time will be returned. since is
 // expected to be configured in a time zone the Active Directory server will
 // understand, most likely UTC.
-func GetDetails(url, user, pass string, base *ldap.DN, since time.Time, pagingSize uint32, dialer *net.Dialer, tlsconfig *tls.Config) ([]Entry, error) {
+func GetDetails(url, user, pass string, base *ldap.DN, since time.Time, userAttrs, grpAttrs []string, pagingSize uint32, dialer *net.Dialer, tlsconfig *tls.Config) ([]Entry, error) {
 	if base == nil || len(base.RDNs) == 0 {
 		return nil, fmt.Errorf("%w: no path", ErrInvalidDistinguishedName)
 	}
@@ -76,7 +76,7 @@ func GetDetails(url, user, pass string, base *ldap.DN, since time.Time, pagingSi
 	// Get groups in the directory. Get all groups independent of the
 	// since parameter as they may not have changed for changed users.
 	var groups directory
-	grps, err := search(conn, baseDN, "(objectClass=group)", pagingSize)
+	grps, err := search(conn, baseDN, "(objectClass=group)", grpAttrs, pagingSize)
 	if err != nil {
 		// Allow continuation if groups query fails, but warn.
 		errs = []error{fmt.Errorf("%w: %w", ErrGroups, err)}
@@ -90,7 +90,7 @@ func GetDetails(url, user, pass string, base *ldap.DN, since time.Time, pagingSi
 	if sinceFmtd != "" {
 		userFilter = "(&(objectClass=user)(whenChanged>=" + sinceFmtd + "))"
 	}
-	usrs, err := search(conn, baseDN, userFilter, pagingSize)
+	usrs, err := search(conn, baseDN, userFilter, userAttrs, pagingSize)
 	if err != nil {
 		errs = append(errs, fmt.Errorf("%w: %w", ErrUsers, err))
 		return nil, errors.Join(errs...)
@@ -100,7 +100,7 @@ func GetDetails(url, user, pass string, base *ldap.DN, since time.Time, pagingSi
 
 	// Also collect users that are members of groups that have changed.
 	if sinceFmtd != "" {
-		grps, err := search(conn, baseDN, "(&(objectClass=groups)(whenChanged>="+sinceFmtd+"))", pagingSize)
+		grps, err := search(conn, baseDN, "(&(objectClass=groups)(whenChanged>="+sinceFmtd+"))", grpAttrs, pagingSize)
 		if err != nil {
 			// Allow continuation if groups query fails, but warn.
 			errs = append(errs, fmt.Errorf("failed to collect changed groups: %w: %w", ErrGroups, err))
@@ -121,7 +121,7 @@ func GetDetails(url, user, pass string, base *ldap.DN, since time.Time, pagingSi
 					modGrps[i] = "(memberOf=" + u + ")"
 				}
 				query := "(&(objectClass=user)(|" + strings.Join(modGrps, "") + ")"
-				usrs, err := search(conn, baseDN, query, pagingSize)
+				usrs, err := search(conn, baseDN, query, userAttrs, pagingSize)
 				if err != nil {
 					errs = append(errs, fmt.Errorf("failed to collect users of changed groups%w: %w", ErrUsers, err))
 				} else {
@@ -176,7 +176,7 @@ func whenChanged(user map[string]any, groups []any) time.Time {
 // search performs an LDAP filter search on conn at the LDAP base. If paging
 // is non-zero, page sizing will be used. See [ldap.Conn.SearchWithPaging] for
 // details.
-func search(conn *ldap.Conn, base, filter string, pagingSize uint32) (*ldap.SearchResult, error) {
+func search(conn *ldap.Conn, base, filter string, attrs []string, pagingSize uint32) (*ldap.SearchResult, error) {
 	srch := &ldap.SearchRequest{
 		BaseDN:       base,
 		Scope:        ldap.ScopeWholeSubtree,
@@ -185,7 +185,7 @@ func search(conn *ldap.Conn, base, filter string, pagingSize uint32) (*ldap.Sear
 		TimeLimit:    0,
 		TypesOnly:    false,
 		Filter:       filter,
-		Attributes:   nil,
+		Attributes:   attrs,
 		Controls:     nil,
 	}
 	if pagingSize != 0 {

x-pack/filebeat/input/entityanalytics/provider/activedirectory/internal/activedirectory/activedirectory_test.go

@@ -45,7 +45,7 @@ func Test(t *testing.T) {
 
 	var times []time.Time
 	t.Run("full", func(t *testing.T) {
-		users, err := GetDetails(url, user, pass, base, time.Time{}, 0, nil, nil)
+		users, err := GetDetails(url, user, pass, base, time.Time{}, nil, nil, 0, nil, nil)
 		if err != nil {
 			t.Fatalf("unexpected error from GetDetails: %v", err)
 		}
@@ -91,7 +91,7 @@ func Test(t *testing.T) {
 				want++
 			}
 		}
-		users, err := GetDetails(url, user, pass, base, since, 0, nil, nil)
+		users, err := GetDetails(url, user, pass, base, since, nil, nil, 0, nil, nil)
 		if err != nil {
 			t.Fatalf("unexpected error from GetDetails: %v", err)
 		}