[entityanalytics_ad] Remove members from group information by default (#14173)

The `entityanalytics_ad.groups.member` field can be very large if the
user is a member of a large group.

We want information about groups a user belongs, but we don't need a
list of every other member of the group, so this field is no longer
populated by default.

Alongside the existing advanced option to request specific group
attributes, there is a new advanced option (off by default) to preserve
the member list.

Author: chrisberkhout

packages/entityanalytics_ad/changelog.yml

@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "0.14.1"
+  changes:
+    - description: Remove members from group information by default.
+      type: bugfix
+      link: https://github.com/elastic/integrations/pull/14173
 - version: "0.14.0"
   changes:
     - description: Remove redundant installation instructions.

packages/entityanalytics_ad/data_stream/user/_dev/test/pipeline/test-user.json-expected.json

@@ -54,7 +54,6 @@
                         "id": "S-1-5-21-372676048-1189045421-4047760665-520",
                         "instance_type": "4",
                         "is_critical_system_object": true,
-                        "member": "CN=Administrator,CN=Users,DC=testserver,DC=local",
                         "member_of": "CN=Denied RODC Password Replication Group,CN=Users,DC=testserver,DC=local",
                         "name": "Group Policy Creator Owners",
                         "object_category": "CN=Group,CN=Schema,CN=Configuration,DC=testserver,DC=local",
@@ -85,7 +84,6 @@
                         "id": "S-1-5-21-372676048-1189045421-4047760665-512",
                         "instance_type": "4",
                         "is_critical_system_object": true,
-                        "member": "CN=Administrator,CN=Users,DC=testserver,DC=local",
                         "member_of": [
                             "CN=Denied RODC Password Replication Group,CN=Users,DC=testserver,DC=local",
                             "CN=Administrators,CN=Builtin,DC=testserver,DC=local"
@@ -119,7 +117,6 @@
                         "id": "S-1-5-21-372676048-1189045421-4047760665-519",
                         "instance_type": "4",
                         "is_critical_system_object": true,
-                        "member": "CN=Administrator,CN=Users,DC=testserver,DC=local",
                         "member_of": [
                             "CN=Denied RODC Password Replication Group,CN=Users,DC=testserver,DC=local",
                             "CN=Administrators,CN=Builtin,DC=testserver,DC=local"
@@ -153,7 +150,6 @@
                         "id": "S-1-5-21-372676048-1189045421-4047760665-518",
                         "instance_type": "4",
                         "is_critical_system_object": true,
-                        "member": "CN=Administrator,CN=Users,DC=testserver,DC=local",
                         "member_of": "CN=Denied RODC Password Replication Group,CN=Users,DC=testserver,DC=local",
                         "name": "Schema Admins",
                         "object_category": "CN=Group,CN=Schema,CN=Configuration,DC=testserver,DC=local",
@@ -396,16 +392,6 @@
                         "id": "S-1-5-21-372676048-1189045421-4047760665-572",
                         "instance_type": "4",
                         "is_critical_system_object": true,
-                        "member": [
-                            "CN=Read-only Domain Controllers,CN=Users,DC=testserver,DC=local",
-                            "CN=Group Policy Creator Owners,CN=Users,DC=testserver,DC=local",
-                            "CN=Domain Admins,CN=Users,DC=testserver,DC=local",
-                            "CN=Cert Publishers,CN=Users,DC=testserver,DC=local",
-                            "CN=Enterprise Admins,CN=Users,DC=testserver,DC=local",
-                            "CN=Schema Admins,CN=Users,DC=testserver,DC=local",
-                            "CN=Domain Controllers,CN=Users,DC=testserver,DC=local",
-                            "CN=krbtgt,CN=Users,DC=testserver,DC=local"
-                        ],
                         "name": "Denied RODC Password Replication Group",
                         "object_category": "CN=Group,CN=Schema,CN=Configuration,DC=testserver,DC=local",
                         "object_class": [

packages/entityanalytics_ad/data_stream/user/agent/stream/entity-analytics.yml.hbs

@@ -25,6 +25,9 @@ tags:
 {{#if preserve_duplicate_custom_fields}}
   - preserve_duplicate_custom_fields
 {{/if}}
+{{#if preserve_group_member_list}}
+  - preserve_group_member_list
+{{/if}}
 {{#each tags as |tag|}}
   - {{tag}}
 {{/each}}

packages/entityanalytics_ad/data_stream/user/elasticsearch/ingest_pipeline/entity.yml

@@ -326,6 +326,18 @@ processors:
       value: true
       if: ctx.activedirectory?.user?.uac_list?.contains('TRUSTED_TO_AUTHENTICATE_FOR_DELEGATION') == true
 
+  - foreach:
+      tag: foreach_group
+      field: activedirectory.groups
+      if: ctx.tags?.contains('preserve_group_member_list') != true
+      ignore_missing: true
+      processor:
+        remove:
+          tag: remove_member_list_from_group
+          description: Remove the member list because of its size and irrelevance to the user
+          field: _ingest._value.member
+          ignore_missing: true
+
   - script:
       lang: painless
       ignore_failure: true

packages/entityanalytics_ad/data_stream/user/manifest.yml

@@ -112,6 +112,14 @@ streams:
         multi: true
         required: false
         show_user: false
+      - name: preserve_group_member_list
+        required: true
+        show_user: false
+        title: Preserve group member list
+        description: Unless set, the member list will be deleted from group attributes to avoid large documents.
+        type: bool
+        multi: false
+        default: false
       - name: tags
         type: text
         title: Tags

packages/entityanalytics_ad/manifest.yml

@@ -1,7 +1,7 @@
 format_version: "3.0.2"
 name: entityanalytics_ad
 title: Active Directory Entity Analytics
-version: "0.14.0"
+version: "0.14.1"
 description: "Collect User Identities from Active Directory Entity with Elastic Agent."
 type: integration
 categories: