[m365_defender] Unique entity_id values

packages/m365_defender/changelog.yml

@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "5.13.1"
+  changes:
+    - description: Unique `entity_id` values to avoid irrelevant documents in the Event Analyzer.
+      type: bugfix
+      link: https://github.com/elastic/integrations/pull/18891
 - version: "5.13.0"
   changes:
     - description: Add timestamp range filter and storage tier exclusion to latest transform source queries to reduce scan scope and improve performance.

packages/m365_defender/data_stream/alert/_dev/test/pipeline/test-alert.log-expected.json

@@ -203,7 +203,7 @@
                     "powershell.exe  -NoExit -ExecutionPolicy Bypass -WindowStyle Hidden $ErrorActionPreference= 'silentlycontinue';(New-Object System.Net.WebClient).DownloadFile('http://127.0.0.1/1.exe', 'C:\\\\test-WDATP-test\\\\invoice.exe');Start-Process 'C:\\\\test-WDATP-test\\\\invoice.exe'"
                 ],
                 "entity_id": [
-                    "8224"
+                    "rVnodapfrTDpseiB0HSKS88ozVs="
                 ],
                 "executable": "C:\\Windows\\System32\\WindowsPowerShell\\v1.0\\powershell.exe",
                 "hash": {
@@ -219,7 +219,7 @@
                 ],
                 "parent": {
                     "entity_id": [
-                        "5772"
+                        "zogwXHGFe3FclpZDb7EC8VkjELw="
                     ],
                     "executable": "C:\\Windows\\System32\\cmd.exe",
                     "pid": [
@@ -508,7 +508,7 @@
                     "powershell.exe  -NoExit -ExecutionPolicy Bypass -WindowStyle Hidden $ErrorActionPreference= 'silentlycontinue';(New-Object System.Net.WebClient).DownloadFile('http://127.0.0.1/1.exe', 'C:\\\\test-WDATP-test\\\\invoice.exe');Start-Process 'C:\\\\test-WDATP-test\\\\invoice.exe'"
                 ],
                 "entity_id": [
-                    "8224"
+                    "mEduHGao/Bxr3ydQddVjif+e5JU="
                 ],
                 "executable": "C:\\Windows\\System32\\WindowsPowerShell\\v1.0\\powershell.exe",
                 "hash": {
@@ -524,7 +524,7 @@
                 ],
                 "parent": {
                     "entity_id": [
-                        "5772"
+                        "q+HpjZ9DnrUoFj5nLaWFfIk08TY="
                     ],
                     "executable": "C:\\Windows\\System32\\cmd.exe",
                     "pid": [
@@ -819,7 +819,7 @@
                     "powershell.exe  -NoExit -ExecutionPolicy Bypass -WindowStyle Hidden $ErrorActionPreference= 'silentlycontinue';(New-Object System.Net.WebClient).DownloadFile('http://127.0.0.1/1.exe', 'C:\\\\test-WDATP-test\\\\invoice.exe');Start-Process 'C:\\\\test-WDATP-test\\\\invoice.exe'"
                 ],
                 "entity_id": [
-                    "8224"
+                    "mEduHGao/Bxr3ydQddVjif+e5JU="
                 ],
                 "executable": "C:\\Windows\\System32\\WindowsPowerShell\\v1.0\\powershell.exe",
                 "hash": {
@@ -835,7 +835,7 @@
                 ],
                 "parent": {
                     "entity_id": [
-                        "5772"
+                        "q+HpjZ9DnrUoFj5nLaWFfIk08TY="
                     ],
                     "executable": "C:\\Windows\\System32\\cmd.exe",
                     "pid": [
@@ -1126,7 +1126,7 @@
                     "powershell.exe  -NoExit -ExecutionPolicy Bypass -WindowStyle Hidden $ErrorActionPreference= 'silentlycontinue';(New-Object System.Net.WebClient).DownloadFile('http://127.0.0.1/1.exe', 'C:\\\\test-WDATP-test\\\\invoice.exe');Start-Process 'C:\\\\test-WDATP-test\\\\invoice.exe'"
                 ],
                 "entity_id": [
-                    "8224"
+                    "mEduHGao/Bxr3ydQddVjif+e5JU="
                 ],
                 "executable": "C:\\Windows\\System32\\WindowsPowerShell\\v1.0\\powershell.exe",
                 "hash": {
@@ -1142,7 +1142,7 @@
                 ],
                 "parent": {
                     "entity_id": [
-                        "5772"
+                        "q+HpjZ9DnrUoFj5nLaWFfIk08TY="
                     ],
                     "executable": "C:\\Windows\\System32\\cmd.exe",
                     "pid": [

packages/m365_defender/data_stream/alert/elasticsearch/ingest_pipeline/default.yml

@@ -2373,6 +2373,8 @@ processors:
         def fileSize = new HashSet();
         def processPid = new HashSet();
         def processParentPid = new HashSet();
+        def processEntityId = new HashSet();
+        def processParentEntityId = new HashSet();
 
         for (evidence in ctx.json.evidence) {
           maybeAddExecutable(processExecutable, evidence?.image_file);
@@ -2392,6 +2394,12 @@ processors:
             if (evidence?.parent_process?.id != null) {
               processParentPid.add(evidence.parent_process.id);
             }
+            if (evidence?.process?.id != null && evidence?.process?.creation_datetime != null && evidence?.mde_device_id != null) {
+              processEntityId.add(evidence.process.id + '|' + evidence.process.creation_datetime + '|' + evidence.mde_device_id);
+            }
+            if (evidence?.parent_process?.id != null && evidence?.parent_process?.creation_datetime != null && evidence?.mde_device_id != null) {
+              processParentEntityId.add(evidence.parent_process.id + '|' + evidence.parent_process.creation_datetime + '|' + evidence.mde_device_id);
+            }
           }
         }
 
@@ -2408,6 +2416,12 @@ processors:
         if (!processParentPid.isEmpty()) {
           ctx.process.parent.pid = convertToOrderedArray(processParentPid);
         }
+        if (!processEntityId.isEmpty()) {
+          ctx.process.entity_id = convertToOrderedArray(processEntityId);
+        }
+        if (!processParentEntityId.isEmpty()) {
+          ctx.process.parent.entity_id = convertToOrderedArray(processParentEntityId);
+        }
         if (!processExecutable.isEmpty()) {
           def execList = new ArrayList(processExecutable);
           if (execList.size() == 1) {
@@ -2641,25 +2655,27 @@ processors:
           ctx.process.user.name = convertToOrderedArray(processUserName);
         }
   - foreach:
-      field: process.pid
-      tag: foreach_process_pid
-      if: ctx.process?.pid instanceof List
+      field: process.parent.entity_id
+      tag: foreach_fingerprint_parent_process_entity_id
+      if: ctx.process?.parent?.entity_id instanceof List
       processor:
-        append:
-          field: process.entity_id
-          tag: append_process_pid_to_entity_id
-          value: '{{{_ingest._value}}}'
-          allow_duplicates: false
+        fingerprint:
+          fields:
+            - _ingest._value
+          tag: fingerprint_parent_process_entity_id
+          target_field: _ingest._value
+          ignore_missing: true
   - foreach:
-      field: process.parent.pid
-      tag: foreach_process_parent_pid
-      if: ctx.process?.parent?.pid instanceof List
+      field: process.entity_id
+      tag: foreach_fingerprint_process_entity_id
+      if: ctx.process?.entity_id instanceof List
       processor:
-        append:
-          field: process.parent.entity_id
-          tag: append_process_parent_pid_to_entity_id
-          value: '{{{_ingest._value}}}'
-          allow_duplicates: false
+        fingerprint:
+          fields:
+            - _ingest._value
+          tag: fingerprint_process_entity_id
+          target_field: _ingest._value
+          ignore_missing: true
   - foreach:
       field: json.evidence
       tag: foreach_evidence_remove_fields

packages/m365_defender/data_stream/event/_dev/test/pipeline/test-alert.log-expected.json

@@ -473,7 +473,7 @@
                 ],
                 "args_count": 6,
                 "command_line": "tar -df alternatives.tar.0 -C /var/lib/dpkg alternatives",
-                "entity_id": "216032",
+                "entity_id": "vhRr+4jfPR2WxTNPulnM8uREqLI=",
                 "executable": "/usr/bin/tar",
                 "hash": {
                     "md5": "a2952b0188d52b53bac2707cd465499d",
@@ -491,7 +491,7 @@
                         "status": "Unknown"
                     },
                     "command_line": "/bin/sh /usr/libexec/dpkg/dpkg-db-backup",
-                    "entity_id": "216032",
+                    "entity_id": "zLqW1HNhmlwmllqL+BPGhS+UsZQ=",
                     "executable": "/usr/bin/dash",
                     "group_leader": {
                         "name": "dash",
@@ -734,7 +734,7 @@
                 ],
                 "args_count": 3,
                 "command_line": "python -m detection_rules",
-                "entity_id": "225349",
+                "entity_id": "pnXgNUR2vwGuWbfUhjvis4VtYGE=",
                 "executable": "/usr/bin/python3.12",
                 "hash": {
                     "md5": "4c1306feb059d94fc543e5606b99f9d0",
@@ -751,7 +751,7 @@
                         "status": "Unknown"
                     },
                     "command_line": "-zsh",
-                    "entity_id": "225349",
+                    "entity_id": "EvLA7kdPybXD6PQLtK5vPvyzBVk=",
                     "executable": "/usr/bin/zsh",
                     "group_leader": {
                         "name": "zsh",

packages/m365_defender/data_stream/event/_dev/test/pipeline/test-cloud.log-expected.json

@@ -182,11 +182,9 @@
                 ],
                 "args_count": 7,
                 "command_line": "nginx: master process nginx -g daemon off;",
-                "entity_id": "1",
                 "executable": "/usr/sbin",
                 "name": "nginx",
                 "parent": {
-                    "entity_id": "100",
                     "name": "containerd-shim",
                     "pid": 100
                 },