digital_guardian: fix mapping of dg_alert.alert_wb field and add export profile guid

packages/digital_guardian/changelog.yml

@@ -1,4 +1,15 @@
 # newer versions go on top
+- version: "1.4.0"
+  changes:
+    - description: Add export profile GUID to documents.
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/12818
+    - description: Fix mapping type of `dg_alert.alert_wb`.
+      type: bugfix
+      link: https://github.com/elastic/integrations/pull/12818
+    - description: Fix dot expansion.
+      type: bugfix
+      link: https://github.com/elastic/integrations/pull/12818
 - version: "1.3.2"
   changes:
     - description: Updated SSL description to be uniform and to include links to documentation.

packages/digital_guardian/data_stream/arc/_dev/test/pipeline/test-dg-arc.log

@@ -1,3 +1,4 @@
 {"dg_comment":"-","dg_description":"This file outlook.exe was going to [demo.digitalg@gmail.com]","dg_guid":"1dc3c1fa-5474-4fc0-a7c3-74ff42d28e5e","dg_name":"test has attached a Salesforce data to an email","dg_tenant":"279b59f3-02f3-44ea-a7c3-9bac2eb0224d","dg_utype":"Incident","inc_assign":"test@dgdemo","inc_creator":"dg","inc_id":"230523-WIQHA","inc_mtime":"2023-05-23 06:56:39","inc_sev":"Critical","inc_state":"Created"}
 {"dg_comment":"-","dg_description":"-","dg_guid":"c742c377-b429-428a-b0c9-515cbbf143be","dg_name":"Demo 10","dg_tenant":"279b59f3-02f3-44ea-a7c3-9bac2eb0224d","dg_utype":"Incident","inc_assign":"demo@dgdemo","inc_creator":"demo@dgdemo","inc_id":"230523-RG0AB","inc_mtime":"2023-05-23 11:53:11","inc_sev":"Critical","inc_state":"Escalated"}
 {"dg_comment":"-","dg_description":"-","dg_guid":"c742c377-b429-428a-b0c9-515cbbf143be","dg_name":"Demo 10","dg_tenant":"279b59f3-02f3-44ea-a7c3-9bac2eb0224d","dg_utype":"Incident","inc_assign":"demo@dgdemo","inc_creator":"demo@dgdemo","inc_id":"230523-RG0AB","inc_mtime":"2023-05-23 11:53:11","inc_sev":"Critical","inc_state":"Escalated","dg_time":"2024-11-05 07:20:41 PM","dg_processed_time":1730834913309,"dg_local_timestamp":"2024-11-05 02:20:41 PM","pi_fal":"2024-11-04 02:32:20 PM","pi_fcl":"2024-06-20 12:53:34 PM","pi_fml":"2024-11-04 09:37:26 AM","dg_attachments.dg_file_size":"1.8 MB","dg_file_size":"10.4 KB"}
+{"dg_alert.alert_al":"High","dg_alert.alert_at":"Prompt","dg_alert.alert_bc":"User Decision","dg_alert.alert_did":"-","dg_alert.alert_etl":"2025-01-22 02:09:02 PM","dg_alert.alert_etu":"2025-01-22 07:09:02 PM","dg_alert.alert_ur":"Photo for team activity","dg_alert.alert_wb":"No","dg_alert.dg_category_name":"0 _MacPRD:AllComputerPol","dg_alert.dg_detection_source":"Alert","dg_alert.dg_name":"prompt-justify external file uploads (Mac)","dg_alert.dg_policy.dg_category_name":".01 MAC Prod Global","dg_alert.dg_policy.dg_name":"__Mac All Agents","dg_alert.dg_rule_action_type":"Prompt"}

packages/digital_guardian/data_stream/arc/_dev/test/pipeline/test-dg-arc.log-expected.json

@@ -90,7 +90,10 @@
             "@timestamp": "2024-11-05T19:20:41.000Z",
             "digital_guardian": {
                 "arc": {
-                    "dg_attachments.dg_file_size": "1.8 MB",
+                    "dg_attachments": {
+                        "dg_file_size": "1.8 MB",
+                        "dg_file_size_bytes": 1800000
+                    },
                     "dg_file_size": "10.4 KB",
                     "dg_file_size_bytes": 10400,
                     "dg_guid": "c742c377-b429-428a-b0c9-515cbbf143be",
@@ -136,6 +139,40 @@
             "user": {
                 "name": "demo@dgdemo"
             }
+        },
+        {
+            "digital_guardian": {
+                "arc": {
+                    "dg_alert": {
+                        "alert_al": "High",
+                        "alert_at": "Prompt",
+                        "alert_bc": "User Decision",
+                        "alert_etl": "2025-01-22T14:09:02.000Z",
+                        "alert_etu": "2025-01-22T19:09:02.000Z",
+                        "alert_ur": "Photo for team activity",
+                        "alert_wb": "No",
+                        "dg_category_name": "0 _MacPRD:AllComputerPol",
+                        "dg_detection_source": "Alert",
+                        "dg_name": "prompt-justify external file uploads (Mac)",
+                        "dg_policy": {
+                            "dg_category_name": ".01 MAC Prod Global",
+                            "dg_name": "__Mac All Agents"
+                        },
+                        "dg_rule_action_type": "Prompt"
+                    }
+                }
+            },
+            "ecs": {
+                "version": "8.11.0"
+            },
+            "event": {
+                "kind": "alert",
+                "original": "{\"dg_alert.alert_al\":\"High\",\"dg_alert.alert_at\":\"Prompt\",\"dg_alert.alert_bc\":\"User Decision\",\"dg_alert.alert_did\":\"-\",\"dg_alert.alert_etl\":\"2025-01-22 02:09:02 PM\",\"dg_alert.alert_etu\":\"2025-01-22 07:09:02 PM\",\"dg_alert.alert_ur\":\"Photo for team activity\",\"dg_alert.alert_wb\":\"No\",\"dg_alert.dg_category_name\":\"0 _MacPRD:AllComputerPol\",\"dg_alert.dg_detection_source\":\"Alert\",\"dg_alert.dg_name\":\"prompt-justify external file uploads (Mac)\",\"dg_alert.dg_policy.dg_category_name\":\".01 MAC Prod Global\",\"dg_alert.dg_policy.dg_name\":\"__Mac All Agents\",\"dg_alert.dg_rule_action_type\":\"Prompt\"}"
+            },
+            "tags": [
+                "preserve_original_event",
+                "preserve_duplicate_custom_fields"
+            ]
         }
     ]
 }

packages/digital_guardian/data_stream/arc/agent/stream/cel.yml.hbs

@@ -19,12 +19,14 @@ auth.oauth2:
   client.secret: {{client_secret}}
   token_url: {{auth_server_url}}/as/token.oauth2
   scopes: {{scope}}
+state:
+  export_profile: {{export_profile}}
 redact:
   fields: ~
 program: |
   state.with(
     request("POST",
-      state.url + "/rest/1.0/export_profiles/{{export_profile}}/export_and_ack"
+      state.url.trim_right("/") + "/rest/1.0/export_profiles/" + state.export_profile + "/export_and_ack"
     ).with({
       "Header":{
           "Accept": ["application/json"],
@@ -34,7 +36,7 @@ program: |
         (has(body.fields) && has(body.data) ? 
           body.fields.map(e, e.name).as(field_names, {
             "events": body.data.map(d, zip(field_names, d).as(e, {
-              "message": e.encode_json(),
+              "message": e.with({"export_profile": state.export_profile}).encode_json(),
             }))
           })
         : 

packages/digital_guardian/data_stream/arc/elasticsearch/ingest_pipeline/default.yml

@@ -31,7 +31,8 @@ processors:
             field: error.message
             value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}'
   - dot_expander:
-      field: digital_guardian.arc
+      field: "*"
+      path: digital_guardian.arc
   - script:
       description: Drops null/empty/na values recursively.
       tag: script_to_remove_na_values
@@ -189,6 +190,24 @@ processors:
         - "yyyy-MM-dd HH:mm:ss"
         - "ISO8601"
       if: ctx.digital_guardian?.arc?.inc_mtime != null
+  - date:
+      field: digital_guardian.arc.dg_alert.alert_etl
+      tag: date_dg_alert_alert_etl
+      target_field: digital_guardian.arc.dg_alert.alert_etl
+      formats:
+        - "yyyy-MM-dd hh:mm:ss a"
+        - "yyyy-MM-dd HH:mm:ss"
+        - "ISO8601"
+      if: ctx.digital_guardian?.arc?.dg_alert?.alert_etl != null
+  - date:
+      field: digital_guardian.arc.dg_alert.alert_etu
+      tag: date_dg_alert_alert_etu
+      target_field: digital_guardian.arc.dg_alert.alert_etu
+      formats:
+        - "yyyy-MM-dd hh:mm:ss a"
+        - "yyyy-MM-dd HH:mm:ss"
+        - "ISO8601"
+      if: ctx.digital_guardian?.arc?.dg_alert?.alert_etu != null
 
 # Choose a @timestamp value for the event
   - set:

packages/digital_guardian/data_stream/arc/fields/fields.yml

@@ -1,18 +1,48 @@
 - name: digital_guardian.arc
   type: group
   fields:
+    - name: dg_alert.alert_al
+      type: keyword
+      description: Alert AL
+    - name: dg_alert.alert_at
+      type: keyword
+      description: Alert AT
+    - name: dg_alert.alert_bc
+      type: keyword
+      description: Alert BC
     - name: dg_alert.alert_did
       type: keyword
       description: Alert DID
+    - name: dg_alert.alert_etl
+      type: keyword
+      description: Alert ETL
+    - name: dg_alert.alert_etu
+      type: date
+      description: Alert ETU
+    - name: dg_alert.alert_ur
+      type: date
+      description: Alert UR
     - name: dg_alert.alert_wb
-      type: integer
+      type: keyword
       description: Alert WB
+    - name: dg_alert.dg_detection_source
+      type: keyword
+      description: Alert Detection Source
     - name: dg_alert.dg_category_name
       type: keyword
       description: Alert Category Name
+    - name: dg_alert.dg_name
+      type: keyword
+      description: Alert Name
+    - name: dg_alert.dg_policy.dg_category_name
+      type: keyword
+      description: Alert Policy Category Name
     - name: dg_alert.dg_policy.dg_name
       type: keyword
       description: Alert Policy Name
+    - name: dg_alert.dg_rule_action_type
+      type: keyword
+      description: Alert Rule Action Type
     - name: dg_attachments.dg_file_size_bytes
       type: long
       description: File Size in Bytes
@@ -73,6 +103,9 @@
     - name: dg_utype
       type: keyword
       description: Operation Type
+    - name: export_profile
+      type: keyword
+      description: Export Profile GUID for the Event
     - name: inc_assign
       type: keyword
       description: Incident Assignee

packages/digital_guardian/data_stream/arc/sample_event.json

@@ -1,15 +1,15 @@
 {
-    "@timestamp": "2023-05-23T06:56:39.000Z",
+    "@timestamp": "2025-02-18T04:00:28.647Z",
     "agent": {
-        "ephemeral_id": "bc19c27a-7a31-4b0c-b04b-b3be2ab95a02",
-        "id": "1edfb948-2ef5-4b96-8747-225d782bb6dd",
-        "name": "docker-fleet-agent",
+        "ephemeral_id": "3d727e8f-6944-41c1-a55a-dd22db00d883",
+        "id": "8ae590fa-6e28-49e6-9e43-f64705ab4e6b",
+        "name": "elastic-agent-15774",
         "type": "filebeat",
         "version": "8.13.0"
     },
     "data_stream": {
         "dataset": "digital_guardian.arc",
-        "namespace": "19912",
+        "namespace": "94938",
         "type": "logs"
     },
     "digital_guardian": {
@@ -19,6 +19,7 @@
             "dg_name": "test has attached a Salesforce data to an email",
             "dg_tenant": "279b59f3-02f3-44ea-a7c3-9bac2eb0224d",
             "dg_utype": "Incident",
+            "export_profile": "abc123",
             "inc_assign": "test@dgdemo",
             "inc_creator": "dg",
             "inc_id": "230523-WIQHA",
@@ -31,7 +32,7 @@
         "version": "8.11.0"
     },
     "elastic_agent": {
-        "id": "1edfb948-2ef5-4b96-8747-225d782bb6dd",
+        "id": "8ae590fa-6e28-49e6-9e43-f64705ab4e6b",
         "snapshot": false,
         "version": "8.13.0"
     },
@@ -40,9 +41,9 @@
         "agent_id_status": "verified",
         "dataset": "digital_guardian.arc",
         "id": "1dc3c1fa-5474-4fc0-a7c3-74ff42d28e5e",
-        "ingested": "2024-07-30T15:23:06Z",
+        "ingested": "2025-02-18T04:00:31Z",
         "kind": "alert",
-        "original": "{\"dg_comment\":\"-\",\"dg_description\":\"This file outlook.exe was going to [demo.digitalg@gmail.com]\",\"dg_guid\":\"1dc3c1fa-5474-4fc0-a7c3-74ff42d28e5e\",\"dg_name\":\"test has attached a Salesforce data to an email\",\"dg_tenant\":\"279b59f3-02f3-44ea-a7c3-9bac2eb0224d\",\"dg_utype\":\"Incident\",\"inc_assign\":\"test@dgdemo\",\"inc_creator\":\"dg\",\"inc_id\":\"230523-WIQHA\",\"inc_mtime\":\"2023-05-23 06:56:39\",\"inc_sev\":\"Critical\",\"inc_state\":\"Created\"}",
+        "original": "{\"dg_comment\":\"-\",\"dg_description\":\"This file outlook.exe was going to [demo.digitalg@gmail.com]\",\"dg_guid\":\"1dc3c1fa-5474-4fc0-a7c3-74ff42d28e5e\",\"dg_name\":\"test has attached a Salesforce data to an email\",\"dg_tenant\":\"279b59f3-02f3-44ea-a7c3-9bac2eb0224d\",\"dg_utype\":\"Incident\",\"export_profile\":\"abc123\",\"inc_assign\":\"test@dgdemo\",\"inc_creator\":\"dg\",\"inc_id\":\"230523-WIQHA\",\"inc_mtime\":\"2023-05-23 06:56:39\",\"inc_sev\":\"Critical\",\"inc_state\":\"Created\"}",
         "severity": 1
     },
     "input": {
@@ -66,4 +67,4 @@
     "user": {
         "name": "dg"
     }
-}
+}

packages/digital_guardian/docs/README.md

@@ -79,17 +79,17 @@ An example event for `arc` looks as following:
 
 ```json
 {
-    "@timestamp": "2023-05-23T06:56:39.000Z",
+    "@timestamp": "2025-02-18T04:00:28.647Z",
     "agent": {
-        "ephemeral_id": "bc19c27a-7a31-4b0c-b04b-b3be2ab95a02",
-        "id": "1edfb948-2ef5-4b96-8747-225d782bb6dd",
-        "name": "docker-fleet-agent",
+        "ephemeral_id": "3d727e8f-6944-41c1-a55a-dd22db00d883",
+        "id": "8ae590fa-6e28-49e6-9e43-f64705ab4e6b",
+        "name": "elastic-agent-15774",
         "type": "filebeat",
         "version": "8.13.0"
     },
     "data_stream": {
         "dataset": "digital_guardian.arc",
-        "namespace": "19912",
+        "namespace": "94938",
         "type": "logs"
     },
     "digital_guardian": {
@@ -99,6 +99,7 @@ An example event for `arc` looks as following:
             "dg_name": "test has attached a Salesforce data to an email",
             "dg_tenant": "279b59f3-02f3-44ea-a7c3-9bac2eb0224d",
             "dg_utype": "Incident",
+            "export_profile": "abc123",
             "inc_assign": "test@dgdemo",
             "inc_creator": "dg",
             "inc_id": "230523-WIQHA",
@@ -111,7 +112,7 @@ An example event for `arc` looks as following:
         "version": "8.11.0"
     },
     "elastic_agent": {
-        "id": "1edfb948-2ef5-4b96-8747-225d782bb6dd",
+        "id": "8ae590fa-6e28-49e6-9e43-f64705ab4e6b",
         "snapshot": false,
         "version": "8.13.0"
     },
@@ -120,9 +121,9 @@ An example event for `arc` looks as following:
         "agent_id_status": "verified",
         "dataset": "digital_guardian.arc",
         "id": "1dc3c1fa-5474-4fc0-a7c3-74ff42d28e5e",
-        "ingested": "2024-07-30T15:23:06Z",
+        "ingested": "2025-02-18T04:00:31Z",
         "kind": "alert",
-        "original": "{\"dg_comment\":\"-\",\"dg_description\":\"This file outlook.exe was going to [demo.digitalg@gmail.com]\",\"dg_guid\":\"1dc3c1fa-5474-4fc0-a7c3-74ff42d28e5e\",\"dg_name\":\"test has attached a Salesforce data to an email\",\"dg_tenant\":\"279b59f3-02f3-44ea-a7c3-9bac2eb0224d\",\"dg_utype\":\"Incident\",\"inc_assign\":\"test@dgdemo\",\"inc_creator\":\"dg\",\"inc_id\":\"230523-WIQHA\",\"inc_mtime\":\"2023-05-23 06:56:39\",\"inc_sev\":\"Critical\",\"inc_state\":\"Created\"}",
+        "original": "{\"dg_comment\":\"-\",\"dg_description\":\"This file outlook.exe was going to [demo.digitalg@gmail.com]\",\"dg_guid\":\"1dc3c1fa-5474-4fc0-a7c3-74ff42d28e5e\",\"dg_name\":\"test has attached a Salesforce data to an email\",\"dg_tenant\":\"279b59f3-02f3-44ea-a7c3-9bac2eb0224d\",\"dg_utype\":\"Incident\",\"export_profile\":\"abc123\",\"inc_assign\":\"test@dgdemo\",\"inc_creator\":\"dg\",\"inc_id\":\"230523-WIQHA\",\"inc_mtime\":\"2023-05-23 06:56:39\",\"inc_sev\":\"Critical\",\"inc_state\":\"Created\"}",
         "severity": 1
     },
     "input": {
@@ -157,10 +158,20 @@ An example event for `arc` looks as following:
 | data_stream.dataset | Data stream dataset. | constant_keyword |
 | data_stream.namespace | Data stream namespace. | constant_keyword |
 | data_stream.type | Data stream type. | constant_keyword |
+| digital_guardian.arc.dg_alert.alert_al | Alert AL | keyword |
+| digital_guardian.arc.dg_alert.alert_at | Alert AT | keyword |
+| digital_guardian.arc.dg_alert.alert_bc | Alert BC | keyword |
 | digital_guardian.arc.dg_alert.alert_did | Alert DID | keyword |
-| digital_guardian.arc.dg_alert.alert_wb | Alert WB | integer |
+| digital_guardian.arc.dg_alert.alert_etl | Alert ETL | keyword |
+| digital_guardian.arc.dg_alert.alert_etu | Alert ETU | date |
+| digital_guardian.arc.dg_alert.alert_ur | Alert UR | date |
+| digital_guardian.arc.dg_alert.alert_wb | Alert WB | keyword |
 | digital_guardian.arc.dg_alert.dg_category_name | Alert Category Name | keyword |
+| digital_guardian.arc.dg_alert.dg_detection_source | Alert Detection Source | keyword |
+| digital_guardian.arc.dg_alert.dg_name | Alert Name | keyword |
+| digital_guardian.arc.dg_alert.dg_policy.dg_category_name | Alert Policy Category Name | keyword |
 | digital_guardian.arc.dg_alert.dg_policy.dg_name | Alert Policy Name | keyword |
+| digital_guardian.arc.dg_alert.dg_rule_action_type | Alert Rule Action Type | keyword |
 | digital_guardian.arc.dg_attachments.dg_file_size | File Size | keyword |
 | digital_guardian.arc.dg_attachments.dg_file_size_bytes | File Size in Bytes | long |
 | digital_guardian.arc.dg_comment | Comment | keyword |
@@ -181,6 +192,7 @@ An example event for `arc` looks as following:
 | digital_guardian.arc.dg_tenant | Tenant ID | keyword |
 | digital_guardian.arc.dg_time | Event Time | date |
 | digital_guardian.arc.dg_utype | Operation Type | keyword |
+| digital_guardian.arc.export_profile | Export Profile GUID for the Event | keyword |
 | digital_guardian.arc.inc_assign | Incident Assignee | keyword |
 | digital_guardian.arc.inc_creator | Incident Creator | keyword |
 | digital_guardian.arc.inc_id | Incident ID | keyword |

packages/digital_guardian/manifest.yml

@@ -1,7 +1,7 @@
 format_version: "3.0.3"
 name: digital_guardian
 title: Digital Guardian
-version: "1.3.2"
+version: "1.4.0"
 description: Collect logs from Digital Guardian with Elastic Agent.
 type: integration
 categories: