ti_*: Fix ECS date mapping on threat fields (elastic#10674)

Fix ECS date mapping for threat fields. ecs@mappings component template is missing threat fields mapped as date. Example: fields such as first_seen, last_seen, modified_at are being mapped as keyword in transform's source datastream-backed indices. The transform's destination indices are not effected as they are not datastream-backed and mappings are explicitly defined as date. This causes field type conflicts. - Explicitly add ECS threat fields that are of type date into source data-stream backed fields. - Ensure fields are correctly mapped using system tests.
harnish-elastic · Aug 6, 2024 · 3632d84 · 3632d84
1 parent 6919f9f
commit 3632d84
Show file tree

Hide file tree

Showing 86 changed files with 899 additions and 605 deletions.
diff --git a/packages/ti_anomali/changelog.yml b/packages/ti_anomali/changelog.yml
@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "1.22.1"
+  changes:
+    - description: Fix ECS date mapping on threat fields.
+      type: bugfix
+      link: https://github.com/elastic/integrations/pull/10674
 - version: "1.22.0"
   changes:
     - description: Update the kibana constraint to ^8.13.0. Modified the field definitions to remove ECS fields made redundant by the ecs@mappings component template.

diff --git a/packages/ti_anomali/data_stream/threatstream/fields/ecs.yml b/packages/ti_anomali/data_stream/threatstream/fields/ecs.yml
@@ -0,0 +1,6 @@
+- external: ecs
+  name: threat.indicator.first_seen
+- external: ecs
+  name: threat.indicator.last_seen
+- external: ecs
+  name: threat.indicator.modified_at
diff --git a/packages/ti_anomali/data_stream/threatstream/sample_event.json b/packages/ti_anomali/data_stream/threatstream/sample_event.json
@@ -1,11 +1,11 @@
 {
     "@timestamp": "2020-10-08T12:22:11.000Z",
     "agent": {
-        "ephemeral_id": "5f5fdd12-5b96-4370-aae2-3f4ca99136eb",
-        "id": "8130bdff-3530-4540-8c03-ba091c47a24f",
+        "ephemeral_id": "2f4f6445-5077-4a66-8582-2c74e071b6dd",
+        "id": "36b03887-7783-4bc4-b8c5-6f8997e4cd1a",
         "name": "docker-fleet-agent",
         "type": "filebeat",
-        "version": "8.11.0"
+        "version": "8.13.0"
     },
     "anomali": {
         "threatstream": {
@@ -30,24 +30,24 @@
     },
     "data_stream": {
         "dataset": "ti_anomali.threatstream",
-        "namespace": "ep",
+        "namespace": "44735",
         "type": "logs"
     },
     "ecs": {
         "version": "8.11.0"
     },
     "elastic_agent": {
-        "id": "8130bdff-3530-4540-8c03-ba091c47a24f",
+        "id": "36b03887-7783-4bc4-b8c5-6f8997e4cd1a",
         "snapshot": false,
-        "version": "8.11.0"
+        "version": "8.13.0"
     },
     "event": {
         "agent_id_status": "verified",
         "category": [
             "threat"
         ],
         "dataset": "ti_anomali.threatstream",
-        "ingested": "2023-12-22T11:03:22Z",
+        "ingested": "2024-08-01T07:49:22Z",
         "kind": "enrichment",
         "original": "{\"added_at\":\"2020-10-08T12:22:11\",\"classification\":\"public\",\"confidence\":20,\"country\":\"FR\",\"date_first\":\"2020-10-08T12:21:50\",\"date_last\":\"2020-10-08T12:24:42\",\"detail2\":\"imported by user 184\",\"domain\":\"d4xgfj.example.net\",\"id\":3135167627,\"import_session_id\":1400,\"itype\":\"mal_domain\",\"lat\":-49.1,\"lon\":94.4,\"org\":\"OVH Hosting\",\"resource_uri\":\"/api/v1/intelligence/P46279656657/\",\"severity\":\"high\",\"source\":\"Default Organization\",\"source_feed_id\":3143,\"srcip\":\"89.160.20.156\",\"state\":\"active\",\"trusted_circle_ids\":\"122\",\"update_id\":3786618776,\"value_type\":\"domain\"}",
         "severity": 7,

diff --git a/packages/ti_anomali/docs/README.md b/packages/ti_anomali/docs/README.md
@@ -44,11 +44,11 @@ An example event for `threatstream` looks as following:
 {
     "@timestamp": "2020-10-08T12:22:11.000Z",
     "agent": {
-        "ephemeral_id": "5f5fdd12-5b96-4370-aae2-3f4ca99136eb",
-        "id": "8130bdff-3530-4540-8c03-ba091c47a24f",
+        "ephemeral_id": "2f4f6445-5077-4a66-8582-2c74e071b6dd",
+        "id": "36b03887-7783-4bc4-b8c5-6f8997e4cd1a",
         "name": "docker-fleet-agent",
         "type": "filebeat",
-        "version": "8.11.0"
+        "version": "8.13.0"
     },
     "anomali": {
         "threatstream": {
@@ -73,24 +73,24 @@ An example event for `threatstream` looks as following:
     },
     "data_stream": {
         "dataset": "ti_anomali.threatstream",
-        "namespace": "ep",
+        "namespace": "44735",
         "type": "logs"
     },
     "ecs": {
         "version": "8.11.0"
     },
     "elastic_agent": {
-        "id": "8130bdff-3530-4540-8c03-ba091c47a24f",
+        "id": "36b03887-7783-4bc4-b8c5-6f8997e4cd1a",
         "snapshot": false,
-        "version": "8.11.0"
+        "version": "8.13.0"
     },
     "event": {
         "agent_id_status": "verified",
         "category": [
             "threat"
         ],
         "dataset": "ti_anomali.threatstream",
-        "ingested": "2023-12-22T11:03:22Z",
+        "ingested": "2024-08-01T07:49:22Z",
         "kind": "enrichment",
         "original": "{\"added_at\":\"2020-10-08T12:22:11\",\"classification\":\"public\",\"confidence\":20,\"country\":\"FR\",\"date_first\":\"2020-10-08T12:21:50\",\"date_last\":\"2020-10-08T12:24:42\",\"detail2\":\"imported by user 184\",\"domain\":\"d4xgfj.example.net\",\"id\":3135167627,\"import_session_id\":1400,\"itype\":\"mal_domain\",\"lat\":-49.1,\"lon\":94.4,\"org\":\"OVH Hosting\",\"resource_uri\":\"/api/v1/intelligence/P46279656657/\",\"severity\":\"high\",\"source\":\"Default Organization\",\"source_feed_id\":3143,\"srcip\":\"89.160.20.156\",\"state\":\"active\",\"trusted_circle_ids\":\"122\",\"update_id\":3786618776,\"value_type\":\"domain\"}",
         "severity": 7,
@@ -178,4 +178,7 @@ An example event for `threatstream` looks as following:
 | log.offset | Offset of the entry in the log file. | long |
 | threat.feed.dashboard_id | Dashboard ID used for Kibana CTI UI | constant_keyword |
 | threat.feed.name | Display friendly feed name | constant_keyword |
+| threat.indicator.first_seen | The date and time when intelligence source first reported sighting this indicator. | date |
+| threat.indicator.last_seen | The date and time when intelligence source last reported sighting this indicator. | date |
+| threat.indicator.modified_at | The date and time when intelligence source last modified information for this indicator. | date |
 
diff --git a/packages/ti_anomali/manifest.yml b/packages/ti_anomali/manifest.yml
@@ -1,6 +1,6 @@
 name: ti_anomali
 title: Anomali
-version: "1.22.0"
+version: "1.22.1"
 description: Ingest threat intelligence indicators from Anomali with Elastic Agent.
 type: integration
 format_version: 3.0.2

diff --git a/packages/ti_cif3/changelog.yml b/packages/ti_cif3/changelog.yml
@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "1.14.1"
+  changes:
+    - description: Fix ECS date mapping on threat fields.
+      type: bugfix
+      link: https://github.com/elastic/integrations/pull/10674
 - version: "1.14.0"
   changes:
     - description: Update the kibana constraint to ^8.13.0. Modified the field definitions to remove ECS fields made redundant by the ecs@mappings component template.

diff --git a/packages/ti_cif3/data_stream/feed/fields/ecs.yml b/packages/ti_cif3/data_stream/feed/fields/ecs.yml
@@ -1,3 +1,9 @@
+- name: threat.indicator.first_seen
+  external: ecs
+- name: threat.indicator.last_seen
+  external: ecs
+- name: threat.indicator.modified_at
+  external: ecs
 - name: threat.indicator.tls.client.ja3
   level: extended
   type: keyword

diff --git a/packages/ti_cif3/data_stream/feed/sample_event.json b/packages/ti_cif3/data_stream/feed/sample_event.json
@@ -1,11 +1,11 @@
 {
-    "@timestamp": "2024-04-10T04:46:58.281Z",
+    "@timestamp": "2024-08-01T08:05:14.040Z",
     "agent": {
-        "ephemeral_id": "94c530db-5c8f-407c-939b-cd1d21d547fc",
-        "id": "28f0e936-c71c-4f75-8919-506fed4d20e7",
+        "ephemeral_id": "b351d699-2fd0-49f7-99e1-a7a471a29a62",
+        "id": "36b03887-7783-4bc4-b8c5-6f8997e4cd1a",
         "name": "docker-fleet-agent",
         "type": "filebeat",
-        "version": "8.12.1"
+        "version": "8.13.0"
     },
     "cif3": {
         "deleted_at": "2022-09-03T20:25:53.000Z",
@@ -17,25 +17,25 @@
     },
     "data_stream": {
         "dataset": "ti_cif3.feed",
-        "namespace": "ep",
+        "namespace": "26457",
         "type": "logs"
     },
     "ecs": {
         "version": "8.11.0"
     },
     "elastic_agent": {
-        "id": "28f0e936-c71c-4f75-8919-506fed4d20e7",
+        "id": "36b03887-7783-4bc4-b8c5-6f8997e4cd1a",
         "snapshot": false,
-        "version": "8.12.1"
+        "version": "8.13.0"
     },
     "event": {
         "agent_id_status": "verified",
         "category": [
             "threat"
         ],
-        "created": "2024-04-10T04:46:58.281Z",
+        "created": "2024-08-01T08:05:14.040Z",
         "dataset": "ti_cif3.feed",
-        "ingested": "2024-04-10T04:47:10Z",
+        "ingested": "2024-08-01T08:05:26Z",
         "kind": "enrichment",
         "original": "{\"application\":\"https\",\"asn\":8075,\"asn_desc\":\"microsoft-corp-msn-as-block\",\"cc\":\"br\",\"city\":\"campinas\",\"confidence\":10,\"count\":1,\"firsttime\":\"2022-07-20T20:25:53.000000Z\",\"group\":[\"everyone\"],\"indicator\":\"20.206.75.106\",\"indicator_ipv4\":\"20.206.75.106\",\"itype\":\"ipv4\",\"lasttime\":\"2022-07-20T20:25:53.000000Z\",\"latitude\":-22.9035,\"location\":[-47.0565,-22.9035],\"longitude\":-47.0565,\"portlist\":\"443\",\"protocol\":\"tcp\",\"provider\":\"sslbl.abuse.ch\",\"reference\":\"https://sslbl.abuse.ch/blacklist/sslipblacklist.csv\",\"region\":\"sao paulo\",\"reporttime\":\"2022-07-21T20:33:26.585967Z\",\"tags\":[\"botnet\"],\"timezone\":\"america/sao_paulo\",\"tlp\":\"white\",\"uuid\":\"ac240898-1443-4d7e-a98a-1daed220c162\"}",
         "type": [

diff --git a/packages/ti_cif3/docs/README.md b/packages/ti_cif3/docs/README.md
@@ -79,20 +79,23 @@ CIFv3 `confidence` field values (0..10) are converted to ECS confidence (None, L
 | log.flags | Flags for the log file. | keyword |
 | log.offset | Offset of the entry in the log file. | long |
 | threat.feed.name | Display friendly feed name | constant_keyword |
+| threat.indicator.first_seen | The date and time when intelligence source first reported sighting this indicator. | date |
+| threat.indicator.last_seen | The date and time when intelligence source last reported sighting this indicator. | date |
+| threat.indicator.modified_at | The date and time when intelligence source last modified information for this indicator. | date |
 | threat.indicator.tls.client.ja3 | An md5 hash that identifies clients based on their TLS handshake. | keyword |
 
 
 An example event for `feed` looks as following:
 
 ```json
 {
-    "@timestamp": "2024-04-10T04:46:58.281Z",
+    "@timestamp": "2024-08-01T08:05:14.040Z",
     "agent": {
-        "ephemeral_id": "94c530db-5c8f-407c-939b-cd1d21d547fc",
-        "id": "28f0e936-c71c-4f75-8919-506fed4d20e7",
+        "ephemeral_id": "b351d699-2fd0-49f7-99e1-a7a471a29a62",
+        "id": "36b03887-7783-4bc4-b8c5-6f8997e4cd1a",
         "name": "docker-fleet-agent",
         "type": "filebeat",
-        "version": "8.12.1"
+        "version": "8.13.0"
     },
     "cif3": {
         "deleted_at": "2022-09-03T20:25:53.000Z",
@@ -104,25 +107,25 @@ An example event for `feed` looks as following:
     },
     "data_stream": {
         "dataset": "ti_cif3.feed",
-        "namespace": "ep",
+        "namespace": "26457",
         "type": "logs"
     },
     "ecs": {
         "version": "8.11.0"
     },
     "elastic_agent": {
-        "id": "28f0e936-c71c-4f75-8919-506fed4d20e7",
+        "id": "36b03887-7783-4bc4-b8c5-6f8997e4cd1a",
         "snapshot": false,
-        "version": "8.12.1"
+        "version": "8.13.0"
     },
     "event": {
         "agent_id_status": "verified",
         "category": [
             "threat"
         ],
-        "created": "2024-04-10T04:46:58.281Z",
+        "created": "2024-08-01T08:05:14.040Z",
         "dataset": "ti_cif3.feed",
-        "ingested": "2024-04-10T04:47:10Z",
+        "ingested": "2024-08-01T08:05:26Z",
         "kind": "enrichment",
         "original": "{\"application\":\"https\",\"asn\":8075,\"asn_desc\":\"microsoft-corp-msn-as-block\",\"cc\":\"br\",\"city\":\"campinas\",\"confidence\":10,\"count\":1,\"firsttime\":\"2022-07-20T20:25:53.000000Z\",\"group\":[\"everyone\"],\"indicator\":\"20.206.75.106\",\"indicator_ipv4\":\"20.206.75.106\",\"itype\":\"ipv4\",\"lasttime\":\"2022-07-20T20:25:53.000000Z\",\"latitude\":-22.9035,\"location\":[-47.0565,-22.9035],\"longitude\":-47.0565,\"portlist\":\"443\",\"protocol\":\"tcp\",\"provider\":\"sslbl.abuse.ch\",\"reference\":\"https://sslbl.abuse.ch/blacklist/sslipblacklist.csv\",\"region\":\"sao paulo\",\"reporttime\":\"2022-07-21T20:33:26.585967Z\",\"tags\":[\"botnet\"],\"timezone\":\"america/sao_paulo\",\"tlp\":\"white\",\"uuid\":\"ac240898-1443-4d7e-a98a-1daed220c162\"}",
         "type": [

diff --git a/packages/ti_cif3/manifest.yml b/packages/ti_cif3/manifest.yml
@@ -1,7 +1,7 @@
 format_version: "3.0.2"
 name: ti_cif3
 title: "Collective Intelligence Framework v3"
-version: "1.14.0"
+version: "1.14.1"
 description: "Ingest threat indicators from a Collective Intelligence Framework v3 instance with Elastic Agent."
 type: integration
 categories:

diff --git a/packages/ti_crowdstrike/changelog.yml b/packages/ti_crowdstrike/changelog.yml
@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "1.1.3"
+  changes:
+    - description: Fix ECS date mapping on threat fields.
+      type: bugfix
+      link: https://github.com/elastic/integrations/pull/10674
 - version: "1.1.2"
   changes:
     - description: Fix handling of timestamps with positive time zone offsets.

diff --git a/packages/ti_crowdstrike/data_stream/intel/fields/ecs.yml b/packages/ti_crowdstrike/data_stream/intel/fields/ecs.yml
@@ -0,0 +1,6 @@
+- external: ecs
+  name: threat.indicator.first_seen
+- external: ecs
+  name: threat.indicator.last_seen
+- external: ecs
+  name: threat.indicator.modified_at
diff --git a/packages/ti_crowdstrike/data_stream/intel/sample_event.json b/packages/ti_crowdstrike/data_stream/intel/sample_event.json
@@ -1,24 +1,24 @@
 {
     "@timestamp": "2023-11-21T06:16:01.000Z",
     "agent": {
-        "ephemeral_id": "ee250a38-ef6d-486c-a245-6d0dd0785a11",
-        "id": "803f2aef-a6c1-47c8-b64d-e484bb967db4",
+        "ephemeral_id": "6d3e7b87-a3f6-47b1-81a5-0264e901b3f9",
+        "id": "36b03887-7783-4bc4-b8c5-6f8997e4cd1a",
         "name": "docker-fleet-agent",
         "type": "filebeat",
-        "version": "8.12.0"
+        "version": "8.13.0"
     },
     "data_stream": {
         "dataset": "ti_crowdstrike.intel",
-        "namespace": "ep",
+        "namespace": "36922",
         "type": "logs"
     },
     "ecs": {
         "version": "8.11.0"
     },
     "elastic_agent": {
-        "id": "803f2aef-a6c1-47c8-b64d-e484bb967db4",
+        "id": "36b03887-7783-4bc4-b8c5-6f8997e4cd1a",
         "snapshot": false,
-        "version": "8.12.0"
+        "version": "8.13.0"
     },
     "event": {
         "agent_id_status": "verified",
@@ -27,7 +27,7 @@
         ],
         "dataset": "ti_crowdstrike.intel",
         "id": "hash_sha256_c98e1a7f563824cd448b47613743dcd1c853742b78f42b000192b83d",
-        "ingested": "2024-03-28T10:49:11Z",
+        "ingested": "2024-08-01T08:31:15Z",
         "kind": "enrichment",
         "original": "{\"_marker\":\"17005473618d17ae6353d123235e4158c5c81f25f0\",\"actors\":[\"SALTYSPIDER\"],\"deleted\":false,\"domain_types\":[\"abc.com\"],\"id\":\"hash_sha256_c98e1a7f563824cd448b47613743dcd1c853742b78f42b000192b83d\",\"indicator\":\"c98e192bf71a7f97563824cd448b47613743dcd1c853742b78f42b000192b83d\",\"ip_address_types\":[\"81.2.69.192\"],\"kill_chains\":[\"Installation\",\"C2\"],\"labels\":[{\"created_on\":1700547356,\"last_valid_on\":1700547360,\"name\":\"MaliciousConfidence/High\"},{\"created_on\":1700547359,\"last_valid_on\":1700547359,\"name\":\"Malware/Mofksys\"},{\"created_on\":1700547359,\"last_valid_on\":1700547359,\"name\":\"ThreatType/Commodity\"},{\"created_on\":1700547359,\"last_valid_on\":1700547359,\"name\":\"ThreatType/CredentialHarvesting\"},{\"created_on\":1700547359,\"last_valid_on\":1700547359,\"name\":\"ThreatType/InformationStealer\"}],\"last_updated\":1700547361,\"malicious_confidence\":\"high\",\"malware_families\":[\"Mofksys\"],\"published_date\":1700547356,\"relations\":[{\"created_date\":1700547339,\"id\":\"domain.com.yy\",\"indicator\":\"domain.ds\",\"last_valid_date\":1700547339,\"type\":\"domain\"},{\"created_date\":1700547339,\"id\":\"domain.xx.yy\",\"indicator\":\"domain.xx.fd\",\"last_valid_date\":1700547339,\"type\":\"domain\"}],\"reports\":[\"reports\"],\"targets\":[\"abc\"],\"threat_types\":[\"Commodity\",\"CredentialHarvesting\",\"InformationStealer\"],\"type\":\"hash_sha256\",\"vulnerabilities\":[\"vuln\"]}",
         "type": [

diff --git a/packages/ti_crowdstrike/data_stream/ioc/fields/ecs.yml b/packages/ti_crowdstrike/data_stream/ioc/fields/ecs.yml
@@ -0,0 +1,6 @@
+- external: ecs
+  name: threat.indicator.first_seen
+- external: ecs
+  name: threat.indicator.last_seen
+- external: ecs
+  name: threat.indicator.modified_at
diff --git a/packages/ti_crowdstrike/data_stream/ioc/sample_event.json b/packages/ti_crowdstrike/data_stream/ioc/sample_event.json
@@ -1,24 +1,24 @@
 {
     "@timestamp": "2023-11-01T10:22:23.106Z",
     "agent": {
-        "ephemeral_id": "ca4c5a70-0aa1-4cb3-867c-3c099798eef4",
-        "id": "803f2aef-a6c1-47c8-b64d-e484bb967db4",
+        "ephemeral_id": "6b69edbe-1d0f-4094-80d6-12915b7784ed",
+        "id": "36b03887-7783-4bc4-b8c5-6f8997e4cd1a",
         "name": "docker-fleet-agent",
         "type": "filebeat",
-        "version": "8.12.0"
+        "version": "8.13.0"
     },
     "data_stream": {
         "dataset": "ti_crowdstrike.ioc",
-        "namespace": "ep",
+        "namespace": "60867",
         "type": "logs"
     },
     "ecs": {
         "version": "8.11.0"
     },
     "elastic_agent": {
-        "id": "803f2aef-a6c1-47c8-b64d-e484bb967db4",
+        "id": "36b03887-7783-4bc4-b8c5-6f8997e4cd1a",
         "snapshot": false,
-        "version": "8.12.0"
+        "version": "8.13.0"
     },
     "event": {
         "action": "detect-again",
@@ -28,7 +28,7 @@
         ],
         "dataset": "ti_crowdstrike.ioc",
         "id": "34874a88935860cf6yyfc856d6abb6f35a29d8c077195ed6291aa8373696b44",
-        "ingested": "2024-03-28T10:50:10Z",
+        "ingested": "2024-08-01T08:32:09Z",
         "kind": "enrichment",
         "original": "{\"action\":\"detect again\",\"applied_globally\":true,\"created_by\":\"abc.it@example.com\",\"created_on\":\"2023-11-01T10:22:23.10607613Z\",\"deleted\":false,\"description\":\"IS-38887\",\"expired\":false,\"from_parent\":false,\"id\":\"34874a88935860cf6yyfc856d6abb6f35a29d8c077195ed6291aa8373696b44\",\"metadata\":{\"filename\":\"High_Serverity_Heuristic_Sandbox_Threat.docx\"},\"modified_by\":\"example.it@ex.com\",\"modified_on\":\"2023-11-01T10:22:23.10607613Z\",\"platforms\":[\"windows\",\"mac\",\"linux\"],\"severity\":\"critical\",\"tags\":[\"IS-38887\"],\"type\":\"ipv4\",\"value\":\"81.2.69.192\"}",
         "type": [