elastic · kaiyan-sheng · Jul 11, 2024 · May 17, 2024 · May 17, 2024 · Jun 13, 2024
@@ -1,3 +1,3 @@
 dependencies:
   ecs:
-    reference: git@v8.0.0
+    reference: git@v8.11.0
@@ -1,8 +1,9 @@
 # Amazon Data Firehose
-Amazon Data Firehose integration offers users a way to stream logs from Firehose to Elastic Cloud.
-This integration includes predefined rules that automatically route AWS service logs to the respective integrations, which
-include field mappings, ingest pipelines, predefined dashboards and ect. Here is a list of log types that are supported
-by this integration:
+Amazon Data Firehose integration offers users a way to stream logs and CloudWatch metrics from Firehose to Elastic Cloud.
+This integration includes predefined rules that automatically route AWS service logs and CloudWatch metrics to the respective integrations, which
+include field mappings, ingest pipelines, predefined dashboards and ect. 
+
+Here is a list of log types that are supported by this integration:
 
 | AWS service log    | Log destination          |
 |--------------------|--------------------------|
@@ -17,6 +18,31 @@ by this integration:
 | VPC Flow           | Firehose, CloudWatch, S3 |
 | WAF                | Firehose, CloudWatch. S3 |
 
+Here is a list of CloudWatch metrics that are supported by this integration:
+
+| AWS service monitoring metrics |
+|--------------------------------|
+| API Gateway                    |
+| DynamoDB                       |
+| EBS                            |
+| EC2                            |
+| ECS                            |
+| ELB                            |
+| EMR                            |
+| Network Firewall               |
+| Kafka                          |
+| Kinesis                        |
+| Lambda                         |
+| NATGateway                     |
+| RDS                            |
+| S3                             |
+| S3 Storage Lens                |
+| SNS                            |
+| SQS                            |
+| TransitGateway                 |
+| Usage                          |
+| VPN                            |
+
 ## Limitation
 It is not possible to configure a delivery stream to send data to Elastic Cloud via PrivateLink (VPC endpoint). 
 This is a current limitation in Firehose, which we are working with AWS to resolve.
@@ -91,6 +117,12 @@ This is a current limitation in Firehose, which we are working with AWS to resol
        This parameter will increase the data volume in Elasticsearch and should be used with care.
 
 3. Send data to the Firehose delivery stream
-
+    1. logs
     Consult the [AWS documentation](https://docs.aws.amazon.com/firehose/latest/dev/basic-write.html) for details on how to
     configure a variety of log sources to send data to Firehose delivery streams.
+
+    2. metrics
+    Consult the [AWS documentation](https://docs.aws.amazon.com/AmazonCloudWatch/latest/monitoring/CloudWatch-metric-streams-setup.html)
+    for details on how to set up a metric stream in CloudWatch and 
+    [Custom setup with Firehose](https://docs.aws.amazon.com/AmazonCloudWatch/latest/monitoring/CloudWatch-metric-streams-setup-datalake.html) 
+    to send metrics to Firehose. For Elastic, we only support JSON and OpenTelemetry 1.0.0 formats for the metrics.
@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "1.1.0"
+  changes:
+    - description: Add routing rules for metrics from Firehose.
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/9916
 - version: "1.0.0"
   changes:
     - description: Release package as GA.

@@ -9,17 +9,14 @@
             "aws.firehose.subscription_filters": "[apigateway-to-firehose]",
             "aws.kinesis.name": "firehose-apigateway-logs-to-elastic",
             "aws.kinesis.type": "deliverystream",
-            "cloud": {
-                "provider": "aws"
-            },
             "cloud.account.id": "123456",
             "cloud.provider": "aws",
             "cloud.region": "us-east-1",
             "data_stream.dataset": "aws.apigateway_logs",
             "data_stream.namespace": "default",
             "data_stream.type": "logs",
             "ecs": {
-                "version": "8.0.0"
+                "version": "8.11.0"
             },
             "event.id": "37670326805251200781477669690942747782212394134076063744",
             "message": "{\"requestId\":\"GQIVriFLIAMEMsA=\",\"ip\":\"1.128.0.0\",\"requestTime\":\"09/Jun/2023:12:54:08 +0000\",\"httpMethod\":\"GET\",\"routeKey\":\"GET /\",\"status\":\"200\",\"protocol\":\"HTTP/1.1\",\"responseLength\":\"47140\"}"
@@ -33,17 +30,14 @@
             "aws.firehose.subscription_filters": "[apigateway-to-firehose]",
             "aws.kinesis.name": "firehose-apigateway-logs-to-elastic",
             "aws.kinesis.type": "deliverystream",
-            "cloud": {
-                "provider": "aws"
-            },
             "cloud.account.id": "123456",
             "cloud.provider": "aws",
             "cloud.region": "us-east-1",
             "data_stream.dataset": "aws.apigateway_logs",
             "data_stream.namespace": "default",
             "data_stream.type": "logs",
             "ecs": {
-                "version": "8.0.0"
+                "version": "8.11.0"
             },
             "event.id": "37670326805251200781477669690942747782212394134076063744",
             "message": "{\"requestId\":\"Iq9gjE_aIAMFZTg=\",\"ip\":\"1.128.0.0\",\"caller\":\"-\",\"user\":\"-\",\"requestTime\":\"26/Jul/2023:12:20:44 +0000\",\"eventType\":\"CONNECT\",\"routeKey\":\"$connect\",\"status\":\"500\",\"connectionId\":\"Iq8gj1UmIAMCKpA=\",\"apiId\":\"z1ctxygne5\",\"stage\":\"production\",\"domainName\":\"z1ctxygne5.execute-api.us-east-1.amazonaws.com\"}"
@@ -57,17 +51,14 @@
             "aws.firehose.subscription_filters": "[apigateway-to-firehose]",
             "aws.kinesis.name": "firehose-apigateway-logs-to-elastic",
             "aws.kinesis.type": "deliverystream",
-            "cloud": {
-                "provider": "aws"
-            },
             "cloud.account.id": "123456",
             "cloud.provider": "aws",
             "cloud.region": "us-east-1",
             "data_stream.dataset": "aws.apigateway_logs",
             "data_stream.namespace": "default",
             "data_stream.type": "logs",
             "ecs": {
-                "version": "8.0.0"
+                "version": "8.11.0"
             },
             "event.id": "37670326805251200781477669690942747782212394134076063744",
             "message": "{\"requestId\":\"48752d0f-c99d-4cfa-a5a7-f3c6834d19e5\",\"ip\":\"1.128.0.0\",\"caller\":\"-\",\"user\":\"-\",\"requestTime\":\"10/Jun/2023:15:36:28 +0000\",\"httpMethod\":\"GET\",\"resourcePath\":\"/pets\",\"status\":\"200\",\"protocol\":\"HTTP/1.1\",\"responseLength\":\"184\"}"

@@ -1,19 +1,19 @@
 {
-  "events": [
-    {
-      "cloud.region": "us-east-1",
-      "aws.firehose.arn": "arn:aws:firehose:us-east-2:123456:deliverystream/firehose-cloudfront-logs-to-elastic",
-      "data_stream.namespace": "default",
-      "message": "2022-04-19 12:29:36 SEA19-C2 10157 81.2.69.143 POST d111111abcdef8.cloudfront.net /getApplications 200 https://test.com/global Mozilla/5.0%20(Windows%20NT%2010.0;%20Win64;%20x64)%20AppleWebKit/537.36%20(KHTML,%20like%20Gecko)%20Chrome/100.0.4896.127%20Safari/537.36 source=global - Miss hrsHM5OM6sTIXUleC1G20YtDxMf5Cq0Jbz0pwhVpod2kgEn_W6akCQ== test.com https 1057 0.238 - TLSv1.3 TLS_AES_128_GCM_SHA256 Miss HTTP/2.0 - - 4203 0.238 Miss application/json;charset=UTF-8 - - -",
-      "aws.kinesis.type": "deliverystream",
-      "data_stream.type": "logs",
-      "aws.firehose.request_id": "971ae05f-a128-4a7f-b623-30f9bc513e55",
-      "cloud.provider": "aws",
-      "@timestamp": "2023-07-25T21:04:35Z",
-      "cloud.account.id": "123456",
-      "data_stream.dataset": "awsfirehose",
-      "aws.kinesis.name": "firehose-cloudfront-logs-to-elastic",
-      "event.id": "37670326805251200781477669690942747782212394134076063744"
-    }
-  ]
+    "events": [
+        {
+            "cloud.region": "us-east-1",
+            "aws.firehose.arn": "arn:aws:firehose:us-east-2:123456:deliverystream/firehose-cloudfront-logs-to-elastic",
+            "data_stream.namespace": "default",
+            "message": "2022-04-19 12:29:36 SEA19-C2 10157 81.2.69.143 POST d111111abcdef8.cloudfront.net /getApplications 200 https://test.com/global Mozilla/5.0%20(Windows%20NT%2010.0;%20Win64;%20x64)%20AppleWebKit/537.36%20(KHTML,%20like%20Gecko)%20Chrome/100.0.4896.127%20Safari/537.36 source=global - Miss hrsHM5OM6sTIXUleC1G20YtDxMf5Cq0Jbz0pwhVpod2kgEn_W6akCQ== test.com https 1057 0.238 - TLSv1.3 TLS_AES_128_GCM_SHA256 Miss HTTP/2.0 - - 4203 0.238 Miss application/json;charset=UTF-8 - - -",
+            "aws.kinesis.type": "deliverystream",
+            "data_stream.type": "logs",
+            "aws.firehose.request_id": "971ae05f-a128-4a7f-b623-30f9bc513e55",
+            "cloud.provider": "aws",
+            "@timestamp": "2023-07-25T21:04:35Z",
+            "cloud.account.id": "123456",
+            "data_stream.dataset": "awsfirehose",
+            "aws.kinesis.name": "firehose-cloudfront-logs-to-elastic",
+            "event.id": "37670326805251200781477669690942747782212394134076063744"
+        }
+    ]
 }
@@ -6,17 +6,14 @@
             "aws.firehose.request_id": "971ae05f-a128-4a7f-b623-30f9bc513e55",
             "aws.kinesis.name": "firehose-cloudfront-logs-to-elastic",
             "aws.kinesis.type": "deliverystream",
-            "cloud": {
-                "provider": "aws"
-            },
             "cloud.account.id": "123456",
             "cloud.provider": "aws",
             "cloud.region": "us-east-1",
             "data_stream.dataset": "aws.cloudfront_logs",
             "data_stream.namespace": "default",
             "data_stream.type": "logs",
             "ecs": {
-                "version": "8.0.0"
+                "version": "8.11.0"
             },
             "event.id": "37670326805251200781477669690942747782212394134076063744",
             "message": "2022-04-19 12:29:36 SEA19-C2 10157 81.2.69.143 POST d111111abcdef8.cloudfront.net /getApplications 200 https://test.com/global Mozilla/5.0%20(Windows%20NT%2010.0;%20Win64;%20x64)%20AppleWebKit/537.36%20(KHTML,%20like%20Gecko)%20Chrome/100.0.4896.127%20Safari/537.36 source=global - Miss hrsHM5OM6sTIXUleC1G20YtDxMf5Cq0Jbz0pwhVpod2kgEn_W6akCQ== test.com https 1057 0.238 - TLSv1.3 TLS_AES_128_GCM_SHA256 Miss HTTP/2.0 - - 4203 0.238 Miss application/json;charset=UTF-8 - - -"

@@ -9,17 +9,14 @@
             "aws.firehose.subscription_filters": "[cloudtrail-to-firehose]",
             "aws.kinesis.name": "firehose-cloudtrail-logs-to-elastic",
             "aws.kinesis.type": "deliverystream",
-            "cloud": {
-                "provider": "aws"
-            },
             "cloud.account.id": "123456",
             "cloud.provider": "aws",
             "cloud.region": "us-east-2",
             "data_stream.dataset": "aws.cloudtrail",
             "data_stream.namespace": "default",
             "data_stream.type": "logs",
             "ecs": {
-                "version": "8.0.0"
+                "version": "8.11.0"
             },
             "event.id": "37670326805251200781477669690942747782212394134076063744",
             "message": "{\"eventVersion\":\"1.08\",\"userIdentity\":{\"type\":\"AWSService\",\"invokedBy\":\"cloudtrail.amazonaws.com\"},\"eventTime\":\"2023-07-17T21:02:26Z\",\"eventSource\":\"sts.amazonaws.com\",\"eventName\":\"AssumeRole\",\"awsRegion\":\"sa-east-1\",\"sourceIPAddress\":\"cloudtrail.amazonaws.com\",\"userAgent\":\"cloudtrail.amazonaws.com\",\"requestParameters\":{\"roleArn\":\"arn:aws:iam::123456:role/service-role/aws-cloudtrail-logs-123456-b888baff_Role\",\"roleSessionName\":\"CLOUDWATCH_LOGS_DELIVERY_SESSION\"},\"responseElements\":{\"credentials\":{\"accessKeyId\":\"ASIAZEDJODE3A5LVGLFB\",\"sessionToken\":\"IQoJb3JpZ2luX2VjEGUaCXNhLWVhc3QtMSJHMEUCIHgHmtcrhwDhosJlQVky+C2zsYDKuR99qVlNjGIp8FLWAiEAsJtTDQ3Arq8iXEOHwv0ImEQdGb5tbgc+fLpoK58Enb4q9AII3v//////////ARAEGgw2MjcyODYzNTAxMzQiDN5gNdfO4ZdSqDmmwSrIAicTBYZg+ZXjwiJTN/Bz2YsMWYU6psw5znG3/Gh3EJ1P3RCmB7d79X6XZzFVi2u2xdrnaY/sTKDfp1jdl8OoAsSKYwJiGbzjoQlv59bB6JqPbKfAKUPAmz6JEMWNFgWTtaQL9rNkdPz23u/1msoUSzxCcxR9f3A2dD4yqnVpNJe8ipuhxpBMzQ61vcGL4G5hQEDM/o8sORP2PXbK4O7QAuWOyuryYkHAPwY9RrL0WHfflGBEBQV6XlidGpsRCtIppZVn025n3DQOypDEaL3fKp0gUsMkDH+frFjxop4o4wRYC3CxXe3XRJ5/Te886rQry7RUfXlQtiCfojZO5ohcLB+z6Y/uCK0IHp3zrfl5shKsQIAFt7p0B8W7PK5yHE4W9HHRiktJ9wTtq1YCTaWECpnjW0bISNgumRmDOAJvVHAjSjfkr4yAlJkw4qm8pQY6vwGbBiuf98AfRFrXMy01hVdE3GNTBrIS68zxUJaOjBLgw8l0nEC00L+LPuqaASFWz65Dnq5JAjXaDD9E3iCi4klp4gZFAcj7uGgeBIPkP7Bpr4SvBfnnqCgE2oyFrWke3NnYtqkL5iHLJeGlOTrvI5ND2H4jurQv0KbiqwHt6DmGF3poZOrtf8R3piNcuCCDLU8RvhRVLHy5rKPzsWgNokBc9XXmgltwvB6rIgdZhBJzupzmy/NSoWZcOeH2ooEELw==\",\"expiration\":\"Jul 12, 2023, 10:02:26 PM\"},\"assumedRoleUser\":{\"assumedRoleId\":\"AROAZEDJODE3NLJAH2FZC:CLOUDWATCH_LOGS_DELIVERY_SESSION\",\"arn\":\"arn:aws:sts::123456:assumed-role/aws-cloudtrail-logs-123456-b888baff_Role/CLOUDWATCH_LOGS_DELIVERY_SESSION\"}},\"requestID\":\"041c9e5f-a031-47d2-a4a0-011bc8d5352c\",\"eventID\":\"3096b662-7aa9-43e6-8bee-541a45686745\",\"readOnly\":true,\"resources\":[{\"accountId\":\"123456\",\"type\":\"AWS::IAM::Role\",\"ARN\":\"arn:aws:iam::123456:role/service-role/aws-cloudtrail-logs-123456-b888baff_Role\"}],\"eventType\":\"AwsApiCall\",\"managementEvent\":true,\"recipientAccountId\":\"123456\",\"sharedEventID\":\"a1c94275-884f-4c1f-b8dc-2e1bf4c94d29\",\"eventCategory\":\"Management\"}"

@@ -1,49 +1,49 @@
 {
-  "events": [
-    {
-      "cloud.region": "us-east-1",
-      "aws.firehose.arn": "arn:aws:firehose:us-east-2:123456:deliverystream/firehose-classic-load-balancer-logs-to-elastic",
-      "data_stream.namespace": "default",
-      "message": "2015-05-13T23:39:43.945958Z my-loadbalancer 192.168.131.39:2817 10.0.0.1:80 0.000073 0.001048 0.000057 200 200 0 29 \"GET http://www.example.com:80/ HTTP/1.1\" \"curl/7.38.0\" - -",
-      "aws.kinesis.type": "deliverystream",
-      "data_stream.type": "logs",
-      "aws.firehose.request_id": "971ae05f-a128-4a7f-b623-30f9bc513e55",
-      "cloud.provider": "aws",
-      "@timestamp": "2023-07-25T21:04:35Z",
-      "cloud.account.id": "123456",
-      "data_stream.dataset": "awsfirehose",
-      "aws.kinesis.name": "firehose-classic-load-balancer-logs-to-elastic",
-      "event.id": "37670326805251200781477669690942747782212394134076063744"
-    },
-    {
-      "cloud.region": "us-east-1",
-      "aws.firehose.arn": "arn:aws:firehose:us-east-2:123456:deliverystream/firehose-application-load-balancer-logs-to-elastic",
-      "data_stream.namespace": "default",
-      "message": "http 2018-07-02T22:23:00.186641Z app/my-loadbalancer/50dc6c495c0c9188 192.168.131.39:2817 10.0.0.1:80 0.000 0.001 0.000 200 200 34 366 \"GET http://www.example.com:80/ HTTP/1.1\" \"curl/7.46.0\" - - arn:aws:elasticloadbalancing:us-east-2:123456789012:targetgroup/my-targets/73e2d6bc24d8a067 \"Root=1-58337262-36d228ad5d99923122bbe354\" \"-\" \"-\" 0 2018-07-02T22:22:48.364000Z \"forward\" \"-\" \"-\" \"10.0.0.1:80\" \"200\" \"-\" \"-\"",
-      "aws.kinesis.type": "deliverystream",
-      "data_stream.type": "logs",
-      "aws.firehose.request_id": "971ae05f-a128-4a7f-b623-30f9bc513e55",
-      "cloud.provider": "aws",
-      "@timestamp": "2023-07-25T21:04:35Z",
-      "cloud.account.id": "123456",
-      "data_stream.dataset": "awsfirehose",
-      "aws.kinesis.name": "firehose-application-load-balancer-logs-to-elastic",
-      "event.id": "37670326805251200781477669690942747782212394134076063744"
-    },
-    {
-      "cloud.region": "us-east-1",
-      "aws.firehose.arn": "arn:aws:firehose:us-east-2:123456:deliverystream/firehose-network-load-balancer-logs-to-elastic",
-      "data_stream.namespace": "default",
-      "message": "tls 2.0 2018-12-20T02:59:40 net/my-network-loadbalancer/c6e77e28c25b2234 g3d4b5e8bb8464cd 72.21.218.154:51341 172.100.100.185:443 5 2 98 246 - arn:aws:acm:us-east-2:671290407336:certificate/2a108f19-aded-46b0-8493-c63eb1ef4a99 - ECDHE-RSA-AES128-SHA tlsv12 - my-network-loadbalancer-c6e77e28c25b2234.elb.us-east-2.amazonaws.com - - - 2018-12-20T02:59:30",
-      "aws.kinesis.type": "deliverystream",
-      "data_stream.type": "logs",
-      "aws.firehose.request_id": "971ae05f-a128-4a7f-b623-30f9bc513e55",
-      "cloud.provider": "aws",
-      "@timestamp": "2023-07-25T21:04:35Z",
-      "cloud.account.id": "123456",
-      "data_stream.dataset": "awsfirehose",
-      "aws.kinesis.name": "firehose-network-load-balancer-logs-to-elastic",
-      "event.id": "37670326805251200781477669690942747782212394134076063744"
-    }
-  ]
+    "events": [
+        {
+            "cloud.region": "us-east-1",
+            "aws.firehose.arn": "arn:aws:firehose:us-east-2:123456:deliverystream/firehose-classic-load-balancer-logs-to-elastic",
+            "data_stream.namespace": "default",
+            "message": "2015-05-13T23:39:43.945958Z my-loadbalancer 192.168.131.39:2817 10.0.0.1:80 0.000073 0.001048 0.000057 200 200 0 29 \"GET http://www.example.com:80/ HTTP/1.1\" \"curl/7.38.0\" - -",
+            "aws.kinesis.type": "deliverystream",
+            "data_stream.type": "logs",
+            "aws.firehose.request_id": "971ae05f-a128-4a7f-b623-30f9bc513e55",
+            "cloud.provider": "aws",
+            "@timestamp": "2023-07-25T21:04:35Z",
+            "cloud.account.id": "123456",
+            "data_stream.dataset": "awsfirehose",
+            "aws.kinesis.name": "firehose-classic-load-balancer-logs-to-elastic",
+            "event.id": "37670326805251200781477669690942747782212394134076063744"
+        },
+        {
+            "cloud.region": "us-east-1",
+            "aws.firehose.arn": "arn:aws:firehose:us-east-2:123456:deliverystream/firehose-application-load-balancer-logs-to-elastic",
+            "data_stream.namespace": "default",
+            "message": "http 2018-07-02T22:23:00.186641Z app/my-loadbalancer/50dc6c495c0c9188 192.168.131.39:2817 10.0.0.1:80 0.000 0.001 0.000 200 200 34 366 \"GET http://www.example.com:80/ HTTP/1.1\" \"curl/7.46.0\" - - arn:aws:elasticloadbalancing:us-east-2:123456789012:targetgroup/my-targets/73e2d6bc24d8a067 \"Root=1-58337262-36d228ad5d99923122bbe354\" \"-\" \"-\" 0 2018-07-02T22:22:48.364000Z \"forward\" \"-\" \"-\" \"10.0.0.1:80\" \"200\" \"-\" \"-\"",
+            "aws.kinesis.type": "deliverystream",
+            "data_stream.type": "logs",
+            "aws.firehose.request_id": "971ae05f-a128-4a7f-b623-30f9bc513e55",
+            "cloud.provider": "aws",
+            "@timestamp": "2023-07-25T21:04:35Z",
+            "cloud.account.id": "123456",
+            "data_stream.dataset": "awsfirehose",
+            "aws.kinesis.name": "firehose-application-load-balancer-logs-to-elastic",
+            "event.id": "37670326805251200781477669690942747782212394134076063744"
+        },
+        {
+            "cloud.region": "us-east-1",
+            "aws.firehose.arn": "arn:aws:firehose:us-east-2:123456:deliverystream/firehose-network-load-balancer-logs-to-elastic",
+            "data_stream.namespace": "default",
+            "message": "tls 2.0 2018-12-20T02:59:40 net/my-network-loadbalancer/c6e77e28c25b2234 g3d4b5e8bb8464cd 72.21.218.154:51341 172.100.100.185:443 5 2 98 246 - arn:aws:acm:us-east-2:671290407336:certificate/2a108f19-aded-46b0-8493-c63eb1ef4a99 - ECDHE-RSA-AES128-SHA tlsv12 - my-network-loadbalancer-c6e77e28c25b2234.elb.us-east-2.amazonaws.com - - - 2018-12-20T02:59:30",
+            "aws.kinesis.type": "deliverystream",
+            "data_stream.type": "logs",
+            "aws.firehose.request_id": "971ae05f-a128-4a7f-b623-30f9bc513e55",
+            "cloud.provider": "aws",
+            "@timestamp": "2023-07-25T21:04:35Z",
+            "cloud.account.id": "123456",
+            "data_stream.dataset": "awsfirehose",
+            "aws.kinesis.name": "firehose-network-load-balancer-logs-to-elastic",
+            "event.id": "37670326805251200781477669690942747782212394134076063744"
+        }
+    ]
 }