diff --git a/packages/aws/_dev/build/build.yml b/packages/aws/_dev/build/build.yml index 49e8fdaa97d..2bfcfc223b0 100644 --- a/packages/aws/_dev/build/build.yml +++ b/packages/aws/_dev/build/build.yml @@ -1,3 +1,3 @@ dependencies: ecs: - reference: git@v8.10.0 + reference: "git@v8.11.0" diff --git a/packages/aws/_dev/build/docs/apigateway.md b/packages/aws/_dev/build/docs/apigateway.md index 241a73692f1..8661e0fd3c0 100644 --- a/packages/aws/_dev/build/docs/apigateway.md +++ b/packages/aws/_dev/build/docs/apigateway.md @@ -65,10 +65,18 @@ For step-by-step instructions on how to set up an integration, see the {{event "apigateway_metrics"}} +**ECS Field Reference** + +Please refer to the following [document](https://www.elastic.co/guide/en/ecs/current/ecs-field-reference.html) for detailed information on ECS fields. + {{fields "apigateway_metrics"}} ## Logs reference {{event "apigateway_logs"}} +**ECS Field Reference** + +Please refer to the following [document](https://www.elastic.co/guide/en/ecs/current/ecs-field-reference.html) for detailed information on ECS fields. + {{fields "apigateway_logs"}} \ No newline at end of file diff --git a/packages/aws/_dev/build/docs/billing.md b/packages/aws/_dev/build/docs/billing.md index b5cf249b2c8..18166bfd774 100644 --- a/packages/aws/_dev/build/docs/billing.md +++ b/packages/aws/_dev/build/docs/billing.md @@ -51,4 +51,8 @@ An example event for `billing` looks as following: {{event "billing"}} +**ECS Field Reference** + +Please refer to the following [document](https://www.elastic.co/guide/en/ecs/current/ecs-field-reference.html) for detailed information on ECS fields. + {{fields "billing"}} diff --git a/packages/aws/_dev/build/docs/cloudfront.md b/packages/aws/_dev/build/docs/cloudfront.md index c49ecbe5462..9af8571c49c 100644 --- a/packages/aws/_dev/build/docs/cloudfront.md +++ b/packages/aws/_dev/build/docs/cloudfront.md @@ -46,6 +46,10 @@ For step-by-step instructions on how to set up an integration, see the The `cloudfront` data stream collects standard logs (also called access logs) from AWS CloudFront. CloudFront standard logs provide detailed records about every request that’s made to a distribution. +**ECS Field Reference** + +Please refer to the following [document](https://www.elastic.co/guide/en/ecs/current/ecs-field-reference.html) for detailed information on ECS fields. + {{fields "cloudfront_logs"}} {{event "cloudfront_logs"}} \ No newline at end of file diff --git a/packages/aws/_dev/build/docs/cloudtrail.md b/packages/aws/_dev/build/docs/cloudtrail.md index 6d5dbd4f636..8b6d42c66a6 100644 --- a/packages/aws/_dev/build/docs/cloudtrail.md +++ b/packages/aws/_dev/build/docs/cloudtrail.md @@ -68,6 +68,10 @@ files to a specific Amazon S3 bucket. of the CloudTrail Digest S3 Objects you'd like to read. If blank, CloudTrail Digest logs will be skipped. +**ECS Field Reference** + +Please refer to the following [document](https://www.elastic.co/guide/en/ecs/current/ecs-field-reference.html) for detailed information on ECS fields. + {{fields "cloudtrail"}} {{event "cloudtrail"}} diff --git a/packages/aws/_dev/build/docs/cloudwatch.md b/packages/aws/_dev/build/docs/cloudwatch.md index 462a318566f..2a28a72e596 100644 --- a/packages/aws/_dev/build/docs/cloudwatch.md +++ b/packages/aws/_dev/build/docs/cloudwatch.md @@ -63,6 +63,10 @@ The `number_of_workers` setting defines the number of workers assigned to readin The `cloudwatch` data stream collects CloudWatch logs. Users can use Amazon CloudWatch logs to monitor, store, and access log files from different sources. +**ECS Field Reference** + +Please refer to the following [document](https://www.elastic.co/guide/en/ecs/current/ecs-field-reference.html) for detailed information on ECS fields. + {{fields "cloudwatch_logs"}} {{event "cloudwatch_logs"}} @@ -71,4 +75,8 @@ CloudWatch logs to monitor, store, and access log files from different sources. {{event "cloudwatch_metrics"}} +**ECS Field Reference** + +Please refer to the following [document](https://www.elastic.co/guide/en/ecs/current/ecs-field-reference.html) for detailed information on ECS fields. + {{fields "cloudwatch_metrics"}} diff --git a/packages/aws/_dev/build/docs/dynamodb.md b/packages/aws/_dev/build/docs/dynamodb.md index bedc46f5ece..dbb67c34110 100644 --- a/packages/aws/_dev/build/docs/dynamodb.md +++ b/packages/aws/_dev/build/docs/dynamodb.md @@ -45,4 +45,8 @@ An example event for `dynamodb` looks like this: {{event "dynamodb"}} +**ECS Field Reference** + +Please refer to the following [document](https://www.elastic.co/guide/en/ecs/current/ecs-field-reference.html) for detailed information on ECS fields. + {{fields "dynamodb"}} \ No newline at end of file diff --git a/packages/aws/_dev/build/docs/ebs.md b/packages/aws/_dev/build/docs/ebs.md index 468b6f3ca21..29c1cd031f2 100644 --- a/packages/aws/_dev/build/docs/ebs.md +++ b/packages/aws/_dev/build/docs/ebs.md @@ -45,4 +45,8 @@ An example event for `ebs` looks like this: {{event "ebs"}} +**ECS Field Reference** + +Please refer to the following [document](https://www.elastic.co/guide/en/ecs/current/ecs-field-reference.html) for detailed information on ECS fields. + {{fields "ebs"}} \ No newline at end of file diff --git a/packages/aws/_dev/build/docs/ec2.md b/packages/aws/_dev/build/docs/ec2.md index 9e113b01b94..e37a63e0892 100644 --- a/packages/aws/_dev/build/docs/ec2.md +++ b/packages/aws/_dev/build/docs/ec2.md @@ -66,6 +66,10 @@ For logs stored in S3, you must export logs from log groups to an Amazon S3 buck With this data stream, EC2 logs will be parsed into fields like `ip_address` and `process.name`. For logs from other services, please use the **AWS CloudWatch** integration. +**ECS Field Reference** + +Please refer to the following [document](https://www.elastic.co/guide/en/ecs/current/ecs-field-reference.html) for detailed information on ECS fields. + {{fields "ec2_logs"}} {{event "ec2_logs"}} @@ -74,4 +78,8 @@ and `process.name`. For logs from other services, please use the **AWS CloudWatc {{event "ec2_metrics"}} +**ECS Field Reference** + +Please refer to the following [document](https://www.elastic.co/guide/en/ecs/current/ecs-field-reference.html) for detailed information on ECS fields. + {{fields "ec2_metrics"}} \ No newline at end of file diff --git a/packages/aws/_dev/build/docs/ecs.md b/packages/aws/_dev/build/docs/ecs.md index 848ded786ce..cfd1631fa06 100644 --- a/packages/aws/_dev/build/docs/ecs.md +++ b/packages/aws/_dev/build/docs/ecs.md @@ -41,4 +41,8 @@ For step-by-step instructions on how to set up an integration, see the {{event "ecs_metrics"}} +**ECS Field Reference** + +Please refer to the following [document](https://www.elastic.co/guide/en/ecs/current/ecs-field-reference.html) for detailed information on ECS fields. + {{fields "ecs_metrics"}} \ No newline at end of file diff --git a/packages/aws/_dev/build/docs/elb.md b/packages/aws/_dev/build/docs/elb.md index a9d6eb44730..a2b24ae531b 100644 --- a/packages/aws/_dev/build/docs/elb.md +++ b/packages/aws/_dev/build/docs/elb.md @@ -69,6 +69,10 @@ The `number_of_workers` setting defines the number of workers assigned to readin The `elb` dataset collects logs from AWS ELBs. +**ECS Field Reference** + +Please refer to the following [document](https://www.elastic.co/guide/en/ecs/current/ecs-field-reference.html) for detailed information on ECS fields. + {{fields "elb_logs"}} {{event "elb_logs"}} @@ -77,4 +81,8 @@ The `elb` dataset collects logs from AWS ELBs. {{event "elb_metrics"}} +**ECS Field Reference** + +Please refer to the following [document](https://www.elastic.co/guide/en/ecs/current/ecs-field-reference.html) for detailed information on ECS fields. + {{fields "elb_metrics"}} diff --git a/packages/aws/_dev/build/docs/emr.md b/packages/aws/_dev/build/docs/emr.md index 4c88fa8b1d5..a19138ac91b 100644 --- a/packages/aws/_dev/build/docs/emr.md +++ b/packages/aws/_dev/build/docs/emr.md @@ -44,10 +44,18 @@ For step-by-step instructions on how to set up an integration, see the {{event "emr_metrics"}} +**ECS Field Reference** + +Please refer to the following [document](https://www.elastic.co/guide/en/ecs/current/ecs-field-reference.html) for detailed information on ECS fields. + {{fields "emr_metrics"}} ## Logs reference {{event "emr_logs"}} +**ECS Field Reference** + +Please refer to the following [document](https://www.elastic.co/guide/en/ecs/current/ecs-field-reference.html) for detailed information on ECS fields. + {{fields "emr_logs"}} \ No newline at end of file diff --git a/packages/aws/_dev/build/docs/firewall.md b/packages/aws/_dev/build/docs/firewall.md index 18abb52b471..c9e1d75ec6c 100644 --- a/packages/aws/_dev/build/docs/firewall.md +++ b/packages/aws/_dev/build/docs/firewall.md @@ -65,6 +65,10 @@ monitor network activity. {{event "firewall_logs" }} +**ECS Field Reference** + +Please refer to the following [document](https://www.elastic.co/guide/en/ecs/current/ecs-field-reference.html) for detailed information on ECS fields. + {{fields "firewall_logs"}} ## Metrics reference @@ -73,4 +77,8 @@ The `firewall_metrics` dataset collects AWS Network Firewall metrics. {{event "firewall_metrics" }} +**ECS Field Reference** + +Please refer to the following [document](https://www.elastic.co/guide/en/ecs/current/ecs-field-reference.html) for detailed information on ECS fields. + {{fields "firewall_metrics"}} \ No newline at end of file diff --git a/packages/aws/_dev/build/docs/guardduty.md b/packages/aws/_dev/build/docs/guardduty.md index bca25f58933..61729246d71 100644 --- a/packages/aws/_dev/build/docs/guardduty.md +++ b/packages/aws/_dev/build/docs/guardduty.md @@ -83,4 +83,8 @@ This is the [`GuardDuty`](https://docs.aws.amazon.com/guardduty/latest/APIRefere {{event "guardduty"}} +**ECS Field Reference** + +Please refer to the following [document](https://www.elastic.co/guide/en/ecs/current/ecs-field-reference.html) for detailed information on ECS fields. + {{fields "guardduty"}} diff --git a/packages/aws/_dev/build/docs/inspector.md b/packages/aws/_dev/build/docs/inspector.md index 56f3891910f..d82096ecf87 100644 --- a/packages/aws/_dev/build/docs/inspector.md +++ b/packages/aws/_dev/build/docs/inspector.md @@ -30,4 +30,8 @@ This is the [`Inspector`](https://docs.aws.amazon.com/inspector/v2/APIReference/ {{event "inspector"}} +**ECS Field Reference** + +Please refer to the following [document](https://www.elastic.co/guide/en/ecs/current/ecs-field-reference.html) for detailed information on ECS fields. + {{fields "inspector"}} \ No newline at end of file diff --git a/packages/aws/_dev/build/docs/kafka.md b/packages/aws/_dev/build/docs/kafka.md index 1a2cbac481b..548a3d82b85 100644 --- a/packages/aws/_dev/build/docs/kafka.md +++ b/packages/aws/_dev/build/docs/kafka.md @@ -45,4 +45,8 @@ For step-by-step instructions on how to set up an integration, see the {{event "kafka_metrics"}} +**ECS Field Reference** + +Please refer to the following [document](https://www.elastic.co/guide/en/ecs/current/ecs-field-reference.html) for detailed information on ECS fields. + {{fields "kafka_metrics"}} diff --git a/packages/aws/_dev/build/docs/kinesis.md b/packages/aws/_dev/build/docs/kinesis.md index 5f9d18d3c22..c122877c553 100644 --- a/packages/aws/_dev/build/docs/kinesis.md +++ b/packages/aws/_dev/build/docs/kinesis.md @@ -41,4 +41,8 @@ For step-by-step instructions on how to set up an integration, see the {{event "kinesis"}} +**ECS Field Reference** + +Please refer to the following [document](https://www.elastic.co/guide/en/ecs/current/ecs-field-reference.html) for detailed information on ECS fields. + {{fields "kinesis"}} \ No newline at end of file diff --git a/packages/aws/_dev/build/docs/lambda.md b/packages/aws/_dev/build/docs/lambda.md index c4c4c9eab01..1727626dc23 100644 --- a/packages/aws/_dev/build/docs/lambda.md +++ b/packages/aws/_dev/build/docs/lambda.md @@ -41,4 +41,8 @@ For step-by-step instructions on how to set up an integration, see the {{event "lambda"}} +**ECS Field Reference** + +Please refer to the following [document](https://www.elastic.co/guide/en/ecs/current/ecs-field-reference.html) for detailed information on ECS fields. + {{fields "lambda"}} \ No newline at end of file diff --git a/packages/aws/_dev/build/docs/natgateway.md b/packages/aws/_dev/build/docs/natgateway.md index c0ab9de8399..067f8c78f24 100644 --- a/packages/aws/_dev/build/docs/natgateway.md +++ b/packages/aws/_dev/build/docs/natgateway.md @@ -41,4 +41,8 @@ For step-by-step instructions on how to set up an integration, see the {{event "natgateway"}} +**ECS Field Reference** + +Please refer to the following [document](https://www.elastic.co/guide/en/ecs/current/ecs-field-reference.html) for detailed information on ECS fields. + {{fields "natgateway"}} \ No newline at end of file diff --git a/packages/aws/_dev/build/docs/rds.md b/packages/aws/_dev/build/docs/rds.md index c8d25b8c990..8e5756ac49c 100644 --- a/packages/aws/_dev/build/docs/rds.md +++ b/packages/aws/_dev/build/docs/rds.md @@ -41,4 +41,8 @@ For step-by-step instructions on how to set up an integration, see the {{event "rds"}} +**ECS Field Reference** + +Please refer to the following [document](https://www.elastic.co/guide/en/ecs/current/ecs-field-reference.html) for detailed information on ECS fields. + {{fields "rds"}} \ No newline at end of file diff --git a/packages/aws/_dev/build/docs/redshift.md b/packages/aws/_dev/build/docs/redshift.md index a48d7be32c1..1584b297d4d 100644 --- a/packages/aws/_dev/build/docs/redshift.md +++ b/packages/aws/_dev/build/docs/redshift.md @@ -41,4 +41,8 @@ For step-by-step instructions on how to set up an integration, see the {{ url "g {{event "redshift" }} +**ECS Field Reference** + +Please refer to the following [document](https://www.elastic.co/guide/en/ecs/current/ecs-field-reference.html) for detailed information on ECS fields. + {{fields "redshift"}} \ No newline at end of file diff --git a/packages/aws/_dev/build/docs/route53.md b/packages/aws/_dev/build/docs/route53.md index 01bd76771c9..fe918cceec3 100644 --- a/packages/aws/_dev/build/docs/route53.md +++ b/packages/aws/_dev/build/docs/route53.md @@ -69,6 +69,10 @@ See the [Route 53 Documentation](https://docs.aws.amazon.com/Route53/latest/Deve {{event "route53_public_logs"}} +**ECS Field Reference** + +Please refer to the following [document](https://www.elastic.co/guide/en/ecs/current/ecs-field-reference.html) for detailed information on ECS fields. + {{fields "route53_public_logs"}} ### Resolver logs @@ -87,4 +91,8 @@ See the [Route 53 Documentation](https://docs.aws.amazon.com/Route53/latest/Deve {{event "route53_resolver_logs"}} +**ECS Field Reference** + +Please refer to the following [document](https://www.elastic.co/guide/en/ecs/current/ecs-field-reference.html) for detailed information on ECS fields. + {{fields "route53_resolver_logs"}} \ No newline at end of file diff --git a/packages/aws/_dev/build/docs/s3.md b/packages/aws/_dev/build/docs/s3.md index 6b6dc6ffd53..83e92b87aed 100644 --- a/packages/aws/_dev/build/docs/s3.md +++ b/packages/aws/_dev/build/docs/s3.md @@ -51,6 +51,10 @@ Server access logs are useful for many applications. For example, access log information can be useful in security and access audits. It can also help users to learn about customer base and understand Amazon S3 bill. +**ECS Field Reference** + +Please refer to the following [document](https://www.elastic.co/guide/en/ecs/current/ecs-field-reference.html) for detailed information on ECS fields. + {{fields "s3access"}} {{event "s3access"}} @@ -61,10 +65,18 @@ to learn about customer base and understand Amazon S3 bill. {{event "s3_daily_storage"}} +**ECS Field Reference** + +Please refer to the following [document](https://www.elastic.co/guide/en/ecs/current/ecs-field-reference.html) for detailed information on ECS fields. + {{fields "s3_daily_storage"}} ### s3_request {{event "s3_request"}} +**ECS Field Reference** + +Please refer to the following [document](https://www.elastic.co/guide/en/ecs/current/ecs-field-reference.html) for detailed information on ECS fields. + {{fields "s3_request"}} diff --git a/packages/aws/_dev/build/docs/s3_storage_lens.md b/packages/aws/_dev/build/docs/s3_storage_lens.md index 537b2c40617..e541c05ea8c 100644 --- a/packages/aws/_dev/build/docs/s3_storage_lens.md +++ b/packages/aws/_dev/build/docs/s3_storage_lens.md @@ -40,4 +40,8 @@ For step-by-step instructions on how to set up an integration, see the {{ url "g {{event "s3_storage_lens"}} +**ECS Field Reference** + +Please refer to the following [document](https://www.elastic.co/guide/en/ecs/current/ecs-field-reference.html) for detailed information on ECS fields. + {{fields "s3_storage_lens"}} \ No newline at end of file diff --git a/packages/aws/_dev/build/docs/securityhub.md b/packages/aws/_dev/build/docs/securityhub.md index 5752d73ef94..18f9b80dd8a 100644 --- a/packages/aws/_dev/build/docs/securityhub.md +++ b/packages/aws/_dev/build/docs/securityhub.md @@ -31,6 +31,10 @@ This is the [`securityhub_findings`](https://docs.aws.amazon.com/securityhub/1.0 {{event "securityhub_findings"}} +**ECS Field Reference** + +Please refer to the following [document](https://www.elastic.co/guide/en/ecs/current/ecs-field-reference.html) for detailed information on ECS fields. + {{fields "securityhub_findings"}} ### Insights @@ -39,4 +43,8 @@ This is the [`securityhub_insights`](https://docs.aws.amazon.com/securityhub/1.0 {{event "securityhub_insights"}} +**ECS Field Reference** + +Please refer to the following [document](https://www.elastic.co/guide/en/ecs/current/ecs-field-reference.html) for detailed information on ECS fields. + {{fields "securityhub_insights"}} \ No newline at end of file diff --git a/packages/aws/_dev/build/docs/sns.md b/packages/aws/_dev/build/docs/sns.md index 58aeb7dfc92..9e3b5f69797 100644 --- a/packages/aws/_dev/build/docs/sns.md +++ b/packages/aws/_dev/build/docs/sns.md @@ -41,4 +41,8 @@ For step-by-step instructions on how to set up an integration, see the {{event "sns"}} +**ECS Field Reference** + +Please refer to the following [document](https://www.elastic.co/guide/en/ecs/current/ecs-field-reference.html) for detailed information on ECS fields. + {{fields "sns"}} \ No newline at end of file diff --git a/packages/aws/_dev/build/docs/sqs.md b/packages/aws/_dev/build/docs/sqs.md index b204bdfd5b7..012fe3b3d25 100644 --- a/packages/aws/_dev/build/docs/sqs.md +++ b/packages/aws/_dev/build/docs/sqs.md @@ -41,4 +41,8 @@ For step-by-step instructions on how to set up an integration, see the {{event "sqs"}} +**ECS Field Reference** + +Please refer to the following [document](https://www.elastic.co/guide/en/ecs/current/ecs-field-reference.html) for detailed information on ECS fields. + {{fields "sqs"}} \ No newline at end of file diff --git a/packages/aws/_dev/build/docs/transitgateway.md b/packages/aws/_dev/build/docs/transitgateway.md index 5446e519750..70ae7f48ff0 100644 --- a/packages/aws/_dev/build/docs/transitgateway.md +++ b/packages/aws/_dev/build/docs/transitgateway.md @@ -41,4 +41,8 @@ For step-by-step instructions on how to set up an integration, see the {{event "transitgateway"}} +**ECS Field Reference** + +Please refer to the following [document](https://www.elastic.co/guide/en/ecs/current/ecs-field-reference.html) for detailed information on ECS fields. + {{fields "transitgateway"}} \ No newline at end of file diff --git a/packages/aws/_dev/build/docs/usage.md b/packages/aws/_dev/build/docs/usage.md index c4aa7e24c94..3fe70bfe187 100644 --- a/packages/aws/_dev/build/docs/usage.md +++ b/packages/aws/_dev/build/docs/usage.md @@ -43,4 +43,8 @@ An example event for `usage`looks like this: {{event "usage"}} +**ECS Field Reference** + +Please refer to the following [document](https://www.elastic.co/guide/en/ecs/current/ecs-field-reference.html) for detailed information on ECS fields. + {{fields "usage"}} \ No newline at end of file diff --git a/packages/aws/_dev/build/docs/vpcflow.md b/packages/aws/_dev/build/docs/vpcflow.md index 0bd5fe7b7b5..118a668074d 100644 --- a/packages/aws/_dev/build/docs/vpcflow.md +++ b/packages/aws/_dev/build/docs/vpcflow.md @@ -80,6 +80,10 @@ The `number_of_workers` setting defines the number of workers assigned to readin > Note: The Parquet format is not supported. +**ECS Field Reference** + +Please refer to the following [document](https://www.elastic.co/guide/en/ecs/current/ecs-field-reference.html) for detailed information on ECS fields. + {{fields "vpcflow"}} {{event "vpcflow"}} diff --git a/packages/aws/_dev/build/docs/vpn.md b/packages/aws/_dev/build/docs/vpn.md index 8dba0169f3e..e1d383aef66 100644 --- a/packages/aws/_dev/build/docs/vpn.md +++ b/packages/aws/_dev/build/docs/vpn.md @@ -41,4 +41,8 @@ For step-by-step instructions on how to set up an integration, see the {{event "vpn"}} +**ECS Field Reference** + +Please refer to the following [document](https://www.elastic.co/guide/en/ecs/current/ecs-field-reference.html) for detailed information on ECS fields. + {{fields "vpn"}} \ No newline at end of file diff --git a/packages/aws/_dev/build/docs/waf.md b/packages/aws/_dev/build/docs/waf.md index 89d3b287640..3b53ca54815 100644 --- a/packages/aws/_dev/build/docs/waf.md +++ b/packages/aws/_dev/build/docs/waf.md @@ -63,6 +63,10 @@ The `number_of_workers` setting defines the number of workers assigned to readin The `waf` dataset is specifically for WAF logs. Export logs from Kinesis Data Firehose to Amazon S3 bucket which has SQS notification setup already. +**ECS Field Reference** + +Please refer to the following [document](https://www.elastic.co/guide/en/ecs/current/ecs-field-reference.html) for detailed information on ECS fields. + {{fields "waf"}} {{event "waf"}} \ No newline at end of file diff --git a/packages/aws/changelog.yml b/packages/aws/changelog.yml index 61af6c71a9f..1b59b181c2c 100644 --- a/packages/aws/changelog.yml +++ b/packages/aws/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "2.17.0" + changes: + - description: ECS version updated to 8.11.0. Update the kibana constraint to ^8.13.0. Modified the field definitions to remove ECS fields made redundant by the ecs@mappings component template. + type: enhancement + link: https://github.com/elastic/integrations/pull/10223 - version: "2.16.0" changes: - description: Add TargetResponseTime metric to ELB Application metrics. diff --git a/packages/aws/data_stream/apigateway_logs/_dev/test/pipeline/test-apigateway-http.log-expected.json b/packages/aws/data_stream/apigateway_logs/_dev/test/pipeline/test-apigateway-http.log-expected.json index fabbc92b67a..300cf5cf1bd 100644 --- a/packages/aws/data_stream/apigateway_logs/_dev/test/pipeline/test-apigateway-http.log-expected.json +++ b/packages/aws/data_stream/apigateway_logs/_dev/test/pipeline/test-apigateway-http.log-expected.json @@ -14,7 +14,7 @@ } }, "ecs": { - "version": "8.0.0" + "version": "8.11.0" }, "event": { "original": "{\"requestId\":\"GQIVriFLIAMEMsA=\",\"ip\":\"1.128.0.0\",\"requestTime\":\"09/Jun/2023:12:54:08 +0000\",\"httpMethod\":\"GET\",\"routeKey\":\"GET /\",\"status\":\"200\",\"protocol\":\"HTTP/1.1\",\"responseLength\":\"47140\"}" @@ -37,7 +37,7 @@ } }, "ecs": { - "version": "8.0.0" + "version": "8.11.0" }, "event": { "original": "{\"requestId\": \"IWeTChtboAMEVUQ=\",\"ip\": \"1.128.0.0\",\"requestTime\": \"20/Jul/2023:07:09:32 +0000\",\"httpMethod\": \"GET\",\"routeKey\": \"GET /\",\"status\": \"200\",\"protocol\": \"HTTP/1.1\",\"responseLength\": \"51880\"}" @@ -60,7 +60,7 @@ } }, "ecs": { - "version": "8.0.0" + "version": "8.11.0" }, "event": { "original": "{\"requestId\": \"IWvN1jOkoAMEVpg=\",\"ip\": \"1.128.0.0\",\"requestTime\": \"20/Jul/2023:09:05:02 +0000\",\"httpMethod\": \"GET\",\"routeKey\": \"GET /\",\"status\": \"200\",\"protocol\": \"HTTP/1.1\",\"responseLength\": \"51898\"}" @@ -86,7 +86,7 @@ } }, "ecs": { - "version": "8.0.0" + "version": "8.11.0" }, "event": { "original": "{\"requestId\":\"Iq8dHhlwIAMEV_g=\",\"ip\":\"1.128.0.0\",\"requestTime\":\"26/Jul/2023:12:13:33 +0000\",\"httpMethod\":\"GET\",\"routeKey\":\"GET /\",\"status\":\"200\",\"protocol\":\"HTTP/1.1\",\"responseLength\":\"51243\",\"stage\":\"$default\",\"apiId\":\"1ax3mj7iqf\",\"domainName\":\"1ax3mj7iqf.execute-api.us-east-1.amazonaws.com\"}" @@ -112,7 +112,7 @@ } }, "ecs": { - "version": "8.0.0" + "version": "8.11.0" }, "event": { "original": "{\"requestId\":\"Iq8dXiijIAMEV8Q=\",\"ip\":\"1.128.0.0\",\"requestTime\":\"26/Jul/2023:12:13:34 +0000\",\"httpMethod\":\"GET\",\"routeKey\":\"GET /\",\"status\":\"200\",\"protocol\":\"HTTP/1.1\",\"responseLength\":\"51205\",\"stage\":\"$default\",\"apiId\":\"1ax3mj7iqf\",\"domainName\":\"1ax3mj7iqf.execute-api.us-east-1.amazonaws.com\"}" diff --git a/packages/aws/data_stream/apigateway_logs/_dev/test/pipeline/test-apigateway-rest.log-expected.json b/packages/aws/data_stream/apigateway_logs/_dev/test/pipeline/test-apigateway-rest.log-expected.json index 01cb89ce7bb..be05962d1c3 100644 --- a/packages/aws/data_stream/apigateway_logs/_dev/test/pipeline/test-apigateway-rest.log-expected.json +++ b/packages/aws/data_stream/apigateway_logs/_dev/test/pipeline/test-apigateway-rest.log-expected.json @@ -16,7 +16,7 @@ } }, "ecs": { - "version": "8.0.0" + "version": "8.11.0" }, "event": { "original": "{\"requestId\":\"48752d0f-c99d-4cfa-a5a7-f3c6834d19e5\",\"ip\":\"1.128.0.0\",\"caller\":\"-\",\"user\":\"-\",\"requestTime\":\"10/Jun/2023:15:36:28 +0000\",\"httpMethod\":\"GET\",\"resourcePath\":\"/pets\",\"status\":\"200\",\"protocol\":\"HTTP/1.1\",\"responseLength\":\"184\"}" @@ -41,7 +41,7 @@ } }, "ecs": { - "version": "8.0.0" + "version": "8.11.0" }, "event": { "original": "{\"requestId\": \"caa4a500-2651-4476-aa1f-a639b858819b\",\"ip\": \"1.128.0.0\",\"caller\": \"-\",\"user\": \"-\",\"requestTime\": \"19/Jul/2023:16:15:46 +0000\",\"httpMethod\": \"GET\",\"resourcePath\": \"/pets/{petId}\",\"status\": \"200\",\"protocol\": \"HTTP/1.1\",\"responseLength\": \"49\"}" @@ -66,7 +66,7 @@ } }, "ecs": { - "version": "8.0.0" + "version": "8.11.0" }, "event": { "original": "{\"requestId\": \"c5a56ba0-fd42-4425-b7d2-5e8836563270\",\"ip\": \"1.128.0.0\",\"caller\": \"-\",\"user\": \"-\",\"requestTime\": \"19/Jul/2023:16:15:44 +0000\",\"httpMethod\": \"GET\",\"resourcePath\": \"/pets/{petId}\",\"status\": \"200\",\"protocol\": \"HTTP/1.1\",\"responseLength\": \"49\"}" @@ -94,7 +94,7 @@ } }, "ecs": { - "version": "8.0.0" + "version": "8.11.0" }, "event": { "original": "{\"requestId\":\"c12d1542-b79e-4e79-a158-fdf36452a81b\",\"ip\":\"1.128.0.0\",\"caller\":\"-\",\"user\":\"-\",\"requestTime\":\"26/Jul/2023:12:07:03 +0000\",\"httpMethod\":\"GET\",\"resourcePath\":\"/pets/{petId}\",\"status\":\"200\",\"protocol\":\"HTTP/1.1\",\"responseLength\":\"48\",\"stage\":\"prod\",\"apiId\":\"asad15n3p0\",\"domainName\":\"asad15n3p0.execute-api.us-east-1.amazonaws.com\"}" @@ -122,7 +122,7 @@ } }, "ecs": { - "version": "8.0.0" + "version": "8.11.0" }, "event": { "original": "{\"requestId\":\"868ee022-bc3a-4acf-97e7-acb7472b3235\",\"ip\":\"1.128.0.0\",\"caller\":\"-\",\"user\":\"-\",\"requestTime\":\"26/Jul/2023:12:06:57 +0000\",\"httpMethod\":\"GET\",\"resourcePath\":\"/\",\"status\":\"200\",\"protocol\":\"HTTP/1.1\",\"responseLength\":\"1310\",\"stage\":\"prod\",\"apiId\":\"asad15n3p0\",\"domainName\":\"asad15n3p0.execute-api.us-east-1.amazonaws.com\"}" diff --git a/packages/aws/data_stream/apigateway_logs/_dev/test/pipeline/test-apigateway-websocket.log-expected.json b/packages/aws/data_stream/apigateway_logs/_dev/test/pipeline/test-apigateway-websocket.log-expected.json index ba720fad1b2..299ca35d868 100644 --- a/packages/aws/data_stream/apigateway_logs/_dev/test/pipeline/test-apigateway-websocket.log-expected.json +++ b/packages/aws/data_stream/apigateway_logs/_dev/test/pipeline/test-apigateway-websocket.log-expected.json @@ -15,7 +15,7 @@ } }, "ecs": { - "version": "8.0.0" + "version": "8.11.0" }, "event": { "original": "{\"requestId\": \"REDACTED\",\"ip\": \"1.128.0.0\",\"caller\": \"-\",\"user\": \"-\",\"requestTime\": \"25/Jul/2023:16:26:02 +0000\",\"eventType\": \"CONNECT\",\"routeKey\": \"$connect\",\"status\": \"500\",\"connectionId\": \"REDACTED\"}" @@ -39,7 +39,7 @@ } }, "ecs": { - "version": "8.0.0" + "version": "8.11.0" }, "event": { "original": "{\"requestId\": \"177a5660-9b20-4614-b620-51dd4d56fb4c\",\"ip\": \"1.128.0.0\",\"caller\": \"-\",\"user\": \"-\",\"requestTime\": \"19/Jul/2023:16:17:17 +0000\",\"eventType\": \"-\",\"routeKey\": \"-\",\"status\": \"403\",\"connectionId\": \"-\"}" @@ -66,7 +66,7 @@ } }, "ecs": { - "version": "8.0.0" + "version": "8.11.0" }, "event": { "original": "{\"requestId\":\"Iq9gwFDNoAMFo1A=\",\"ip\":\"1.128.0.0\",\"caller\":\"-\",\"user\":\"-\",\"requestTime\":\"26/Jul/2023:12:20:46 +0000\",\"eventType\":\"CONNECT\",\"routeKey\":\"$connect\",\"status\":\"500\",\"connectionId\":\"Iq8gj1UmIAMCKpA=\",\"apiId\":\"z1ctxygne5\",\"stage\":\"production\",\"domainName\":\"z1ctxygne5.execute-api.us-east-1.amazonaws.com\"}" @@ -93,7 +93,7 @@ } }, "ecs": { - "version": "8.0.0" + "version": "8.11.0" }, "event": { "original": "{\"requestId\":\"Iq9gjE_aIAMFZTg=\",\"ip\":\"1.128.0.0\",\"caller\":\"-\",\"user\":\"-\",\"requestTime\":\"26/Jul/2023:12:20:44 +0000\",\"eventType\":\"CONNECT\",\"routeKey\":\"$connect\",\"status\":\"500\",\"connectionId\":\"Iq8gj1UmIAMCKpA=\",\"apiId\":\"z1ctxygne5\",\"stage\":\"production\",\"domainName\":\"z1ctxygne5.execute-api.us-east-1.amazonaws.com\"}" diff --git a/packages/aws/data_stream/apigateway_logs/elasticsearch/ingest_pipeline/default.yml b/packages/aws/data_stream/apigateway_logs/elasticsearch/ingest_pipeline/default.yml index 18b911d11cc..d1ba755f0b5 100644 --- a/packages/aws/data_stream/apigateway_logs/elasticsearch/ingest_pipeline/default.yml +++ b/packages/aws/data_stream/apigateway_logs/elasticsearch/ingest_pipeline/default.yml @@ -4,7 +4,7 @@ description: "Pipeline for API Gateway logs in CloudWatch" processors: - set: field: ecs.version - value: '8.0.0' + value: '8.11.0' - rename: field: message target_field: event.original diff --git a/packages/aws/data_stream/apigateway_logs/fields/ecs.yml b/packages/aws/data_stream/apigateway_logs/fields/ecs.yml deleted file mode 100644 index 217ec756ce4..00000000000 --- a/packages/aws/data_stream/apigateway_logs/fields/ecs.yml +++ /dev/null @@ -1,63 +0,0 @@ -- external: ecs - name: ecs.version -- external: ecs - name: error.message -- external: ecs - name: message -- external: ecs - name: tags -- external: ecs - name: host.architecture -- external: ecs - name: host.domain -- external: ecs - name: host.hostname -- external: ecs - name: host.id -- external: ecs - name: host.ip -- external: ecs - name: host.mac -- external: ecs - name: host.name -- external: ecs - name: host.os.family -- external: ecs - name: host.os.kernel -- external: ecs - name: host.os.name -- external: ecs - name: host.os.platform -- external: ecs - name: host.os.version -- external: ecs - name: host.type -- external: ecs - name: cloud.account.id -- external: ecs - name: cloud.availability_zone -- external: ecs - name: cloud.instance.id -- external: ecs - name: cloud.instance.name -- external: ecs - name: cloud.machine.type -- external: ecs - name: cloud.provider -- external: ecs - name: cloud.region -- external: ecs - name: cloud.project.id -- external: ecs - name: container.id -- external: ecs - name: container.image.name -- external: ecs - name: container.labels -- external: ecs - name: container.name -- external: ecs - name: log.file.path -- external: ecs - name: event.dataset - type: constant_keyword diff --git a/packages/aws/data_stream/apigateway_logs/sample_event.json b/packages/aws/data_stream/apigateway_logs/sample_event.json index 7d8c92364b5..51955956b65 100644 --- a/packages/aws/data_stream/apigateway_logs/sample_event.json +++ b/packages/aws/data_stream/apigateway_logs/sample_event.json @@ -38,7 +38,7 @@ "type": "logs" }, "ecs": { - "version": "8.0.0" + "version": "8.11.0" }, "elastic_agent": { "id": "acba78ef-1401-4689-977c-d8c2e5d6a8fa", diff --git a/packages/aws/data_stream/apigateway_metrics/fields/ecs.yml b/packages/aws/data_stream/apigateway_metrics/fields/ecs.yml index 654c9b2e202..303e169c534 100644 --- a/packages/aws/data_stream/apigateway_metrics/fields/ecs.yml +++ b/packages/aws/data_stream/apigateway_metrics/fields/ecs.yml @@ -1,70 +1,9 @@ -- external: ecs - name: cloud - external: ecs name: cloud.account.id dimension: true -- external: ecs - name: cloud.account.name -- external: ecs - name: cloud.availability_zone -- external: ecs - name: cloud.instance.id -- external: ecs - name: cloud.machine.type -- external: ecs - name: cloud.provider - external: ecs name: cloud.region dimension: true -- external: ecs - name: ecs.version -- external: ecs - name: error -- external: ecs - name: error.message -- external: ecs - name: service.type -- external: ecs - name: host.architecture -- external: ecs - name: host.domain -- external: ecs - name: host.hostname -- external: ecs - name: host.id -- external: ecs - name: host.ip -- external: ecs - name: host.mac -- external: ecs - name: host.name -- external: ecs - name: host.os.family -- external: ecs - name: host.os.kernel -- external: ecs - name: host.os.name -- external: ecs - name: host.os.platform -- external: ecs - name: host.os.version -- external: ecs - name: host.type -- external: ecs - name: cloud.instance.name -- external: ecs - name: cloud.project.id -- external: ecs - name: container.id -- external: ecs - name: container.image.name -- external: ecs - name: container.labels -- external: ecs - name: container.name - name: agent.id external: ecs dimension: true -- external: ecs - name: event.dataset - type: constant_keyword diff --git a/packages/aws/data_stream/apigateway_metrics/sample_event.json b/packages/aws/data_stream/apigateway_metrics/sample_event.json index 1c7ef75407b..ac8c5c7e01d 100644 --- a/packages/aws/data_stream/apigateway_metrics/sample_event.json +++ b/packages/aws/data_stream/apigateway_metrics/sample_event.json @@ -1,61 +1,12 @@ { + "@timestamp": "2023-05-08T16:30:00.000Z", "agent": { - "name": "docker-fleet-agent", + "ephemeral_id": "dfa418e2-1fe7-4039-9e44-bec39fa60341", "id": "fe8366bc-f3f8-4901-acce-b2c6788cf21f", + "name": "docker-fleet-agent", "type": "metricbeat", - "ephemeral_id": "dfa418e2-1fe7-4039-9e44-bec39fa60341", "version": "8.6.2" }, - "@timestamp": "2023-05-08T16:30:00.000Z", - "ecs": { - "version": "8.0.0" - }, - "data_stream": { - "namespace": "default", - "type": "metrics", - "dataset": "aws.apigateway_metrics" - }, - "service": { - "type": "aws" - }, - "host": { - "hostname": "docker-fleet-agent", - "os": { - "kernel": "5.15.90.1-microsoft-standard-WSL2", - "codename": "focal", - "name": "Ubuntu", - "family": "debian", - "type": "linux", - "version": "20.04.5 LTS (Focal Fossa)", - "platform": "ubuntu" - }, - "containerized": false, - "ip": [ - "172.18.0.7" - ], - "name": "docker-fleet-agent", - "id": "f91b175388d423fca58155815dfc2279", - "mac": [ - "02-42-AC-12-00-07" - ], - "architecture": "x86_64" - }, - "elastic_agent": { - "id": "fe8336bc-f3f1-4901-ac0a-b266788cf21f", - "version": "8.6.2", - "snapshot": false - }, - "metricset": { - "period": 300000, - "name": "cloudwatch" - }, - "event": { - "duration": 10830411419, - "agent_id_status": "verified", - "ingested": "2023-05-08T16:39:47Z", - "module": "aws", - "dataset": "aws.apigateway_metrics" - }, "aws": { "apigateway": { "metrics": { @@ -65,12 +16,12 @@ "5xx": { "sum": 0 }, - "DataProcessed": { - "avg": 48460 - }, "Count": { "sum": 2 }, + "DataProcessed": { + "avg": 48460 + }, "IntegrationLatency": { "avg": 85.5 }, @@ -85,5 +36,54 @@ "dimensions": { "ApiId": "6am7mj7jqx" } + }, + "data_stream": { + "dataset": "aws.apigateway_metrics", + "namespace": "default", + "type": "metrics" + }, + "ecs": { + "version": "8.11.0" + }, + "elastic_agent": { + "id": "fe8336bc-f3f1-4901-ac0a-b266788cf21f", + "snapshot": false, + "version": "8.6.2" + }, + "event": { + "agent_id_status": "verified", + "dataset": "aws.apigateway_metrics", + "duration": 10830411419, + "ingested": "2023-05-08T16:39:47Z", + "module": "aws" + }, + "host": { + "architecture": "x86_64", + "containerized": false, + "hostname": "docker-fleet-agent", + "id": "f91b175388d423fca58155815dfc2279", + "ip": [ + "172.18.0.7" + ], + "mac": [ + "02-42-AC-12-00-07" + ], + "name": "docker-fleet-agent", + "os": { + "codename": "focal", + "family": "debian", + "kernel": "5.15.90.1-microsoft-standard-WSL2", + "name": "Ubuntu", + "platform": "ubuntu", + "type": "linux", + "version": "20.04.5 LTS (Focal Fossa)" + } + }, + "metricset": { + "name": "cloudwatch", + "period": 300000 + }, + "service": { + "type": "aws" } } \ No newline at end of file diff --git a/packages/aws/data_stream/billing/fields/ecs.yml b/packages/aws/data_stream/billing/fields/ecs.yml index 442ec034a20..2d191287975 100644 --- a/packages/aws/data_stream/billing/fields/ecs.yml +++ b/packages/aws/data_stream/billing/fields/ecs.yml @@ -1,69 +1,6 @@ -- external: ecs - name: cloud - external: ecs name: cloud.account.id dimension: true -- external: ecs - name: cloud.account.name -- external: ecs - name: cloud.availability_zone -- external: ecs - name: cloud.instance.id -- external: ecs - name: cloud.instance.name -- external: ecs - name: cloud.machine.type -- external: ecs - name: cloud.provider -- external: ecs - name: cloud.region -- external: ecs - name: cloud.project.id -- external: ecs - name: container.id -- external: ecs - name: container.image.name -- external: ecs - name: container.labels -- external: ecs - name: container.name -- external: ecs - name: ecs.version -- external: ecs - name: error -- external: ecs - name: error.message -- external: ecs - name: service.type -- external: ecs - name: host.architecture -- external: ecs - name: host.domain -- external: ecs - name: host.hostname -- external: ecs - name: host.id -- external: ecs - name: host.ip -- external: ecs - name: host.mac -- external: ecs - name: host.name -- external: ecs - name: host.os.family -- external: ecs - name: host.os.kernel -- external: ecs - name: host.os.name -- external: ecs - name: host.os.platform -- external: ecs - name: host.os.version -- external: ecs - name: host.type - name: agent.id external: ecs dimension: true -- external: ecs - name: event.dataset - type: constant_keyword diff --git a/packages/aws/data_stream/billing/sample_event.json b/packages/aws/data_stream/billing/sample_event.json index 832bb00231c..4483585806c 100644 --- a/packages/aws/data_stream/billing/sample_event.json +++ b/packages/aws/data_stream/billing/sample_event.json @@ -1,30 +1,14 @@ { "@timestamp": "2020-05-28T17:17:06.212Z", - "cloud": { - "provider": "aws", - "region": "us-east-1", - "account": { - "id": "428152502467", - "name": "elastic-beats" - } - }, - "event": { - "dataset": "aws.billing", - "module": "aws", - "duration": 1938760247 - }, - "metricset": { - "name": "billing", - "period": 43200000 - }, - "ecs": { - "version": "1.5.0" + "agent": { + "ephemeral_id": "17803f33-b617-4ce9-a9ac-e218c02aeb4b", + "id": "12f376ef-5186-4e8b-a175-70f1140a8f30", + "name": "MacBook-Elastic.local", + "type": "metricbeat", + "version": "8.0.0" }, "aws": { "billing": { - "Currency": "USD", - "EstimatedCharges": 39.26, - "ServiceName": "AmazonEKS", "AmortizedCost": { "amount": 51.6, "unit": "USD" @@ -33,10 +17,13 @@ "amount": 51.6, "unit": "USD" }, + "Currency": "USD", + "EstimatedCharges": 39.26, "NormalizedUsageAmount": { "amount": 672, "unit": "N/A" }, + "ServiceName": "AmazonEKS", "UnblendedCost": { "amount": 51.6, "unit": "USD" @@ -47,14 +34,27 @@ } } }, + "cloud": { + "account": { + "id": "428152502467", + "name": "elastic-beats" + }, + "provider": "aws", + "region": "us-east-1" + }, + "ecs": { + "version": "8.11.0" + }, + "event": { + "dataset": "aws.billing", + "duration": 1938760247, + "module": "aws" + }, + "metricset": { + "name": "billing", + "period": 43200000 + }, "service": { "type": "aws" - }, - "agent": { - "id": "12f376ef-5186-4e8b-a175-70f1140a8f30", - "name": "MacBook-Elastic.local", - "type": "metricbeat", - "version": "8.0.0", - "ephemeral_id": "17803f33-b617-4ce9-a9ac-e218c02aeb4b" } } \ No newline at end of file diff --git a/packages/aws/data_stream/cloudfront_logs/_dev/test/pipeline/test-cloudfront.log-expected.json b/packages/aws/data_stream/cloudfront_logs/_dev/test/pipeline/test-cloudfront.log-expected.json index 27a43c0a8a8..6d1e4b341b4 100644 --- a/packages/aws/data_stream/cloudfront_logs/_dev/test/pipeline/test-cloudfront.log-expected.json +++ b/packages/aws/data_stream/cloudfront_logs/_dev/test/pipeline/test-cloudfront.log-expected.json @@ -23,10 +23,12 @@ "domain": "d111111abcdef8.cloudfront.net" }, "ecs": { - "version": "8.0.0" + "version": "8.11.0" }, "event": { - "category": "web", + "category": [ + "web" + ], "id": "SOX4xwn4XV6Q4rgb7XiVGOHms_BGlTAC4KyHmureZmBNrjGdRLiNIQ==", "kind": "event", "original": "2019-12-04\t21:02:31\tLAX1\t392\t89.160.20.112\tGET\td111111abcdef8.cloudfront.net\t/index.html\t200\t-\tMozilla/5.0%20(Windows%20NT%2010.0;%20Win64;%20x64)%20AppleWebKit/537.36%20(KHTML,%20like%20Gecko)%20Chrome/78.0.3904.108%20Safari/537.36\t-\t-\tHit\tSOX4xwn4XV6Q4rgb7XiVGOHms_BGlTAC4KyHmureZmBNrjGdRLiNIQ==\td111111abcdef8.cloudfront.net\thttps\t23\t0.001\t-\tTLSv1.2\tECDHE-RSA-AES128-GCM-SHA256\tHit\tHTTP/2.0\t-\t-\t11040\t0.001\tHit\ttext/html\t78\t-\t-", @@ -138,10 +140,12 @@ "domain": "d111111abcdef8.cloudfront.net" }, "ecs": { - "version": "8.0.0" + "version": "8.11.0" }, "event": { - "category": "web", + "category": [ + "web" + ], "id": "k6WGMNkEzR5BEM_SaF47gjtX9zBDO2m349OY2an0QPEaUum1ZOLrow==", "kind": "event", "original": "2019-12-04\t21:02:31\tLAX1\t392\t2a02:cf40:add:4002:91f2:a9b2:e09a:6fc6\tGET\td111111abcdef8.cloudfront.net\t/index.html\t200\t-\tMozilla/5.0%20(Windows%20NT%2010.0;%20Win64;%20x64)%20AppleWebKit/537.36%20(KHTML,%20like%20Gecko)%20Chrome/78.0.3904.108%20Safari/537.36\t-\t-\tHit\tk6WGMNkEzR5BEM_SaF47gjtX9zBDO2m349OY2an0QPEaUum1ZOLrow==\td111111abcdef8.cloudfront.net\thttps\t23\t0.000\t-\tTLSv1.2\tECDHE-RSA-AES128-GCM-SHA256\tHit\tHTTP/2.0\t-\t-\t11040\t0.000\tHit\ttext/html\t78\t-\t-", @@ -244,10 +248,12 @@ "domain": "d111111abcdef8.cloudfront.net" }, "ecs": { - "version": "8.0.0" + "version": "8.11.0" }, "event": { - "category": "web", + "category": [ + "web" + ], "id": "f37nTMVvnKvV2ZSvEsivup_c2kZ7VXzYdjC-GUQZ5qNs-89BlWazbw==", "kind": "event", "original": "2019-12-04\t21:02:31\tLAX1\t392\t89.160.20.112\tGET\td111111abcdef8.cloudfront.net\t/index.html\t200\t-\tMozilla/5.0%20(Windows%20NT%2010.0;%20Win64;%20x64)%20AppleWebKit/537.36%20(KHTML,%20like%20Gecko)%20Chrome/78.0.3904.108%20Safari/537.36\t-\t-\tHit\tf37nTMVvnKvV2ZSvEsivup_c2kZ7VXzYdjC-GUQZ5qNs-89BlWazbw==\td111111abcdef8.cloudfront.net\thttps\t23\t0.001\t-\tTLSv1.2\tECDHE-RSA-AES128-GCM-SHA256\tHit\tHTTP/2.0\t-\t-\t11040\t0.001\tHit\ttext/html\t78\t-\t-\t", @@ -359,10 +365,12 @@ "domain": "www.example.com" }, "ecs": { - "version": "8.0.0" + "version": "8.11.0" }, "event": { - "category": "web", + "category": [ + "web" + ], "id": "1pkpNfBQ39sYMnjjUQjmH2w1wdJnbHYTbag21o_3OfcQgPzdL2RSSQ==", "kind": "event", "original": "2019-12-13\t22:36:27\tSEA19-C1\t900\t89.160.20.112\tGET\td111111abcdef8.cloudfront.net\t/favicon.ico\t502\thttp://www.example.com/\tMozilla/5.0%20(Windows%20NT%2010.0;%20Win64;%20x64)%20AppleWebKit/537.36%20(KHTML,%20like%20Gecko)%20Chrome/78.0.3904.108%20Safari/537.36\t-\t-\tError\t1pkpNfBQ39sYMnjjUQjmH2w1wdJnbHYTbag21o_3OfcQgPzdL2RSSQ==\twww.example.com\thttp\t675\t0.102\t-\t-\t-\tError\tHTTP/1.1\t-\t-\t25260\t0.102\tOriginDnsError\ttext/html\t507\t-\t-", @@ -471,10 +479,12 @@ "domain": "www.example.com" }, "ecs": { - "version": "8.0.0" + "version": "8.11.0" }, "event": { - "category": "web", + "category": [ + "web" + ], "id": "3AqrZGCnF_g0-5KOvfA7c9XLcf4YGvMFSeFdIetR1N_2y8jSis8Zxg==", "kind": "event", "original": "2019-12-13\t22:36:26\tSEA19-C1\t900\t89.160.20.112\tGET\td111111abcdef8.cloudfront.net\t/\t502\t-\tMozilla/5.0%20(Windows%20NT%2010.0;%20Win64;%20x64)%20AppleWebKit/537.36%20(KHTML,%20like%20Gecko)%20Chrome/78.0.3904.108%20Safari/537.36\t-\t-\tError\t3AqrZGCnF_g0-5KOvfA7c9XLcf4YGvMFSeFdIetR1N_2y8jSis8Zxg==\twww.example.com\thttp\t735\t0.107\t-\t-\t-\tError\tHTTP/1.1\t-\t-\t3802\t0.107\tOriginDnsError\ttext/html\t507\t-\t-", @@ -581,10 +591,12 @@ "domain": "www.example.com" }, "ecs": { - "version": "8.0.0" + "version": "8.11.0" }, "event": { - "category": "web", + "category": [ + "web" + ], "id": "kBkDzGnceVtWHqSCqBUqtA_cEs2T3tFUBbnBNkB9El_uVRhHgcZfcw==", "kind": "event", "original": "2019-12-13\t22:37:02\tSEA19-C2\t900\t89.160.20.112\tGET\td111111abcdef8.cloudfront.net\t/\t502\t-\tcurl/7.55.1\t-\t-\tError\tkBkDzGnceVtWHqSCqBUqtA_cEs2T3tFUBbnBNkB9El_uVRhHgcZfcw==\twww.example.com\thttp\t387\t0.103\t-\t-\t-\tError\tHTTP/1.1\t-\t-\t12644\t0.103\tOriginDnsError\ttext/html\t507\t-\t-", @@ -686,10 +698,12 @@ "domain": "test.com" }, "ecs": { - "version": "8.0.0" + "version": "8.11.0" }, "event": { - "category": "web", + "category": [ + "web" + ], "id": "hrsHM5OM6sTIXUleC1G20YtDxMf5Cq0Jbz0pwhVpod2kgEn_W6akCQ==", "kind": "event", "original": "2022-04-19 12:29:36 SEA19-C2 10157 81.2.69.143 POST d111111abcdef8.cloudfront.net /getApplications 200 https://test.com/global Mozilla/5.0%20(Windows%20NT%2010.0;%20Win64;%20x64)%20AppleWebKit/537.36%20(KHTML,%20like%20Gecko)%20Chrome/100.0.4896.127%20Safari/537.36 source=global - Miss hrsHM5OM6sTIXUleC1G20YtDxMf5Cq0Jbz0pwhVpod2kgEn_W6akCQ== test.com https 1057 0.238 - TLSv1.3 TLS_AES_128_GCM_SHA256 Miss HTTP/2.0 - - 4203 0.238 Miss application/json;charset=UTF-8 - - -", @@ -793,10 +807,12 @@ "domain": "test.com" }, "ecs": { - "version": "8.0.0" + "version": "8.11.0" }, "event": { - "category": "web", + "category": [ + "web" + ], "id": "hrsHM5OM6sTIXUleC1G20YtDxMf5Cq0Jbz0pwhVpod2kgEn_W6akCQ==", "kind": "event", "original": "2022-04-19 12:29:36 SEA19-C2 10157 81.2.69.143 POST d111111abcdef8.cloudfront.net /getApplications 000 https://test.com/global Mozilla/5.0%20(Windows%20NT%2010.0;%20Win64;%20x64)%20AppleWebKit/537.36%20(KHTML,%20like%20Gecko)%20Chrome/100.0.4896.127%20Safari/537.36 source=global - Miss hrsHM5OM6sTIXUleC1G20YtDxMf5Cq0Jbz0pwhVpod2kgEn_W6akCQ== test.com https 1057 0.238 - TLSv1.3 TLS_AES_128_GCM_SHA256 Miss HTTP/2.0 - - 4203 0.238 Miss application/json;charset=UTF-8 - - -", @@ -900,10 +916,12 @@ "domain": "test.com" }, "ecs": { - "version": "8.0.0" + "version": "8.11.0" }, "event": { - "category": "web", + "category": [ + "web" + ], "id": "hrsHM5OM6sTIXUleC1G20YtDxMf5Cq0Jbz0pwhVpod2kgEn_W6akCQ==", "kind": "event", "original": "2022-11-15 08:43:04 SEA19-C2 10157 81.2.69.143 GET d111111abcdef8.cloudfront.net /getApplications 200 https://test.com/global Mozilla/5.0%20(X11;%20Linux%20x86_64)%20AppleWebKit/537.36%20(KHTML,%20like%20Gecko)%20HeadlessChrome/100.0.4896.88%20Safari/537.36 - - Miss hrsHM5OM6sTIXUleC1G20YtDxMf5Cq0Jbz0pwhVpod2kgEn_W6akCQ== test.com https 1057 0.093 81.2.69.142,216.160.83.56 TLSv1.2 ECDHE-RSA-AES128-GCM-SHA256 Miss HTTP/1.1 - - 33359 0.093 Miss application/javascript - - -", diff --git a/packages/aws/data_stream/cloudfront_logs/elasticsearch/ingest_pipeline/default.yml b/packages/aws/data_stream/cloudfront_logs/elasticsearch/ingest_pipeline/default.yml index 0ac7147b2ce..2242c25b7b6 100644 --- a/packages/aws/data_stream/cloudfront_logs/elasticsearch/ingest_pipeline/default.yml +++ b/packages/aws/data_stream/cloudfront_logs/elasticsearch/ingest_pipeline/default.yml @@ -4,16 +4,16 @@ description: "Pipeline for CloudFront standard access logs" processors: - set: field: ecs.version - value: '8.0.0' + value: '8.11.0' - set: field: event.kind value: event - set: field: event.category - value: web + value: ["web"] - append: field: event.type - value: access + value: ["access"] - set: field: cloud.provider value: aws diff --git a/packages/aws/data_stream/cloudfront_logs/fields/ecs.yml b/packages/aws/data_stream/cloudfront_logs/fields/ecs.yml deleted file mode 100644 index 8cdd02f00ba..00000000000 --- a/packages/aws/data_stream/cloudfront_logs/fields/ecs.yml +++ /dev/null @@ -1,159 +0,0 @@ -- external: ecs - name: tags -- external: ecs - name: ecs.version -- external: ecs - name: error.message -- external: ecs - name: event.duration -- external: ecs - name: destination.address -- external: ecs - name: destination.domain -- external: ecs - name: http.request.method -- external: ecs - name: http.request.bytes -- external: ecs - name: http.request.id -- external: ecs - name: http.request.referrer -- external: ecs - name: http.response.body.bytes -- external: ecs - name: http.response.bytes -- external: ecs - name: http.response.status_code -- external: ecs - name: http.version -- external: ecs - name: log.file.path -- external: ecs - name: network.forwarded_ip -- external: ecs - name: network.protocol -- external: ecs - name: network.type -- external: ecs - name: related.ip -- external: ecs - name: related.hosts -- external: ecs - name: source.address -- external: ecs - name: source.ip -- external: ecs - name: source.port -- external: ecs - name: source.as.number -- external: ecs - name: source.as.organization.name -- external: ecs - name: source.geo.city_name -- external: ecs - name: source.geo.continent_name -- external: ecs - name: source.geo.country_iso_code -- external: ecs - name: source.geo.country_name -- description: Longitude and latitude. - level: core - name: source.geo.location - type: geo_point -- external: ecs - name: source.geo.region_iso_code -- external: ecs - name: source.geo.region_name -- external: ecs - name: tls.cipher -- external: ecs - name: tls.version -- external: ecs - name: tls.version_protocol -- external: ecs - name: url.domain -- external: ecs - name: url.extension -- external: ecs - name: url.original -- external: ecs - name: url.path -- external: ecs - name: url.query -- external: ecs - name: url.scheme -- external: ecs - name: url.full -- external: ecs - name: url.registered_domain -- external: ecs - name: url.subdomain -- external: ecs - name: url.top_level_domain -- external: ecs - name: user_agent.device.name -- external: ecs - name: user_agent.name -- external: ecs - name: user_agent.original -- external: ecs - name: user_agent.os.full -- external: ecs - name: user_agent.os.name -- external: ecs - name: user_agent.os.version -- external: ecs - name: user_agent.version -- external: ecs - name: host.architecture -- external: ecs - name: host.domain -- external: ecs - name: host.hostname -- external: ecs - name: host.id -- external: ecs - name: host.ip -- external: ecs - name: host.mac -- external: ecs - name: host.name -- external: ecs - name: host.os.family -- external: ecs - name: host.os.kernel -- external: ecs - name: host.os.name -- external: ecs - name: host.os.platform -- external: ecs - name: host.os.version -- external: ecs - name: host.type -- external: ecs - name: cloud.account.id -- external: ecs - name: cloud.availability_zone -- external: ecs - name: cloud.instance.id -- external: ecs - name: cloud.instance.name -- external: ecs - name: cloud.machine.type -- external: ecs - name: cloud.provider -- external: ecs - name: cloud.region -- external: ecs - name: cloud.project.id -- external: ecs - name: container.id -- external: ecs - name: container.image.name -- external: ecs - name: container.labels -- external: ecs - name: container.name -- external: ecs - name: event.dataset - type: constant_keyword diff --git a/packages/aws/data_stream/cloudfront_logs/sample_event.json b/packages/aws/data_stream/cloudfront_logs/sample_event.json index fc3badc50de..6d37f67b51e 100644 --- a/packages/aws/data_stream/cloudfront_logs/sample_event.json +++ b/packages/aws/data_stream/cloudfront_logs/sample_event.json @@ -41,7 +41,7 @@ "domain": "d111111abcdef8.cloudfront.net" }, "ecs": { - "version": "8.0.0" + "version": "8.11.0" }, "elastic_agent": { "id": "acba78ef-1401-4689-977c-d8c2e5d6a8fa", @@ -50,7 +50,9 @@ }, "event": { "agent_id_status": "verified", - "category": "web", + "category": [ + "web" + ], "dataset": "aws.cloudfront_logs", "id": "SOX4xwn4XV6Q4rgb7XiVGOHms_BGlTAC4KyHmureZmBNrjGdRLiNIQ==", "ingested": "2023-11-03T13:01:05Z", diff --git a/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-add-user-to-group-json.log-expected.json b/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-add-user-to-group-json.log-expected.json index 17f4ea62411..f732eddc1e8 100644 --- a/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-add-user-to-group-json.log-expected.json +++ b/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-add-user-to-group-json.log-expected.json @@ -30,7 +30,7 @@ "region": "us-east-2" }, "ecs": { - "version": "8.0.0" + "version": "8.11.0" }, "event": { "action": "AddUserToGroup", diff --git a/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-assume-role-json.log-expected.json b/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-assume-role-json.log-expected.json index e614c56fa06..826c32604c3 100644 --- a/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-assume-role-json.log-expected.json +++ b/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-assume-role-json.log-expected.json @@ -76,7 +76,7 @@ "region": "us-east-2" }, "ecs": { - "version": "8.0.0" + "version": "8.11.0" }, "event": { "action": "AssumeRole", @@ -211,7 +211,7 @@ "region": "us-east-2" }, "ecs": { - "version": "8.0.0" + "version": "8.11.0" }, "event": { "action": "AssumeRole", diff --git a/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-change-password-json.log-expected.json b/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-change-password-json.log-expected.json index 33771e58e8d..bd43a032680 100644 --- a/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-change-password-json.log-expected.json +++ b/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-change-password-json.log-expected.json @@ -25,7 +25,7 @@ "region": "us-east-1" }, "ecs": { - "version": "8.0.0" + "version": "8.11.0" }, "event": { "action": "ChangePassword", @@ -91,7 +91,7 @@ "region": "us-east-1" }, "ecs": { - "version": "8.0.0" + "version": "8.11.0" }, "event": { "action": "ChangePassword", diff --git a/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-cloudtrail-digest-json.log-expected.json b/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-cloudtrail-digest-json.log-expected.json index a1a070ba8c2..c572d8442ac 100644 --- a/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-cloudtrail-digest-json.log-expected.json +++ b/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-cloudtrail-digest-json.log-expected.json @@ -122,7 +122,7 @@ } }, "ecs": { - "version": "8.0.0" + "version": "8.11.0" }, "event": { "created": "2021-11-11T01:02:03.123456789Z", diff --git a/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-console-login-json.log-expected.json b/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-console-login-json.log-expected.json index 2078d86b428..813a95eba75 100644 --- a/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-console-login-json.log-expected.json +++ b/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-console-login-json.log-expected.json @@ -37,7 +37,7 @@ "region": "us-east-2" }, "ecs": { - "version": "8.0.0" + "version": "8.11.0" }, "event": { "action": "ConsoleLogin", @@ -140,7 +140,7 @@ "region": "us-east-2" }, "ecs": { - "version": "8.0.0" + "version": "8.11.0" }, "event": { "action": "ConsoleLogin", @@ -253,7 +253,7 @@ "region": "us-east-2" }, "ecs": { - "version": "8.0.0" + "version": "8.11.0" }, "event": { "action": "ConsoleLogin", diff --git a/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-create-access-key-json.log-expected.json b/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-create-access-key-json.log-expected.json index b25be1421f3..6fc71a8cbd3 100644 --- a/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-create-access-key-json.log-expected.json +++ b/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-create-access-key-json.log-expected.json @@ -42,7 +42,7 @@ "region": "us-east-1" }, "ecs": { - "version": "8.0.0" + "version": "8.11.0" }, "event": { "action": "CreateAccessKey", diff --git a/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-create-group-json.log-expected.json b/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-create-group-json.log-expected.json index 09020c1e017..ac85bed6f03 100644 --- a/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-create-group-json.log-expected.json +++ b/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-create-group-json.log-expected.json @@ -43,7 +43,7 @@ "region": "us-east-1" }, "ecs": { - "version": "8.0.0" + "version": "8.11.0" }, "event": { "action": "CreateGroup", @@ -119,7 +119,7 @@ "region": "us-east-1" }, "ecs": { - "version": "8.0.0" + "version": "8.11.0" }, "event": { "action": "CreateGroup", diff --git a/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-create-key-pair-json.log-expected.json b/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-create-key-pair-json.log-expected.json index 644d7b6fc61..3f6e1d74fab 100644 --- a/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-create-key-pair-json.log-expected.json +++ b/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-create-key-pair-json.log-expected.json @@ -35,7 +35,7 @@ "region": "us-east-2" }, "ecs": { - "version": "8.0.0" + "version": "8.11.0" }, "event": { "action": "CreateKeyPair", diff --git a/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-create-trail-json.log-expected.json b/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-create-trail-json.log-expected.json index 556ebd11206..b39e1ecb901 100644 --- a/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-create-trail-json.log-expected.json +++ b/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-create-trail-json.log-expected.json @@ -49,7 +49,7 @@ "region": "us-west-2" }, "ecs": { - "version": "8.0.0" + "version": "8.11.0" }, "event": { "action": "CreateTrail", diff --git a/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-create-user-json.log-expected.json b/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-create-user-json.log-expected.json index 50b75cf28e6..2139a0d0fd7 100644 --- a/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-create-user-json.log-expected.json +++ b/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-create-user-json.log-expected.json @@ -35,7 +35,7 @@ "region": "us-east-2" }, "ecs": { - "version": "8.0.0" + "version": "8.11.0" }, "event": { "action": "CreateUser", diff --git a/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-create-virtual-mfa-device-json.log-expected.json b/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-create-virtual-mfa-device-json.log-expected.json index 336badf2579..12544c46bd8 100644 --- a/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-create-virtual-mfa-device-json.log-expected.json +++ b/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-create-virtual-mfa-device-json.log-expected.json @@ -39,7 +39,7 @@ "region": "us-east-1" }, "ecs": { - "version": "8.0.0" + "version": "8.11.0" }, "event": { "action": "CreateVirtualMFADevice", diff --git a/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-deactivate-mfa-device-json.log-expected.json b/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-deactivate-mfa-device-json.log-expected.json index 700279be629..2e817388ddb 100644 --- a/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-deactivate-mfa-device-json.log-expected.json +++ b/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-deactivate-mfa-device-json.log-expected.json @@ -34,7 +34,7 @@ "region": "us-east-1" }, "ecs": { - "version": "8.0.0" + "version": "8.11.0" }, "event": { "action": "DeactivateMFADevice", diff --git a/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-delete-access-key-json.log-expected.json b/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-delete-access-key-json.log-expected.json index a5980384476..aee22039446 100644 --- a/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-delete-access-key-json.log-expected.json +++ b/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-delete-access-key-json.log-expected.json @@ -34,7 +34,7 @@ "region": "us-east-1" }, "ecs": { - "version": "8.0.0" + "version": "8.11.0" }, "event": { "action": "DeleteAccessKey", diff --git a/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-delete-bucket-json.log-expected.json b/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-delete-bucket-json.log-expected.json index 14011e490e5..f5f17cb8b81 100644 --- a/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-delete-bucket-json.log-expected.json +++ b/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-delete-bucket-json.log-expected.json @@ -38,7 +38,7 @@ "region": "us-east-2" }, "ecs": { - "version": "8.0.0" + "version": "8.11.0" }, "event": { "action": "DeleteBucket", diff --git a/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-delete-group-json.log-expected.json b/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-delete-group-json.log-expected.json index 18fa1e14204..5bcfd5d63e5 100644 --- a/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-delete-group-json.log-expected.json +++ b/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-delete-group-json.log-expected.json @@ -33,7 +33,7 @@ "region": "us-east-1" }, "ecs": { - "version": "8.0.0" + "version": "8.11.0" }, "event": { "action": "DeleteGroup", @@ -108,7 +108,7 @@ "region": "us-east-1" }, "ecs": { - "version": "8.0.0" + "version": "8.11.0" }, "event": { "action": "DeleteGroup", diff --git a/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-delete-ssh-public-key-json.log-expected.json b/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-delete-ssh-public-key-json.log-expected.json index 8b75bfcfd10..b4aa347f5ce 100644 --- a/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-delete-ssh-public-key-json.log-expected.json +++ b/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-delete-ssh-public-key-json.log-expected.json @@ -34,7 +34,7 @@ "region": "us-east-1" }, "ecs": { - "version": "8.0.0" + "version": "8.11.0" }, "event": { "action": "DeleteSSHPublicKey", diff --git a/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-delete-trail-json.log-expected.json b/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-delete-trail-json.log-expected.json index be8ef8955d2..874388fddba 100644 --- a/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-delete-trail-json.log-expected.json +++ b/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-delete-trail-json.log-expected.json @@ -29,7 +29,7 @@ "region": "us-west-2" }, "ecs": { - "version": "8.0.0" + "version": "8.11.0" }, "event": { "action": "DeleteTrail", diff --git a/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-delete-user-json.log-expected.json b/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-delete-user-json.log-expected.json index 978fea1e0be..a2c6c8d036f 100644 --- a/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-delete-user-json.log-expected.json +++ b/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-delete-user-json.log-expected.json @@ -33,7 +33,7 @@ "region": "us-east-1" }, "ecs": { - "version": "8.0.0" + "version": "8.11.0" }, "event": { "action": "DeleteUser", diff --git a/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-delete-virtual-mfa-device-json.log-expected.json b/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-delete-virtual-mfa-device-json.log-expected.json index f0d8869bfbb..d5f78f023e2 100644 --- a/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-delete-virtual-mfa-device-json.log-expected.json +++ b/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-delete-virtual-mfa-device-json.log-expected.json @@ -33,7 +33,7 @@ "region": "us-east-1" }, "ecs": { - "version": "8.0.0" + "version": "8.11.0" }, "event": { "action": "DeleteVirtualMFADevice", diff --git a/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-enable-mfa-device-json.log-expected.json b/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-enable-mfa-device-json.log-expected.json index 3c715b00ff6..84625796a4f 100644 --- a/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-enable-mfa-device-json.log-expected.json +++ b/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-enable-mfa-device-json.log-expected.json @@ -33,7 +33,7 @@ "region": "us-east-1" }, "ecs": { - "version": "8.0.0" + "version": "8.11.0" }, "event": { "action": "EnableMFADevice", diff --git a/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-insight-json.log-expected.json b/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-insight-json.log-expected.json index 234e6baf3a2..90b0b73e1b8 100644 --- a/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-insight-json.log-expected.json +++ b/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-insight-json.log-expected.json @@ -63,7 +63,7 @@ "region": "us-east-1" }, "ecs": { - "version": "8.0.0" + "version": "8.11.0" }, "event": { "created": "2021-11-11T01:02:03.123456789Z", diff --git a/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-remove-user-from-group-json.log-expected.json b/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-remove-user-from-group-json.log-expected.json index 14e16c89a6d..12f5de7c6a0 100644 --- a/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-remove-user-from-group-json.log-expected.json +++ b/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-remove-user-from-group-json.log-expected.json @@ -34,7 +34,7 @@ "region": "us-east-1" }, "ecs": { - "version": "8.0.0" + "version": "8.11.0" }, "event": { "action": "RemoveUserFromGroup", diff --git a/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-start-logging-json.log-expected.json b/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-start-logging-json.log-expected.json index b14fc675bab..3e0a4bdcee2 100644 --- a/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-start-logging-json.log-expected.json +++ b/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-start-logging-json.log-expected.json @@ -34,7 +34,7 @@ "region": "us-west-2" }, "ecs": { - "version": "8.0.0" + "version": "8.11.0" }, "event": { "action": "StartLogging", diff --git a/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-stop-logging-json.log-expected.json b/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-stop-logging-json.log-expected.json index ca541c2cdb7..bfe46a69d64 100644 --- a/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-stop-logging-json.log-expected.json +++ b/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-stop-logging-json.log-expected.json @@ -34,7 +34,7 @@ "region": "us-west-2" }, "ecs": { - "version": "8.0.0" + "version": "8.11.0" }, "event": { "action": "StopLogging", diff --git a/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-tls-details-json.log-expected.json b/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-tls-details-json.log-expected.json index 190fd43c854..db0467b3d5b 100644 --- a/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-tls-details-json.log-expected.json +++ b/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-tls-details-json.log-expected.json @@ -45,7 +45,7 @@ "region": "us-east-1" }, "ecs": { - "version": "8.0.0" + "version": "8.11.0" }, "event": { "action": "UploadSSHPublicKey", diff --git a/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-update-access-key-json.log-expected.json b/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-update-access-key-json.log-expected.json index 6a7e008df1c..9854080e05a 100644 --- a/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-update-access-key-json.log-expected.json +++ b/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-update-access-key-json.log-expected.json @@ -35,7 +35,7 @@ "region": "us-east-1" }, "ecs": { - "version": "8.0.0" + "version": "8.11.0" }, "event": { "action": "UpdateAccessKey", diff --git a/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-update-accout-password-policy-json.log-expected.json b/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-update-accout-password-policy-json.log-expected.json index d698c2d2f9b..3990f94963a 100644 --- a/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-update-accout-password-policy-json.log-expected.json +++ b/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-update-accout-password-policy-json.log-expected.json @@ -38,7 +38,7 @@ "region": "us-east-1" }, "ecs": { - "version": "8.0.0" + "version": "8.11.0" }, "event": { "action": "UpdateAccountPasswordPolicy", diff --git a/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-update-group-json.log-expected.json b/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-update-group-json.log-expected.json index a36058d97c2..c7c2d9593aa 100644 --- a/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-update-group-json.log-expected.json +++ b/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-update-group-json.log-expected.json @@ -29,7 +29,7 @@ "region": "us-east-1" }, "ecs": { - "version": "8.0.0" + "version": "8.11.0" }, "event": { "action": "UpdateGroup", @@ -106,7 +106,7 @@ "region": "us-east-1" }, "ecs": { - "version": "8.0.0" + "version": "8.11.0" }, "event": { "action": "UpdateGroup", diff --git a/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-update-login-profile-json.log-expected.json b/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-update-login-profile-json.log-expected.json index eb6e5218b52..f8a9bb013e4 100644 --- a/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-update-login-profile-json.log-expected.json +++ b/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-update-login-profile-json.log-expected.json @@ -33,7 +33,7 @@ "region": "us-east-1" }, "ecs": { - "version": "8.0.0" + "version": "8.11.0" }, "event": { "action": "UpdateLoginProfile", diff --git a/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-update-ssh-public-key-json.log-expected.json b/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-update-ssh-public-key-json.log-expected.json index 4f35fd51e41..1db33a2b229 100644 --- a/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-update-ssh-public-key-json.log-expected.json +++ b/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-update-ssh-public-key-json.log-expected.json @@ -35,7 +35,7 @@ "region": "us-east-1" }, "ecs": { - "version": "8.0.0" + "version": "8.11.0" }, "event": { "action": "UpdateSSHPublicKey", @@ -116,7 +116,7 @@ "region": "us-east-1" }, "ecs": { - "version": "8.0.0" + "version": "8.11.0" }, "event": { "action": "UpdateSSHPublicKey", diff --git a/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-update-trail-json.log-expected.json b/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-update-trail-json.log-expected.json index 8861e837639..ffcf8c91acf 100644 --- a/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-update-trail-json.log-expected.json +++ b/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-update-trail-json.log-expected.json @@ -30,7 +30,7 @@ "region": "us-east-2" }, "ecs": { - "version": "8.0.0" + "version": "8.11.0" }, "event": { "action": "UpdateTrail", @@ -137,7 +137,7 @@ "region": "us-west-2" }, "ecs": { - "version": "8.0.0" + "version": "8.11.0" }, "event": { "action": "UpdateTrail", diff --git a/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-update-user-json.log-expected.json b/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-update-user-json.log-expected.json index 9c0083f3326..4ba9aa8b8a9 100644 --- a/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-update-user-json.log-expected.json +++ b/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-update-user-json.log-expected.json @@ -29,7 +29,7 @@ "region": "us-east-1" }, "ecs": { - "version": "8.0.0" + "version": "8.11.0" }, "event": { "action": "UpdateUser", diff --git a/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-upload-ssh-public-key-json.log-expected.json b/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-upload-ssh-public-key-json.log-expected.json index ccaeacdd84e..92bf119519b 100644 --- a/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-upload-ssh-public-key-json.log-expected.json +++ b/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-upload-ssh-public-key-json.log-expected.json @@ -45,7 +45,7 @@ "region": "us-east-1" }, "ecs": { - "version": "8.0.0" + "version": "8.11.0" }, "event": { "action": "UploadSSHPublicKey", diff --git a/packages/aws/data_stream/cloudtrail/elasticsearch/ingest_pipeline/default.yml b/packages/aws/data_stream/cloudtrail/elasticsearch/ingest_pipeline/default.yml index 91138ed43ec..d3c401c627d 100644 --- a/packages/aws/data_stream/cloudtrail/elasticsearch/ingest_pipeline/default.yml +++ b/packages/aws/data_stream/cloudtrail/elasticsearch/ingest_pipeline/default.yml @@ -23,7 +23,7 @@ processors: name: '{{ IngestPipeline "third-party" }}' - set: field: ecs.version - value: '8.0.0' + value: '8.11.0' - date: field: json.eventTime target_field: "@timestamp" diff --git a/packages/aws/data_stream/cloudtrail/fields/ecs.yml b/packages/aws/data_stream/cloudtrail/fields/ecs.yml deleted file mode 100644 index 11696850162..00000000000 --- a/packages/aws/data_stream/cloudtrail/fields/ecs.yml +++ /dev/null @@ -1,149 +0,0 @@ -- external: ecs - name: cloud.account.id -- external: ecs - name: cloud.region -- external: ecs - name: ecs.version -- external: ecs - name: error.message -- external: ecs - name: event.action -- external: ecs - name: event.created -- external: ecs - name: event.ingested -- external: ecs - name: event.kind -- external: ecs - name: event.original -- external: ecs - name: event.provider -- external: ecs - name: event.type -- external: ecs - name: file.hash.md5 -- external: ecs - name: file.hash.sha1 -- external: ecs - name: file.hash.sha256 -- external: ecs - name: file.hash.sha512 -- external: ecs - name: file.path -- external: ecs - name: group.id -- external: ecs - name: group.name -- external: ecs - name: log.file.path -- external: ecs - name: related.hash -- external: ecs - name: related.user -- external: ecs - name: source.address -- external: ecs - name: source.as.number -- external: ecs - name: source.as.organization.name -- external: ecs - name: source.geo.city_name -- external: ecs - name: source.geo.continent_name -- external: ecs - name: source.geo.country_iso_code -- external: ecs - name: source.geo.country_name -- description: Longitude and latitude. - level: core - name: source.geo.location - type: geo_point -- external: ecs - name: source.geo.region_iso_code -- external: ecs - name: source.geo.region_name -- external: ecs - name: source.ip -- external: ecs - name: tags -- external: ecs - name: user.changes.name -- external: ecs - name: user.id -- external: ecs - name: user.name -- external: ecs - name: user.target.id -- external: ecs - name: user.target.name -- external: ecs - name: user_agent.device.name -- external: ecs - name: user_agent.name -- external: ecs - name: user_agent.original -- external: ecs - name: user_agent.os.full -- external: ecs - name: user_agent.os.name -- external: ecs - name: user_agent.os.version -- external: ecs - name: user_agent.version -- external: ecs - name: host.architecture -- external: ecs - name: host.domain -- external: ecs - name: host.hostname -- external: ecs - name: host.id -- external: ecs - name: host.ip -- external: ecs - name: host.mac -- external: ecs - name: host.name -- external: ecs - name: host.os.family -- external: ecs - name: host.os.kernel -- external: ecs - name: host.os.name -- external: ecs - name: host.os.platform -- external: ecs - name: host.os.version -- external: ecs - name: host.type -- external: ecs - name: cloud.availability_zone -- external: ecs - name: cloud.instance.id -- external: ecs - name: cloud.instance.name -- external: ecs - name: cloud.machine.type -- external: ecs - name: cloud.provider -- external: ecs - name: cloud.project.id -- external: ecs - name: container.id -- external: ecs - name: container.image.name -- external: ecs - name: container.labels -- external: ecs - name: container.name -- external: ecs - name: tls.version -- external: ecs - name: tls.version_protocol -- external: ecs - name: tls.cipher -- external: ecs - name: tls.client.server_name -- external: ecs - name: event.dataset - type: constant_keyword diff --git a/packages/aws/data_stream/cloudtrail/sample_event.json b/packages/aws/data_stream/cloudtrail/sample_event.json index a62dbcba2a1..dd727908bfd 100644 --- a/packages/aws/data_stream/cloudtrail/sample_event.json +++ b/packages/aws/data_stream/cloudtrail/sample_event.json @@ -142,7 +142,7 @@ "type": "logs" }, "ecs": { - "version": "8.0.0" + "version": "8.11.0" }, "elastic_agent": { "id": "acba78ef-1401-4689-977c-d8c2e5d6a8fa", diff --git a/packages/aws/data_stream/cloudwatch_logs/_dev/test/pipeline/test-cloudwatch-ec2.log-expected.json b/packages/aws/data_stream/cloudwatch_logs/_dev/test/pipeline/test-cloudwatch-ec2.log-expected.json index 845c800ca2e..cdd0f7b3a91 100644 --- a/packages/aws/data_stream/cloudwatch_logs/_dev/test/pipeline/test-cloudwatch-ec2.log-expected.json +++ b/packages/aws/data_stream/cloudwatch_logs/_dev/test/pipeline/test-cloudwatch-ec2.log-expected.json @@ -5,7 +5,7 @@ "provider": "aws" }, "ecs": { - "version": "8.0.0" + "version": "8.11.0" }, "event": { "kind": "event", @@ -21,7 +21,7 @@ "provider": "aws" }, "ecs": { - "version": "8.0.0" + "version": "8.11.0" }, "event": { "kind": "event", @@ -37,7 +37,7 @@ "provider": "aws" }, "ecs": { - "version": "8.0.0" + "version": "8.11.0" }, "event": { "kind": "event", @@ -53,7 +53,7 @@ "provider": "aws" }, "ecs": { - "version": "8.0.0" + "version": "8.11.0" }, "event": { "kind": "event", @@ -69,7 +69,7 @@ "provider": "aws" }, "ecs": { - "version": "8.0.0" + "version": "8.11.0" }, "event": { "kind": "event", @@ -85,7 +85,7 @@ "provider": "aws" }, "ecs": { - "version": "8.0.0" + "version": "8.11.0" }, "event": { "kind": "event", diff --git a/packages/aws/data_stream/cloudwatch_logs/elasticsearch/ingest_pipeline/default.yml b/packages/aws/data_stream/cloudwatch_logs/elasticsearch/ingest_pipeline/default.yml index d5b4b356a3a..df225af7a55 100644 --- a/packages/aws/data_stream/cloudwatch_logs/elasticsearch/ingest_pipeline/default.yml +++ b/packages/aws/data_stream/cloudwatch_logs/elasticsearch/ingest_pipeline/default.yml @@ -4,7 +4,7 @@ description: "Pipeline for logs ingested from CloudWatch" processors: - set: field: ecs.version - value: '8.0.0' + value: '8.11.0' - set: field: event.original copy_from: message diff --git a/packages/aws/data_stream/cloudwatch_logs/fields/ecs.yml b/packages/aws/data_stream/cloudwatch_logs/fields/ecs.yml deleted file mode 100644 index 1ab78b29e48..00000000000 --- a/packages/aws/data_stream/cloudwatch_logs/fields/ecs.yml +++ /dev/null @@ -1,62 +0,0 @@ -- external: ecs - name: ecs.version -- external: ecs - name: error.message -- name: message - external: ecs -- external: ecs - name: tags -- name: event.ingested - external: ecs -- external: ecs - name: host.architecture -- external: ecs - name: host.domain -- external: ecs - name: host.hostname -- external: ecs - name: host.id -- external: ecs - name: host.ip -- external: ecs - name: host.mac -- external: ecs - name: host.name -- external: ecs - name: host.os.family -- external: ecs - name: host.os.kernel -- external: ecs - name: host.os.name -- external: ecs - name: host.os.platform -- external: ecs - name: host.os.version -- external: ecs - name: host.type -- external: ecs - name: cloud.account.id -- external: ecs - name: cloud.availability_zone -- external: ecs - name: cloud.instance.id -- external: ecs - name: cloud.instance.name -- external: ecs - name: cloud.machine.type -- external: ecs - name: cloud.provider -- external: ecs - name: cloud.region -- external: ecs - name: cloud.project.id -- external: ecs - name: container.id -- external: ecs - name: container.image.name -- external: ecs - name: container.labels -- external: ecs - name: container.name -- external: ecs - name: event.dataset diff --git a/packages/aws/data_stream/cloudwatch_logs/sample_event.json b/packages/aws/data_stream/cloudwatch_logs/sample_event.json index c9da99332c0..b26b9385210 100644 --- a/packages/aws/data_stream/cloudwatch_logs/sample_event.json +++ b/packages/aws/data_stream/cloudwatch_logs/sample_event.json @@ -1,22 +1,22 @@ { "@timestamp": "2020-02-20T07:02:37.000Z", + "aws": { + "cloudwatch": { + "message": "ip-172-31-81-156 ec2net: [get_meta] Trying to get http://169.254.169.254/latest/meta-data/network/interfaces/macs/12:e2:a9:95:8b:97/local-ipv4s" + } + }, "data_stream": { + "dataset": "aws.cloudwatch_logs", "namespace": "default", - "type": "logs", - "dataset": "aws.cloudwatch_logs" + "type": "logs" }, "ecs": { - "version": "8.0.0" + "version": "8.11.0" }, "event": { "ingested": "2021-07-19T21:47:04.696803300Z", "original": "2020-02-20T07:02:37.000Z Feb 20 07:02:37 ip-172-31-81-156 ec2net: [get_meta] Trying to get http://169.254.169.254/latest/meta-data/network/interfaces/macs/12:e2:a9:95:8b:97/local-ipv4s" }, - "aws": { - "cloudwatch": { - "message": "ip-172-31-81-156 ec2net: [get_meta] Trying to get http://169.254.169.254/latest/meta-data/network/interfaces/macs/12:e2:a9:95:8b:97/local-ipv4s" - } - }, "tags": [ "preserve_original_event" ] diff --git a/packages/aws/data_stream/cloudwatch_metrics/fields/ecs.yml b/packages/aws/data_stream/cloudwatch_metrics/fields/ecs.yml index 654c9b2e202..303e169c534 100644 --- a/packages/aws/data_stream/cloudwatch_metrics/fields/ecs.yml +++ b/packages/aws/data_stream/cloudwatch_metrics/fields/ecs.yml @@ -1,70 +1,9 @@ -- external: ecs - name: cloud - external: ecs name: cloud.account.id dimension: true -- external: ecs - name: cloud.account.name -- external: ecs - name: cloud.availability_zone -- external: ecs - name: cloud.instance.id -- external: ecs - name: cloud.machine.type -- external: ecs - name: cloud.provider - external: ecs name: cloud.region dimension: true -- external: ecs - name: ecs.version -- external: ecs - name: error -- external: ecs - name: error.message -- external: ecs - name: service.type -- external: ecs - name: host.architecture -- external: ecs - name: host.domain -- external: ecs - name: host.hostname -- external: ecs - name: host.id -- external: ecs - name: host.ip -- external: ecs - name: host.mac -- external: ecs - name: host.name -- external: ecs - name: host.os.family -- external: ecs - name: host.os.kernel -- external: ecs - name: host.os.name -- external: ecs - name: host.os.platform -- external: ecs - name: host.os.version -- external: ecs - name: host.type -- external: ecs - name: cloud.instance.name -- external: ecs - name: cloud.project.id -- external: ecs - name: container.id -- external: ecs - name: container.image.name -- external: ecs - name: container.labels -- external: ecs - name: container.name - name: agent.id external: ecs dimension: true -- external: ecs - name: event.dataset - type: constant_keyword diff --git a/packages/aws/data_stream/cloudwatch_metrics/sample_event.json b/packages/aws/data_stream/cloudwatch_metrics/sample_event.json index bf25e887b58..89fc94ea4ba 100644 --- a/packages/aws/data_stream/cloudwatch_metrics/sample_event.json +++ b/packages/aws/data_stream/cloudwatch_metrics/sample_event.json @@ -1,13 +1,5 @@ { "@timestamp": "2020-05-28T17:17:02.812Z", - "event": { - "duration": 14119105951, - "dataset": "aws.cloudwatch_metrics", - "module": "aws" - }, - "ecs": { - "version": "1.5.0" - }, "agent": { "ephemeral_id": "17803f33-b617-4ce9-a9ac-e218c02aeb4b", "id": "12f376ef-5186-4e8b-a175-70f1140a8f30", @@ -15,39 +7,47 @@ "type": "metricbeat", "version": "8.0.0" }, - "service": { - "type": "aws" - }, - "cloud": { - "provider": "aws", - "region": "us-west-2", - "account": { - "name": "elastic-beats", - "id": "428152502467" - } - }, "aws": { + "cloudwatch": { + "namespace": "AWS/EC2" + }, "dimensions": { "InstanceId": "i-0830bfecfa7173cbe" }, "ec2": { "metrics": { - "DiskWriteOps": { - "avg": 0, - "max": 0 - }, "CPUUtilization": { "avg": 0.7661943132361363, "max": 0.833333333333333 + }, + "DiskWriteOps": { + "avg": 0, + "max": 0 } } - }, - "cloudwatch": { - "namespace": "AWS/EC2" } }, + "cloud": { + "account": { + "id": "428152502467", + "name": "elastic-beats" + }, + "provider": "aws", + "region": "us-west-2" + }, + "ecs": { + "version": "8.11.0" + }, + "event": { + "dataset": "aws.cloudwatch_metrics", + "duration": 14119105951, + "module": "aws" + }, "metricset": { - "period": 300000, - "name": "cloudwatch" + "name": "cloudwatch", + "period": 300000 + }, + "service": { + "type": "aws" } } \ No newline at end of file diff --git a/packages/aws/data_stream/dynamodb/fields/ecs.yml b/packages/aws/data_stream/dynamodb/fields/ecs.yml index 654c9b2e202..303e169c534 100644 --- a/packages/aws/data_stream/dynamodb/fields/ecs.yml +++ b/packages/aws/data_stream/dynamodb/fields/ecs.yml @@ -1,70 +1,9 @@ -- external: ecs - name: cloud - external: ecs name: cloud.account.id dimension: true -- external: ecs - name: cloud.account.name -- external: ecs - name: cloud.availability_zone -- external: ecs - name: cloud.instance.id -- external: ecs - name: cloud.machine.type -- external: ecs - name: cloud.provider - external: ecs name: cloud.region dimension: true -- external: ecs - name: ecs.version -- external: ecs - name: error -- external: ecs - name: error.message -- external: ecs - name: service.type -- external: ecs - name: host.architecture -- external: ecs - name: host.domain -- external: ecs - name: host.hostname -- external: ecs - name: host.id -- external: ecs - name: host.ip -- external: ecs - name: host.mac -- external: ecs - name: host.name -- external: ecs - name: host.os.family -- external: ecs - name: host.os.kernel -- external: ecs - name: host.os.name -- external: ecs - name: host.os.platform -- external: ecs - name: host.os.version -- external: ecs - name: host.type -- external: ecs - name: cloud.instance.name -- external: ecs - name: cloud.project.id -- external: ecs - name: container.id -- external: ecs - name: container.image.name -- external: ecs - name: container.labels -- external: ecs - name: container.name - name: agent.id external: ecs dimension: true -- external: ecs - name: event.dataset - type: constant_keyword diff --git a/packages/aws/data_stream/dynamodb/sample_event.json b/packages/aws/data_stream/dynamodb/sample_event.json index 97a1ec7783d..7b4eacb5f7c 100644 --- a/packages/aws/data_stream/dynamodb/sample_event.json +++ b/packages/aws/data_stream/dynamodb/sample_event.json @@ -1,78 +1,78 @@ { "@timestamp": "2022-07-25T21:53:00.000Z", "agent": { - "name": "docker-fleet-agent", + "ephemeral_id": "64a12b83-a4f1-487c-8d2c-9581fda6ca2a", "id": "2d4b09d0-cdb6-445e-ac3f-6415f87b9864", + "name": "docker-fleet-agent", "type": "metricbeat", - "ephemeral_id": "64a12b83-a4f1-487c-8d2c-9581fda6ca2a", "version": "8.3.2" }, - "elastic_agent": { - "id": "2d4b09d0-cdb6-445e-ac3f-6415f87b9864", - "version": "8.3.2", - "snapshot": false - }, - "cloud": { - "provider": "aws", - "region": "eu-central-1", - "account": { - "name": "elastic-beats", - "id": "428152502467" - } - }, - "ecs": { - "version": "8.0.0" - }, - "service": { - "type": "aws" - }, - "data_stream": { - "namespace": "default", - "type": "metrics", - "dataset": "aws.dynamodb" - }, - "metricset": { - "period": 300000, - "name": "cloudwatch" - }, - "event": { - "duration": 10586366300, - "agent_id_status": "verified", - "ingested": "2022-07-25T21:57:51Z", - "module": "aws", - "dataset": "aws.dynamodb" - }, "aws": { "cloudwatch": { "namespace": "AWS/DynamoDB" }, "dynamodb": { "metrics": { - "AccountProvisionedWriteCapacityUtilization": { - "avg": 0.01 - }, - "MaxProvisionedTableWriteCapacityUtilization": { - "max": 0.01 - }, - "MaxProvisionedTableReadCapacityUtilization": { - "max": 0.01 + "AccountMaxReads": { + "max": 80000 }, "AccountMaxTableLevelReads": { "max": 40000 }, - "AccountMaxReads": { + "AccountMaxTableLevelWrites": { + "max": 40000 + }, + "AccountMaxWrites": { "max": 80000 }, "AccountProvisionedReadCapacityUtilization": { "avg": 0.01 }, - "AccountMaxWrites": { - "max": 80000 + "AccountProvisionedWriteCapacityUtilization": { + "avg": 0.01 }, - "AccountMaxTableLevelWrites": { - "max": 40000 + "MaxProvisionedTableReadCapacityUtilization": { + "max": 0.01 + }, + "MaxProvisionedTableWriteCapacityUtilization": { + "max": 0.01 } } } + }, + "cloud": { + "account": { + "id": "428152502467", + "name": "elastic-beats" + }, + "provider": "aws", + "region": "eu-central-1" + }, + "data_stream": { + "dataset": "aws.dynamodb", + "namespace": "default", + "type": "metrics" + }, + "ecs": { + "version": "8.11.0" + }, + "elastic_agent": { + "id": "2d4b09d0-cdb6-445e-ac3f-6415f87b9864", + "snapshot": false, + "version": "8.3.2" + }, + "event": { + "agent_id_status": "verified", + "dataset": "aws.dynamodb", + "duration": 10586366300, + "ingested": "2022-07-25T21:57:51Z", + "module": "aws" + }, + "metricset": { + "name": "cloudwatch", + "period": 300000 + }, + "service": { + "type": "aws" } } \ No newline at end of file diff --git a/packages/aws/data_stream/ebs/fields/ecs.yml b/packages/aws/data_stream/ebs/fields/ecs.yml index 654c9b2e202..303e169c534 100644 --- a/packages/aws/data_stream/ebs/fields/ecs.yml +++ b/packages/aws/data_stream/ebs/fields/ecs.yml @@ -1,70 +1,9 @@ -- external: ecs - name: cloud - external: ecs name: cloud.account.id dimension: true -- external: ecs - name: cloud.account.name -- external: ecs - name: cloud.availability_zone -- external: ecs - name: cloud.instance.id -- external: ecs - name: cloud.machine.type -- external: ecs - name: cloud.provider - external: ecs name: cloud.region dimension: true -- external: ecs - name: ecs.version -- external: ecs - name: error -- external: ecs - name: error.message -- external: ecs - name: service.type -- external: ecs - name: host.architecture -- external: ecs - name: host.domain -- external: ecs - name: host.hostname -- external: ecs - name: host.id -- external: ecs - name: host.ip -- external: ecs - name: host.mac -- external: ecs - name: host.name -- external: ecs - name: host.os.family -- external: ecs - name: host.os.kernel -- external: ecs - name: host.os.name -- external: ecs - name: host.os.platform -- external: ecs - name: host.os.version -- external: ecs - name: host.type -- external: ecs - name: cloud.instance.name -- external: ecs - name: cloud.project.id -- external: ecs - name: container.id -- external: ecs - name: container.image.name -- external: ecs - name: container.labels -- external: ecs - name: container.name - name: agent.id external: ecs dimension: true -- external: ecs - name: event.dataset - type: constant_keyword diff --git a/packages/aws/data_stream/ebs/sample_event.json b/packages/aws/data_stream/ebs/sample_event.json index 603739d9526..640c9d7022a 100644 --- a/packages/aws/data_stream/ebs/sample_event.json +++ b/packages/aws/data_stream/ebs/sample_event.json @@ -1,65 +1,32 @@ { + "@timestamp": "2022-08-03T12:21:00.000Z", "agent": { - "name": "docker-fleet-agent", - "id": "618e6f72-9eef-4992-b60e-12515d538189", "ephemeral_id": "2e8fed31-76b5-4efe-9893-947fd2346abd", - "type": "metricbeat", - "version": "8.2.0" - }, - "elastic_agent": { "id": "618e6f72-9eef-4992-b60e-12515d538189", - "version": "8.2.0", - "snapshot": false - }, - "cloud": { - "provider": "aws", - "region": "us-east-2" - }, - "@timestamp": "2022-08-03T12:21:00.000Z", - "ecs": { - "version": "8.0.0" - }, - "data_stream": { - "namespace": "default", - "type": "metrics", - "dataset": "aws.ebs" - }, - "service": { - "type": "aws" - }, - "host": { - "hostname": "docker-fleet-agent", - "os": { - "kernel": "5.18.11-200.fc36.x86_64", - "codename": "focal", - "name": "Ubuntu", - "type": "linux", - "family": "debian", - "version": "20.04.4 LTS (Focal Fossa)", - "platform": "ubuntu" - }, - "containerized": false, - "ip": [ - "172.18.0.7" - ], "name": "docker-fleet-agent", - "mac": [ - "02-42-AC-12-00-07" - ], - "architecture": "x86_64" - }, - "metricset": { - "period": 300000, - "name": "cloudwatch" + "type": "metricbeat", + "version": "8.2.0" }, "aws": { + "cloudwatch": { + "namespace": "AWS/EBS" + }, + "dimensions": { + "VolumeId": "vol-015d88f45122510a5" + }, "ebs": { "metrics": { + "BurstBalance": { + "avg": 100 + }, + "VolumeIdleTime": { + "sum": 239.87 + }, "VolumeQueueLength": { "avg": 0 }, - "BurstBalance": { - "avg": 100 + "VolumeReadOps": { + "avg": 0 }, "VolumeTotalWriteTime": { "sum": 0.062 @@ -69,27 +36,60 @@ }, "VolumeWriteOps": { "avg": 23 - }, - "VolumeReadOps": { - "avg": 0 - }, - "VolumeIdleTime": { - "sum": 239.87 } } - }, - "cloudwatch": { - "namespace": "AWS/EBS" - }, - "dimensions": { - "VolumeId": "vol-015d88f45122510a5" } }, + "cloud": { + "provider": "aws", + "region": "us-east-2" + }, + "data_stream": { + "dataset": "aws.ebs", + "namespace": "default", + "type": "metrics" + }, + "ecs": { + "version": "8.11.0" + }, + "elastic_agent": { + "id": "618e6f72-9eef-4992-b60e-12515d538189", + "snapshot": false, + "version": "8.2.0" + }, "event": { - "duration": 1320126957, "agent_id_status": "verified", + "dataset": "aws.ebs", + "duration": 1320126957, "ingested": "2022-08-03T12:25:46Z", - "module": "aws", - "dataset": "aws.ebs" + "module": "aws" + }, + "host": { + "architecture": "x86_64", + "containerized": false, + "hostname": "docker-fleet-agent", + "ip": [ + "172.18.0.7" + ], + "mac": [ + "02-42-AC-12-00-07" + ], + "name": "docker-fleet-agent", + "os": { + "codename": "focal", + "family": "debian", + "kernel": "5.18.11-200.fc36.x86_64", + "name": "Ubuntu", + "platform": "ubuntu", + "type": "linux", + "version": "20.04.4 LTS (Focal Fossa)" + } + }, + "metricset": { + "name": "cloudwatch", + "period": 300000 + }, + "service": { + "type": "aws" } } \ No newline at end of file diff --git a/packages/aws/data_stream/ec2_logs/_dev/test/pipeline/test-ec2.log-expected.json b/packages/aws/data_stream/ec2_logs/_dev/test/pipeline/test-ec2.log-expected.json index 09227c00a1b..c6a52ac9d8f 100644 --- a/packages/aws/data_stream/ec2_logs/_dev/test/pipeline/test-ec2.log-expected.json +++ b/packages/aws/data_stream/ec2_logs/_dev/test/pipeline/test-ec2.log-expected.json @@ -8,7 +8,7 @@ } }, "ecs": { - "version": "8.0.0" + "version": "8.11.0" }, "event": { "original": "2020-02-20T07:01:01.000Z Feb 20 07:01:01 ip-172-31-81-156 systemd: Stopping User Slice of root." @@ -29,7 +29,7 @@ } }, "ecs": { - "version": "8.0.0" + "version": "8.11.0" }, "event": { "original": "2020-02-20T07:02:18.000Z Feb 20 07:02:18 ip-172-31-81-156 dhclient[3000]: XMT: Solicit on eth0, interval 125240ms." @@ -50,7 +50,7 @@ } }, "ecs": { - "version": "8.0.0" + "version": "8.11.0" }, "event": { "original": "2020-02-20T07:02:37.000Z Feb 20 07:02:37 ip-172-31-81-156 dhclient[2898]: DHCPREQUEST on eth0 to 172.31.80.1 port 67 (xid=0x4575af22)" @@ -71,7 +71,7 @@ } }, "ecs": { - "version": "8.0.0" + "version": "8.11.0" }, "event": { "original": "2020-02-20T07:02:37.000Z Feb 20 07:02:37 ip-172-31-81-156 dhclient[2898]: DHCPACK from 172.31.80.1 (xid=0x4575af22)" @@ -92,7 +92,7 @@ } }, "ecs": { - "version": "8.0.0" + "version": "8.11.0" }, "event": { "original": "2020-02-20T07:02:37.000Z Feb 20 07:02:37 ip-172-31-81-156 dhclient[2898]: bound to 172.31.81.156 -- renewal in 1599 seconds." @@ -113,7 +113,7 @@ } }, "ecs": { - "version": "8.0.0" + "version": "8.11.0" }, "event": { "original": "2020-02-20T07:02:37.000Z Feb 20 07:02:37 ip-172-31-81-156 ec2net: [get_meta] Trying to get http://169.254.169.254/latest/meta-data/network/interfaces/macs/12:e2:a9:95:8b:97/local-ipv4s" diff --git a/packages/aws/data_stream/ec2_logs/elasticsearch/ingest_pipeline/default.yml b/packages/aws/data_stream/ec2_logs/elasticsearch/ingest_pipeline/default.yml index 739f0862abd..5b0b86fa75b 100644 --- a/packages/aws/data_stream/ec2_logs/elasticsearch/ingest_pipeline/default.yml +++ b/packages/aws/data_stream/ec2_logs/elasticsearch/ingest_pipeline/default.yml @@ -4,7 +4,7 @@ description: "Pipeline for EC2 logs in CloudWatch" processors: - set: field: ecs.version - value: '8.0.0' + value: '8.11.0' - rename: field: message target_field: event.original diff --git a/packages/aws/data_stream/ec2_logs/fields/ecs.yml b/packages/aws/data_stream/ec2_logs/fields/ecs.yml deleted file mode 100644 index 217ec756ce4..00000000000 --- a/packages/aws/data_stream/ec2_logs/fields/ecs.yml +++ /dev/null @@ -1,63 +0,0 @@ -- external: ecs - name: ecs.version -- external: ecs - name: error.message -- external: ecs - name: message -- external: ecs - name: tags -- external: ecs - name: host.architecture -- external: ecs - name: host.domain -- external: ecs - name: host.hostname -- external: ecs - name: host.id -- external: ecs - name: host.ip -- external: ecs - name: host.mac -- external: ecs - name: host.name -- external: ecs - name: host.os.family -- external: ecs - name: host.os.kernel -- external: ecs - name: host.os.name -- external: ecs - name: host.os.platform -- external: ecs - name: host.os.version -- external: ecs - name: host.type -- external: ecs - name: cloud.account.id -- external: ecs - name: cloud.availability_zone -- external: ecs - name: cloud.instance.id -- external: ecs - name: cloud.instance.name -- external: ecs - name: cloud.machine.type -- external: ecs - name: cloud.provider -- external: ecs - name: cloud.region -- external: ecs - name: cloud.project.id -- external: ecs - name: container.id -- external: ecs - name: container.image.name -- external: ecs - name: container.labels -- external: ecs - name: container.name -- external: ecs - name: log.file.path -- external: ecs - name: event.dataset - type: constant_keyword diff --git a/packages/aws/data_stream/ec2_logs/fields/fields.yml b/packages/aws/data_stream/ec2_logs/fields/fields.yml index cf7d5a87890..08cc6ab2b42 100644 --- a/packages/aws/data_stream/ec2_logs/fields/fields.yml +++ b/packages/aws/data_stream/ec2_logs/fields/fields.yml @@ -5,6 +5,3 @@ type: keyword description: | The internet address of the requester. -- name: process.name - type: keyword - description: Process name. diff --git a/packages/aws/data_stream/ec2_logs/sample_event.json b/packages/aws/data_stream/ec2_logs/sample_event.json index 576987db15d..dd25a81a195 100644 --- a/packages/aws/data_stream/ec2_logs/sample_event.json +++ b/packages/aws/data_stream/ec2_logs/sample_event.json @@ -31,7 +31,7 @@ "type": "logs" }, "ecs": { - "version": "8.0.0" + "version": "8.11.0" }, "elastic_agent": { "id": "acba78ef-1401-4689-977c-d8c2e5d6a8fa", diff --git a/packages/aws/data_stream/ec2_metrics/fields/ecs.yml b/packages/aws/data_stream/ec2_metrics/fields/ecs.yml index 32b6df77382..303e169c534 100644 --- a/packages/aws/data_stream/ec2_metrics/fields/ecs.yml +++ b/packages/aws/data_stream/ec2_metrics/fields/ecs.yml @@ -1,46 +1,9 @@ -- external: ecs - name: cloud - external: ecs name: cloud.account.id dimension: true -- external: ecs - name: cloud.account.name -- external: ecs - name: cloud.availability_zone -- external: ecs - name: cloud.instance.id -- external: ecs - name: cloud.machine.type -- external: ecs - name: cloud.provider - external: ecs name: cloud.region dimension: true -- external: ecs - name: ecs.version -- external: ecs - name: error -- external: ecs - name: error.message -- external: ecs - name: service.type -- external: ecs - name: host.cpu.usage -- external: ecs - name: host.disk.read.bytes -- external: ecs - name: host.disk.write.bytes -- external: ecs - name: host.network.egress.bytes -- external: ecs - name: host.network.egress.packets -- external: ecs - name: host.network.ingress.bytes -- external: ecs - name: host.network.ingress.packets - name: agent.id external: ecs dimension: true -- external: ecs - name: event.dataset - type: constant_keyword diff --git a/packages/aws/data_stream/ec2_metrics/sample_event.json b/packages/aws/data_stream/ec2_metrics/sample_event.json index 050d1461100..9e4560e0f1b 100644 --- a/packages/aws/data_stream/ec2_metrics/sample_event.json +++ b/packages/aws/data_stream/ec2_metrics/sample_event.json @@ -1,151 +1,80 @@ { "@timestamp": "2023-08-07T18:35:00.000Z", - "cloud": { - "availability_zone": "eu-north-1c", - "instance": { - "id": "i-0c08512debca266ab" - }, - "provider": "aws", - "machine": { - "type": "t3.medium" - }, - "region": "eu-north-1", - "account": { - "name": "MonitoringAccount", - "id": "627286350134" - } - }, "agent": { - "name": "docker-fleet-agent", + "ephemeral_id": "b8cd4414-f528-43f4-b43f-0edbcc69b46f", "id": "72314f01-98f2-477f-978a-e98d109c640c", + "name": "docker-fleet-agent", "type": "metricbeat", - "ephemeral_id": "b8cd4414-f528-43f4-b43f-0edbcc69b46f", "version": "8.8.1" }, - "ecs": { - "version": "8.0.0" - }, - "service": { - "type": "aws" - }, - "data_stream": { - "namespace": "default", - "type": "metrics", - "dataset": "aws.ec2_metrics" - }, - "host": { - "hostname": "docker-fleet-agent", - "os": { - "kernel": "5.15.49-linuxkit-pr", - "codename": "focal", - "name": "Ubuntu", - "type": "linux", - "family": "debian", - "version": "20.04.6 LTS (Focal Fossa)", - "platform": "ubuntu" + "aws": { + "cloudwatch": { + "namespace": "AWS/EC2" }, - "containerized": false, - "ip": [ - "172.20.0.7" - ], - "name": "docker-fleet-agent", - "cpu": { - "usage": 2.8849988898518673 + "dimensions": { + "InstanceId": "i-0c08512debca266ab" }, - "id": "d08b346fbb8f49f5a2bb1a477f8ceb54", - "mac": [ - "02-42-AC-14-00-07" - ], - "architecture": "aarch64", - "network": { - "ingress": { - "bytes": 1608959, - "packets": 5334 - }, - "egress": { - "bytes": 626755, - "packets": 4977 - } - } - }, - "elastic_agent": { - "id": "72314f01-98f2-477f-978a-e98d109c640c", - "version": "8.8.1", - "snapshot": false - }, - "metricset": { - "period": 300000, - "name": "cloudwatch" - }, - "event": { - "duration": 5858967919, - "agent_id_status": "verified", - "ingested": "2023-08-07T18:41:31Z", - "module": "aws", - "dataset": "aws.ec2_metrics" - }, - "aws": { "ec2": { "instance": { + "core": { + "count": 1 + }, "image": { "id": "ami-00b8290583a865359" }, - "core": { - "count": 1 + "monitoring": { + "state": "disabled" }, "private": { - "ip": "172.31.13.154", - "dns_name": "ip-172-31-13-154.eu-north-1.compute.internal" + "dns_name": "ip-172-31-13-154.eu-north-1.compute.internal", + "ip": "172.31.13.154" }, - "threads_per_core": 2, "public": { - "ip": "16.16.138.5", - "dns_name": "ec2-16-16-138-5.eu-north-1.compute.amazonaws.com" + "dns_name": "ec2-16-16-138-5.eu-north-1.compute.amazonaws.com", + "ip": "16.16.138.5" }, "state": { "code": 16, "name": "running" }, - "monitoring": { - "state": "disabled" - } + "threads_per_core": 2 }, "metrics": { - "NetworkOut": { - "rate": 10445.916666666666, - "sum": 626755 + "CPUCreditBalance": { + "avg": 576 }, - "CPUUtilization": { - "avg": 2.8849988898518673 + "CPUCreditUsage": { + "avg": 0.29100543333333334 }, - "StatusCheckFailed_Instance": { + "CPUSurplusCreditBalance": { "avg": 0 }, - "CPUCreditUsage": { - "avg": 0.29100543333333334 + "CPUSurplusCreditsCharged": { + "avg": 0 }, - "CPUCreditBalance": { - "avg": 576 + "CPUUtilization": { + "avg": 2.8849988898518673 }, - "NetworkPacketsOut": { - "rate": 82.95, - "sum": 4977 + "NetworkIn": { + "rate": 26815.983333333334, + "sum": 1608959 + }, + "NetworkOut": { + "rate": 10445.916666666666, + "sum": 626755 }, "NetworkPacketsIn": { "rate": 88.9, "sum": 5334 }, - "NetworkIn": { - "rate": 26815.983333333334, - "sum": 1608959 + "NetworkPacketsOut": { + "rate": 82.95, + "sum": 4977 }, "StatusCheckFailed": { "avg": 0 }, - "CPUSurplusCreditsCharged": { - "avg": 0 - }, - "CPUSurplusCreditBalance": { + "StatusCheckFailed_Instance": { "avg": 0 }, "StatusCheckFailed_System": { @@ -153,16 +82,87 @@ } } }, - "cloudwatch": { - "namespace": "AWS/EC2" - }, - "dimensions": { - "InstanceId": "i-0c08512debca266ab" - }, "tags": { "aws:autoscaling:groupName": "eks-firehose-50c386d7-c8b1-bde8-5d42-d3841ca7ecfe", - "aws:ec2launchtemplate:version": "1", - "aws:ec2launchtemplate:id": "lt-09e1cdf590e35c687" + "aws:ec2launchtemplate:id": "lt-09e1cdf590e35c687", + "aws:ec2launchtemplate:version": "1" } + }, + "cloud": { + "account": { + "id": "627286350134", + "name": "MonitoringAccount" + }, + "availability_zone": "eu-north-1c", + "instance": { + "id": "i-0c08512debca266ab" + }, + "machine": { + "type": "t3.medium" + }, + "provider": "aws", + "region": "eu-north-1" + }, + "data_stream": { + "dataset": "aws.ec2_metrics", + "namespace": "default", + "type": "metrics" + }, + "ecs": { + "version": "8.11.0" + }, + "elastic_agent": { + "id": "72314f01-98f2-477f-978a-e98d109c640c", + "snapshot": false, + "version": "8.8.1" + }, + "event": { + "agent_id_status": "verified", + "dataset": "aws.ec2_metrics", + "duration": 5858967919, + "ingested": "2023-08-07T18:41:31Z", + "module": "aws" + }, + "host": { + "architecture": "aarch64", + "containerized": false, + "cpu": { + "usage": 2.8849988898518673 + }, + "hostname": "docker-fleet-agent", + "id": "d08b346fbb8f49f5a2bb1a477f8ceb54", + "ip": [ + "172.20.0.7" + ], + "mac": [ + "02-42-AC-14-00-07" + ], + "name": "docker-fleet-agent", + "network": { + "egress": { + "bytes": 626755, + "packets": 4977 + }, + "ingress": { + "bytes": 1608959, + "packets": 5334 + } + }, + "os": { + "codename": "focal", + "family": "debian", + "kernel": "5.15.49-linuxkit-pr", + "name": "Ubuntu", + "platform": "ubuntu", + "type": "linux", + "version": "20.04.6 LTS (Focal Fossa)" + } + }, + "metricset": { + "name": "cloudwatch", + "period": 300000 + }, + "service": { + "type": "aws" } } \ No newline at end of file diff --git a/packages/aws/data_stream/ecs_metrics/fields/ecs.yml b/packages/aws/data_stream/ecs_metrics/fields/ecs.yml index 654c9b2e202..303e169c534 100644 --- a/packages/aws/data_stream/ecs_metrics/fields/ecs.yml +++ b/packages/aws/data_stream/ecs_metrics/fields/ecs.yml @@ -1,70 +1,9 @@ -- external: ecs - name: cloud - external: ecs name: cloud.account.id dimension: true -- external: ecs - name: cloud.account.name -- external: ecs - name: cloud.availability_zone -- external: ecs - name: cloud.instance.id -- external: ecs - name: cloud.machine.type -- external: ecs - name: cloud.provider - external: ecs name: cloud.region dimension: true -- external: ecs - name: ecs.version -- external: ecs - name: error -- external: ecs - name: error.message -- external: ecs - name: service.type -- external: ecs - name: host.architecture -- external: ecs - name: host.domain -- external: ecs - name: host.hostname -- external: ecs - name: host.id -- external: ecs - name: host.ip -- external: ecs - name: host.mac -- external: ecs - name: host.name -- external: ecs - name: host.os.family -- external: ecs - name: host.os.kernel -- external: ecs - name: host.os.name -- external: ecs - name: host.os.platform -- external: ecs - name: host.os.version -- external: ecs - name: host.type -- external: ecs - name: cloud.instance.name -- external: ecs - name: cloud.project.id -- external: ecs - name: container.id -- external: ecs - name: container.image.name -- external: ecs - name: container.labels -- external: ecs - name: container.name - name: agent.id external: ecs dimension: true -- external: ecs - name: event.dataset - type: constant_keyword diff --git a/packages/aws/data_stream/ecs_metrics/sample_event.json b/packages/aws/data_stream/ecs_metrics/sample_event.json index 1bc293f29db..c0c005f8d86 100644 --- a/packages/aws/data_stream/ecs_metrics/sample_event.json +++ b/packages/aws/data_stream/ecs_metrics/sample_event.json @@ -1,85 +1,85 @@ { + "@timestamp": "2022-07-26T08:59:00.000Z", "agent": { - "name": "4b4f1fd6f3ff", + "ephemeral_id": "0c23896b-0bfe-469f-bf76-7203a2d52568", "id": "8c424f1d-e9b1-4aab-8ce5-77dceb4becfb", + "name": "4b4f1fd6f3ff", "type": "metricbeat", - "ephemeral_id": "0c23896b-0bfe-469f-bf76-7203a2d52568", "version": "8.1.0" }, - "elastic_agent": { - "id": "8c424f1d-e9b1-4aab-8ce5-77dceb4becfb", - "version": "8.1.0", - "snapshot": false + "aws": { + "cloudwatch": { + "namespace": "AWS/ECS" + }, + "dimensions": { + "ClusterName": "integration-cluster-1", + "ServiceName": "integration-service-1" + }, + "ecs": { + "metrics": { + "CPUUtilization": { + "avg": 100.040084913373 + }, + "MemoryUtilization": { + "avg": 9.195963541666666 + } + } + } }, "cloud": { - "provider": "aws", - "region": "eu-west-1", "account": { - "name": "elastic-observability", - "id": "627286350134" - } + "id": "627286350134", + "name": "elastic-observability" + }, + "provider": "aws", + "region": "eu-west-1" + }, + "data_stream": { + "dataset": "aws.ecs_metrics", + "namespace": "default", + "type": "metrics" }, - "@timestamp": "2022-07-26T08:59:00.000Z", "ecs": { - "version": "8.0.0" + "version": "8.11.0" }, - "service": { - "type": "aws" + "elastic_agent": { + "id": "8c424f1d-e9b1-4aab-8ce5-77dceb4becfb", + "snapshot": false, + "version": "8.1.0" }, - "data_stream": { - "namespace": "default", - "type": "metrics", - "dataset": "aws.ecs_metrics" + "event": { + "agent_id_status": "verified", + "dataset": "aws.ecs_metrics", + "duration": 1862196584, + "ingested": "2022-07-26T09:04:12Z", + "module": "aws" }, "host": { - "hostname": "4b4f1fd6f3ff", - "os": { - "kernel": "5.10.104-linuxkit", - "codename": "focal", - "name": "Ubuntu", - "family": "debian", - "type": "linux", - "version": "20.04.3 LTS (Focal Fossa)", - "platform": "ubuntu" - }, + "architecture": "aarch64", "containerized": false, + "hostname": "4b4f1fd6f3ff", "ip": [ "172.19.0.4" ], - "name": "4b4f1fd6f3ff", "mac": [ "02-42-AC-13-00-04" ], - "architecture": "aarch64" + "name": "4b4f1fd6f3ff", + "os": { + "codename": "focal", + "family": "debian", + "kernel": "5.10.104-linuxkit", + "name": "Ubuntu", + "platform": "ubuntu", + "type": "linux", + "version": "20.04.3 LTS (Focal Fossa)" + } }, "metricset": { - "period": 300000, - "name": "cloudwatch" + "name": "cloudwatch", + "period": 300000 }, - "aws": { - "ecs": { - "metrics": { - "CPUUtilization": { - "avg": 100.040084913373 - }, - "MemoryUtilization": { - "avg": 9.195963541666666 - } - } - }, - "cloudwatch": { - "namespace": "AWS/ECS" - }, - "dimensions": { - "ServiceName": "integration-service-1", - "ClusterName": "integration-cluster-1" - } - }, - "event": { - "duration": 1862196584, - "agent_id_status": "verified", - "ingested": "2022-07-26T09:04:12Z", - "module": "aws", - "dataset": "aws.ecs_metrics" + "service": { + "type": "aws" } } \ No newline at end of file diff --git a/packages/aws/data_stream/elb_logs/_dev/test/pipeline/test-alb.log-expected.json b/packages/aws/data_stream/elb_logs/_dev/test/pipeline/test-alb.log-expected.json index c10d956ddb0..16e408602da 100644 --- a/packages/aws/data_stream/elb_logs/_dev/test/pipeline/test-alb.log-expected.json +++ b/packages/aws/data_stream/elb_logs/_dev/test/pipeline/test-alb.log-expected.json @@ -46,7 +46,7 @@ "provider": "aws" }, "ecs": { - "version": "8.2.0" + "version": "8.11.0" }, "event": { "category": [ @@ -124,7 +124,7 @@ "provider": "aws" }, "ecs": { - "version": "8.2.0" + "version": "8.11.0" }, "event": { "category": [ diff --git a/packages/aws/data_stream/elb_logs/elasticsearch/ingest_pipeline/default.yml b/packages/aws/data_stream/elb_logs/elasticsearch/ingest_pipeline/default.yml index f60949ae61f..875a507422a 100644 --- a/packages/aws/data_stream/elb_logs/elasticsearch/ingest_pipeline/default.yml +++ b/packages/aws/data_stream/elb_logs/elasticsearch/ingest_pipeline/default.yml @@ -4,7 +4,7 @@ description: "Pipeline for ELB logs" processors: - set: field: ecs.version - value: '8.2.0' + value: '8.11.0' - rename: field: message target_field: event.original diff --git a/packages/aws/data_stream/elb_logs/fields/ecs.yml b/packages/aws/data_stream/elb_logs/fields/ecs.yml deleted file mode 100644 index f20e10fa2e4..00000000000 --- a/packages/aws/data_stream/elb_logs/fields/ecs.yml +++ /dev/null @@ -1,133 +0,0 @@ -- external: ecs - name: ecs.version -- external: ecs - name: error.message -- external: ecs - name: tags -- external: ecs - name: url.domain -- external: ecs - name: url.original -- external: ecs - name: url.path -- external: ecs - name: url.port -- external: ecs - name: url.scheme -- external: ecs - name: url.query -- external: ecs - name: user_agent.device.name -- external: ecs - name: user_agent.name -- external: ecs - name: user_agent.version -- name: destination.domain - external: ecs -- name: event.start - external: ecs -- name: destination.bytes - external: ecs -- name: http.response.status_code - external: ecs -- name: http.request.body.bytes - external: ecs -- name: http.response.body.bytes - external: ecs -- name: http.request.method - external: ecs -- name: http.request.referrer - external: ecs -- name: http.version - external: ecs -- name: user_agent.original - external: ecs -- name: cloud.provider - external: ecs -- name: event.kind - external: ecs -- name: event.category - external: ecs -- name: event.outcome - external: ecs -- name: trace.id - external: ecs -- name: event.end - external: ecs -- name: source.address - external: ecs -- name: source.ip - external: ecs -- name: source.as.number - external: ecs -- name: source.as.organization.name - external: ecs -- name: source.geo.city_name - external: ecs -- name: source.geo.continent_name - external: ecs -- name: source.geo.country_iso_code - external: ecs -- name: source.geo.country_name - external: ecs -- name: source.geo.location - type: geo_point - description: Longitude and latitude. - external: ecs -- name: source.geo.region_name - external: ecs -- name: source.port - external: ecs -- external: ecs - name: host.architecture -- external: ecs - name: host.domain -- external: ecs - name: host.hostname -- external: ecs - name: host.id -- external: ecs - name: host.ip -- external: ecs - name: host.mac -- external: ecs - name: host.name -- external: ecs - name: host.os.family -- external: ecs - name: host.os.kernel -- external: ecs - name: host.os.name -- external: ecs - name: host.os.platform -- external: ecs - name: host.os.version -- external: ecs - name: host.type -- external: ecs - name: cloud.account.id -- external: ecs - name: cloud.availability_zone -- external: ecs - name: cloud.instance.id -- external: ecs - name: cloud.instance.name -- external: ecs - name: cloud.machine.type -- external: ecs - name: cloud.region -- external: ecs - name: cloud.project.id -- external: ecs - name: container.id -- external: ecs - name: container.image.name -- external: ecs - name: container.labels -- external: ecs - name: container.name -- external: ecs - name: log.file.path -- external: ecs - name: event.dataset - type: constant_keyword diff --git a/packages/aws/data_stream/elb_logs/sample_event.json b/packages/aws/data_stream/elb_logs/sample_event.json index 791924a2df4..5328412c329 100644 --- a/packages/aws/data_stream/elb_logs/sample_event.json +++ b/packages/aws/data_stream/elb_logs/sample_event.json @@ -66,7 +66,7 @@ "type": "logs" }, "ecs": { - "version": "8.2.0" + "version": "8.11.0" }, "elastic_agent": { "id": "acba78ef-1401-4689-977c-d8c2e5d6a8fa", diff --git a/packages/aws/data_stream/elb_metrics/fields/ecs.yml b/packages/aws/data_stream/elb_metrics/fields/ecs.yml index 654c9b2e202..303e169c534 100644 --- a/packages/aws/data_stream/elb_metrics/fields/ecs.yml +++ b/packages/aws/data_stream/elb_metrics/fields/ecs.yml @@ -1,70 +1,9 @@ -- external: ecs - name: cloud - external: ecs name: cloud.account.id dimension: true -- external: ecs - name: cloud.account.name -- external: ecs - name: cloud.availability_zone -- external: ecs - name: cloud.instance.id -- external: ecs - name: cloud.machine.type -- external: ecs - name: cloud.provider - external: ecs name: cloud.region dimension: true -- external: ecs - name: ecs.version -- external: ecs - name: error -- external: ecs - name: error.message -- external: ecs - name: service.type -- external: ecs - name: host.architecture -- external: ecs - name: host.domain -- external: ecs - name: host.hostname -- external: ecs - name: host.id -- external: ecs - name: host.ip -- external: ecs - name: host.mac -- external: ecs - name: host.name -- external: ecs - name: host.os.family -- external: ecs - name: host.os.kernel -- external: ecs - name: host.os.name -- external: ecs - name: host.os.platform -- external: ecs - name: host.os.version -- external: ecs - name: host.type -- external: ecs - name: cloud.instance.name -- external: ecs - name: cloud.project.id -- external: ecs - name: container.id -- external: ecs - name: container.image.name -- external: ecs - name: container.labels -- external: ecs - name: container.name - name: agent.id external: ecs dimension: true -- external: ecs - name: event.dataset - type: constant_keyword diff --git a/packages/aws/data_stream/elb_metrics/fields/fields.yml b/packages/aws/data_stream/elb_metrics/fields/fields.yml index 02c59fd70f9..0ecd95c2b6d 100644 --- a/packages/aws/data_stream/elb_metrics/fields/fields.yml +++ b/packages/aws/data_stream/elb_metrics/fields/fields.yml @@ -165,7 +165,7 @@ type: long metric_type: gauge unit: s - description: The time elapsed after the request leaves the load balancer until the target starts to send the response headers. + description: The time elapsed after the request leaves the load balancer until the target starts to send the response headers. - name: networkelb type: group fields: diff --git a/packages/aws/data_stream/elb_metrics/sample_event.json b/packages/aws/data_stream/elb_metrics/sample_event.json index 4a5591f61a4..755d9e41365 100644 --- a/packages/aws/data_stream/elb_metrics/sample_event.json +++ b/packages/aws/data_stream/elb_metrics/sample_event.json @@ -1,93 +1,93 @@ { "@timestamp": "2022-06-08T18:19:00.000Z", "agent": { - "name": "docker-fleet-agent", + "ephemeral_id": "8c94e850-82e2-42ae-bd41-44ce7bbbb50c", "id": "90bfb41e-b925-420f-973e-9c1115297278", + "name": "docker-fleet-agent", "type": "metricbeat", - "ephemeral_id": "8c94e850-82e2-42ae-bd41-44ce7bbbb50c", "version": "8.2.0" }, - "elastic_agent": { - "id": "90bfb41e-b925-420f-973e-9c1115297278", - "version": "8.2.0", - "snapshot": false + "aws": { + "cloudwatch": { + "namespace": "AWS/ELB" + }, + "elb": { + "metrics": { + "HTTPCode_Backend_2XX": { + "sum": 31 + }, + "HTTPCode_Backend_4XX": { + "sum": 2 + }, + "HealthyHostCount": { + "max": 2 + }, + "Latency": { + "avg": 0.0010771534659645772 + }, + "RequestCount": { + "sum": 33 + }, + "UnHealthyHostCount": { + "max": 0 + } + } + } }, "cloud": { - "provider": "aws", - "region": "eu-central-1", "account": { - "name": "elastic-beats", - "id": "123456789" - } - }, - "ecs": { - "version": "8.0.0" + "id": "123456789", + "name": "elastic-beats" + }, + "provider": "aws", + "region": "eu-central-1" }, "data_stream": { + "dataset": "aws.elb_metrics", "namespace": "default", - "type": "metrics", - "dataset": "aws.elb_metrics" + "type": "metrics" }, - "service": { - "type": "aws" + "ecs": { + "version": "8.11.0" + }, + "elastic_agent": { + "id": "90bfb41e-b925-420f-973e-9c1115297278", + "snapshot": false, + "version": "8.2.0" + }, + "event": { + "agent_id_status": "verified", + "dataset": "aws.elb_metrics", + "duration": 15866718200, + "ingested": "2022-06-08T18:20:24Z", + "module": "aws" }, "host": { - "hostname": "docker-fleet-agent", - "os": { - "kernel": "5.10.47-linuxkit", - "codename": "focal", - "name": "Ubuntu", - "family": "debian", - "type": "linux", - "version": "20.04.4 LTS (Focal Fossa)", - "platform": "ubuntu" - }, + "architecture": "x86_64", "containerized": true, + "hostname": "docker-fleet-agent", "ip": [ "192.168.96.7" ], - "name": "docker-fleet-agent", "mac": [ "02-42-C0-A8-60-07" ], - "architecture": "x86_64" + "name": "docker-fleet-agent", + "os": { + "codename": "focal", + "family": "debian", + "kernel": "5.10.47-linuxkit", + "name": "Ubuntu", + "platform": "ubuntu", + "type": "linux", + "version": "20.04.4 LTS (Focal Fossa)" + } }, "metricset": { - "period": 60000, - "name": "cloudwatch" - }, - "aws": { - "elb": { - "metrics": { - "HealthyHostCount": { - "max": 2 - }, - "UnHealthyHostCount": { - "max": 0 - }, - "HTTPCode_Backend_4XX": { - "sum": 2 - }, - "HTTPCode_Backend_2XX": { - "sum": 31 - }, - "RequestCount": { - "sum": 33 - }, - "Latency": { - "avg": 0.0010771534659645772 - } - } - }, - "cloudwatch": { - "namespace": "AWS/ELB" - } + "name": "cloudwatch", + "period": 60000 }, - "event": { - "duration": 15866718200, - "agent_id_status": "verified", - "ingested": "2022-06-08T18:20:24Z", - "module": "aws", - "dataset": "aws.elb_metrics" + "service": { + "type": "aws" } } \ No newline at end of file diff --git a/packages/aws/data_stream/emr_logs/_dev/test/pipeline/test-emr-hadoop.log-expected.json b/packages/aws/data_stream/emr_logs/_dev/test/pipeline/test-emr-hadoop.log-expected.json index 62e5737f5ab..8a4cdd1a111 100644 --- a/packages/aws/data_stream/emr_logs/_dev/test/pipeline/test-emr-hadoop.log-expected.json +++ b/packages/aws/data_stream/emr_logs/_dev/test/pipeline/test-emr-hadoop.log-expected.json @@ -3,7 +3,7 @@ { "@timestamp": "2023-06-26T13:45:49.685Z", "ecs": { - "version": "8.0.0" + "version": "8.11.0" }, "event": { "original": "2023-06-26 13:45:49,685 INFO namenode.NameNode: STARTUP_MSG: \n/************************************************************\nSTARTUP_MSG: Starting NameNode\nSTARTUP_MSG: host = ip-172-31-25-102.eu-central-1.compute.internal/172.31.25.102\nSTARTUP_MSG: args = [-format, -nonInteractive]\nSTARTUP_MSG: version = 3.3.3-amzn-3\nSTARTUP_MSG: classpath = /etc/hadoop/conf:/usr/lib/hadoop/lib/jetty-security-9.4.48.v20220622.jar:/usr/lib/hadoop/lib/accessors-smart-2.4.7.jar:/usr/lib/hadoop/lib/jersey-core-1.19.jar:/usr/lib/hadoop/lib/animal-sniffer-annotations-1.17.jar\nSTARTUP_MSG: build = Unknown -r Unknown; compiled by 'release' on 2023-05-31T03:49Z\nSTARTUP_MSG: java = 1.8.0_372\n************************************************************/" @@ -23,7 +23,7 @@ { "@timestamp": "2023-06-26T13:45:49.697Z", "ecs": { - "version": "8.0.0" + "version": "8.11.0" }, "event": { "original": "2023-06-26 13:45:49,697 INFO namenode.NameNode: registered UNIX signal handlers for [TERM, HUP, INT]" @@ -42,7 +42,7 @@ { "@timestamp": "2023-06-26T13:45:49.823Z", "ecs": { - "version": "8.0.0" + "version": "8.11.0" }, "event": { "original": "2023-06-26 13:45:49,823 INFO namenode.NameNode: createNameNode [-format, -nonInteractive]" @@ -61,7 +61,7 @@ { "@timestamp": "2023-06-26T13:45:50.318Z", "ecs": { - "version": "8.0.0" + "version": "8.11.0" }, "event": { "original": "2023-06-26 13:45:50,318 INFO common.Util: Assuming 'file' scheme for path /mnt/namenode in configuration." @@ -80,7 +80,7 @@ { "@timestamp": "2023-06-26T13:45:50.319Z", "ecs": { - "version": "8.0.0" + "version": "8.11.0" }, "event": { "original": "2023-06-26 13:45:50,319 INFO common.Util: Assuming 'file' scheme for path /mnt1/namenode in configuration." @@ -99,7 +99,7 @@ { "@timestamp": "2023-06-26T13:45:50.319Z", "ecs": { - "version": "8.0.0" + "version": "8.11.0" }, "event": { "original": "2023-06-26 13:45:50,319 INFO common.Util: Assuming 'file' scheme for path /mnt/namenode in configuration." @@ -118,7 +118,7 @@ { "@timestamp": "2023-06-26T13:45:50.319Z", "ecs": { - "version": "8.0.0" + "version": "8.11.0" }, "event": { "original": "2023-06-26 13:45:50,319 INFO common.Util: Assuming 'file' scheme for path /mnt1/namenode in configuration." @@ -137,7 +137,7 @@ { "@timestamp": "2023-06-26T13:45:50.330Z", "ecs": { - "version": "8.0.0" + "version": "8.11.0" }, "event": { "original": "2023-06-26 13:45:50,330 INFO namenode.NameNode: Formatting using clusterid: CID-1b3b14b6-5518-47c3-b981-e5cb6b0ce38c" @@ -156,7 +156,7 @@ { "@timestamp": "2023-06-26T13:45:50.394Z", "ecs": { - "version": "8.0.0" + "version": "8.11.0" }, "event": { "original": "2023-06-26 13:45:50,394 INFO namenode.FSEditLog (main): Edit logging is async:true" @@ -175,7 +175,7 @@ { "@timestamp": "2023-06-26T13:45:50.484Z", "ecs": { - "version": "8.0.0" + "version": "8.11.0" }, "event": { "original": "2023-06-26 13:45:50,484 INFO namenode.FSNamesystem: KeyProvider: KeyProviderCryptoExtension: org.apache.hadoop.crypto.key.kms.LoadBalancingKMSClientProvider@a530d0a" @@ -194,7 +194,7 @@ { "@timestamp": "2023-06-26T13:45:50.486Z", "ecs": { - "version": "8.0.0" + "version": "8.11.0" }, "event": { "original": "2023-06-26 13:45:50,486 INFO namenode.FSNamesystem: fsLock is fair: true" @@ -213,7 +213,7 @@ { "@timestamp": "2023-06-26T13:45:50.486Z", "ecs": { - "version": "8.0.0" + "version": "8.11.0" }, "event": { "original": "2023-06-26 13:45:50,486 INFO namenode.FSNamesystem: Detailed lock hold time metrics enabled: false" @@ -232,7 +232,7 @@ { "@timestamp": "2023-06-26T13:45:50.492Z", "ecs": { - "version": "8.0.0" + "version": "8.11.0" }, "event": { "original": "2023-06-26 13:45:50,492 INFO namenode.FSNamesystem: fsOwner = hdfs (auth:SIMPLE)" @@ -251,7 +251,7 @@ { "@timestamp": "2023-06-26T13:45:50.493Z", "ecs": { - "version": "8.0.0" + "version": "8.11.0" }, "event": { "original": "2023-06-26 13:45:50,493 INFO namenode.FSNamesystem: supergroup = hdfsadmingroup" @@ -270,7 +270,7 @@ { "@timestamp": "2023-06-26T13:45:50.493Z", "ecs": { - "version": "8.0.0" + "version": "8.11.0" }, "event": { "original": "2023-06-26 13:45:50,493 INFO namenode.FSNamesystem: isPermissionEnabled = true" @@ -289,7 +289,7 @@ { "@timestamp": "2023-06-26T13:45:50.493Z", "ecs": { - "version": "8.0.0" + "version": "8.11.0" }, "event": { "original": "2023-06-26 13:45:50,493 INFO namenode.FSNamesystem: isStoragePolicyEnabled = true" @@ -308,7 +308,7 @@ { "@timestamp": "2023-06-26T13:45:50.493Z", "ecs": { - "version": "8.0.0" + "version": "8.11.0" }, "event": { "original": "2023-06-26 13:45:50,493 INFO namenode.FSNamesystem: HA Enabled: false" @@ -327,7 +327,7 @@ { "@timestamp": "2023-06-26T13:45:50.566Z", "ecs": { - "version": "8.0.0" + "version": "8.11.0" }, "event": { "original": "2023-06-26 13:45:50,566 INFO common.Util: dfs.datanode.fileio.profiling.sampling.percentage set to 0. Disabling file IO profiling" @@ -346,7 +346,7 @@ { "@timestamp": "2023-06-26T13:45:50.601Z", "ecs": { - "version": "8.0.0" + "version": "8.11.0" }, "event": { "original": "2023-06-26 13:45:50,601 INFO blockmanagement.DatanodeManager: dfs.block.invalidate.limit: configured=1000, counted=60, effected=1000" @@ -365,7 +365,7 @@ { "@timestamp": "2023-06-26T13:45:50.601Z", "ecs": { - "version": "8.0.0" + "version": "8.11.0" }, "event": { "original": "2023-06-26 13:45:50,601 INFO blockmanagement.DatanodeManager: dfs.namenode.datanode.registration.ip-hostname-check=true" @@ -384,7 +384,7 @@ { "@timestamp": "2023-06-26T13:45:50.610Z", "ecs": { - "version": "8.0.0" + "version": "8.11.0" }, "event": { "original": "2023-06-26 13:45:50,610 INFO blockmanagement.BlockManager: dfs.namenode.startup.delay.block.deletion.sec is set to 000:00:00:00.000" @@ -403,7 +403,7 @@ { "@timestamp": "2023-06-26T13:45:50.611Z", "ecs": { - "version": "8.0.0" + "version": "8.11.0" }, "event": { "original": "2023-06-26 13:45:50,611 INFO blockmanagement.BlockManager: The block deletion will start around 2023 Jun 26 13:45:50" @@ -422,7 +422,7 @@ { "@timestamp": "2023-06-26T13:45:50.612Z", "ecs": { - "version": "8.0.0" + "version": "8.11.0" }, "event": { "original": "2023-06-26 13:45:50,612 INFO util.GSet: Computing capacity for map BlocksMap" @@ -441,7 +441,7 @@ { "@timestamp": "2023-06-26T13:45:50.613Z", "ecs": { - "version": "8.0.0" + "version": "8.11.0" }, "event": { "original": "2023-06-26 13:45:50,613 INFO util.GSet: VM type = 64-bit" @@ -460,7 +460,7 @@ { "@timestamp": "2023-06-26T13:45:50.614Z", "ecs": { - "version": "8.0.0" + "version": "8.11.0" }, "event": { "original": "2023-06-26 13:45:50,614 INFO util.GSet: 2.0% max memory 864 MB = 17.3 MB" @@ -479,7 +479,7 @@ { "@timestamp": "2023-06-26T13:45:50.614Z", "ecs": { - "version": "8.0.0" + "version": "8.11.0" }, "event": { "original": "2023-06-26 13:45:50,614 INFO util.GSet: capacity = 2^21 = 2097152 entries" @@ -498,7 +498,7 @@ { "@timestamp": "2023-06-26T13:45:50.623Z", "ecs": { - "version": "8.0.0" + "version": "8.11.0" }, "event": { "original": "2023-06-26 13:45:50,623 INFO blockmanagement.BlockManager: Storage policy satisfier is disabled" @@ -517,7 +517,7 @@ { "@timestamp": "2023-06-26T13:45:50.623Z", "ecs": { - "version": "8.0.0" + "version": "8.11.0" }, "event": { "original": "2023-06-26 13:45:50,623 INFO blockmanagement.BlockManager: dfs.block.access.token.enable = false" @@ -536,7 +536,7 @@ { "@timestamp": "2023-06-26T13:45:50.631Z", "ecs": { - "version": "8.0.0" + "version": "8.11.0" }, "event": { "original": "2023-06-26 13:45:50,631 INFO blockmanagement.BlockManagerSafeMode: dfs.namenode.safemode.threshold-pct = 0.999" @@ -555,7 +555,7 @@ { "@timestamp": "2023-06-26T13:45:50.631Z", "ecs": { - "version": "8.0.0" + "version": "8.11.0" }, "event": { "original": "2023-06-26 13:45:50,631 INFO blockmanagement.BlockManagerSafeMode: dfs.namenode.safemode.min.datanodes = 0" @@ -574,7 +574,7 @@ { "@timestamp": "2023-06-26T13:45:50.631Z", "ecs": { - "version": "8.0.0" + "version": "8.11.0" }, "event": { "original": "2023-06-26 13:45:50,631 INFO blockmanagement.BlockManagerSafeMode: dfs.namenode.safemode.extension = 5000" @@ -593,7 +593,7 @@ { "@timestamp": "2023-06-26T13:45:50.632Z", "ecs": { - "version": "8.0.0" + "version": "8.11.0" }, "event": { "original": "2023-06-26 13:45:50,632 INFO blockmanagement.BlockManager: defaultReplication = 1" @@ -612,7 +612,7 @@ { "@timestamp": "2023-06-26T13:45:50.632Z", "ecs": { - "version": "8.0.0" + "version": "8.11.0" }, "event": { "original": "2023-06-26 13:45:50,632 INFO blockmanagement.BlockManager: maxReplication = 512" @@ -631,7 +631,7 @@ { "@timestamp": "2023-06-26T13:45:50.632Z", "ecs": { - "version": "8.0.0" + "version": "8.11.0" }, "event": { "original": "2023-06-26 13:45:50,632 INFO blockmanagement.BlockManager: minReplication = 1" @@ -650,7 +650,7 @@ { "@timestamp": "2023-06-26T13:45:50.632Z", "ecs": { - "version": "8.0.0" + "version": "8.11.0" }, "event": { "original": "2023-06-26 13:45:50,632 INFO blockmanagement.BlockManager: maxReplicationStreams = 100" @@ -669,7 +669,7 @@ { "@timestamp": "2023-06-26T13:45:50.633Z", "ecs": { - "version": "8.0.0" + "version": "8.11.0" }, "event": { "original": "2023-06-26 13:45:50,633 INFO blockmanagement.BlockManager: redundancyRecheckInterval = 3000ms" @@ -688,7 +688,7 @@ { "@timestamp": "2023-06-26T13:45:50.633Z", "ecs": { - "version": "8.0.0" + "version": "8.11.0" }, "event": { "original": "2023-06-26 13:45:50,633 INFO blockmanagement.BlockManager: encryptDataTransfer = false" @@ -707,7 +707,7 @@ { "@timestamp": "2023-06-26T13:45:50.633Z", "ecs": { - "version": "8.0.0" + "version": "8.11.0" }, "event": { "original": "2023-06-26 13:45:50,633 INFO blockmanagement.BlockManager: maxNumBlocksToLog = 1000" @@ -726,7 +726,7 @@ { "@timestamp": "2023-06-26T13:45:50.670Z", "ecs": { - "version": "8.0.0" + "version": "8.11.0" }, "event": { "original": "2023-06-26 13:45:50,670 INFO namenode.FSDirectory: GLOBAL serial map: bits=29 maxEntries=536870911" @@ -745,7 +745,7 @@ { "@timestamp": "2023-06-26T13:45:50.671Z", "ecs": { - "version": "8.0.0" + "version": "8.11.0" }, "event": { "original": "2023-06-26 13:45:50,671 INFO namenode.FSDirectory: USER serial map: bits=24 maxEntries=16777215" @@ -764,7 +764,7 @@ { "@timestamp": "2023-06-26T13:45:50.671Z", "ecs": { - "version": "8.0.0" + "version": "8.11.0" }, "event": { "original": "2023-06-26 13:45:50,671 INFO namenode.FSDirectory: GROUP serial map: bits=24 maxEntries=16777215" @@ -783,7 +783,7 @@ { "@timestamp": "2023-06-26T13:45:50.671Z", "ecs": { - "version": "8.0.0" + "version": "8.11.0" }, "event": { "original": "2023-06-26 13:45:50,671 INFO namenode.FSDirectory: XATTR serial map: bits=24 maxEntries=16777215" @@ -802,7 +802,7 @@ { "@timestamp": "2023-06-26T13:45:50.696Z", "ecs": { - "version": "8.0.0" + "version": "8.11.0" }, "event": { "original": "2023-06-26 13:45:50,696 INFO util.GSet: Computing capacity for map INodeMap" @@ -821,7 +821,7 @@ { "@timestamp": "2023-06-26T13:45:50.696Z", "ecs": { - "version": "8.0.0" + "version": "8.11.0" }, "event": { "original": "2023-06-26 13:45:50,696 INFO util.GSet: VM type = 64-bit" @@ -840,7 +840,7 @@ { "@timestamp": "2023-06-26T13:45:50.697Z", "ecs": { - "version": "8.0.0" + "version": "8.11.0" }, "event": { "original": "2023-06-26 13:45:50,697 INFO util.GSet: 1.0% max memory 864 MB = 8.6 MB" @@ -859,7 +859,7 @@ { "@timestamp": "2023-06-26T13:45:50.697Z", "ecs": { - "version": "8.0.0" + "version": "8.11.0" }, "event": { "original": "2023-06-26 13:45:50,697 INFO util.GSet: capacity = 2^20 = 1048576 entries" @@ -878,7 +878,7 @@ { "@timestamp": "2023-06-26T13:45:50.699Z", "ecs": { - "version": "8.0.0" + "version": "8.11.0" }, "event": { "original": "2023-06-26 13:45:50,699 INFO namenode.FSDirectory: ACLs enabled? true" @@ -897,7 +897,7 @@ { "@timestamp": "2023-06-26T13:45:50.699Z", "ecs": { - "version": "8.0.0" + "version": "8.11.0" }, "event": { "original": "2023-06-26 13:45:50,699 INFO namenode.FSDirectory: POSIX ACL inheritance enabled? true" @@ -916,7 +916,7 @@ { "@timestamp": "2023-06-26T13:45:50.699Z", "ecs": { - "version": "8.0.0" + "version": "8.11.0" }, "event": { "original": "2023-06-26 13:45:50,699 INFO namenode.FSDirectory: XAttrs enabled? true" @@ -935,7 +935,7 @@ { "@timestamp": "2023-06-26T13:45:50.700Z", "ecs": { - "version": "8.0.0" + "version": "8.11.0" }, "event": { "original": "2023-06-26 13:45:50,700 INFO namenode.NameNode: Caching file names occurring more than 10 times" @@ -954,7 +954,7 @@ { "@timestamp": "2023-06-26T13:45:50.708Z", "ecs": { - "version": "8.0.0" + "version": "8.11.0" }, "event": { "original": "2023-06-26 13:45:50,708 INFO namenode.ReencryptionHandler: Configured throttleLimitHandlerRatio=1.0 for re-encryption" @@ -973,7 +973,7 @@ { "@timestamp": "2023-06-26T13:45:50.717Z", "ecs": { - "version": "8.0.0" + "version": "8.11.0" }, "event": { "original": "2023-06-26 13:45:50,717 INFO snapshot.SnapshotManager: Loaded config captureOpenFiles: false, skipCaptureAccessTimeOnlyChange: false, snapshotDiffAllowSnapRootDescendant: true, maxSnapshotLimit: 65536" @@ -992,7 +992,7 @@ { "@timestamp": "2023-06-26T13:45:50.720Z", "ecs": { - "version": "8.0.0" + "version": "8.11.0" }, "event": { "original": "2023-06-26 13:45:50,720 INFO snapshot.SnapshotManager: SkipList is disabled" @@ -1011,7 +1011,7 @@ { "@timestamp": "2023-06-26T13:45:50.748Z", "ecs": { - "version": "8.0.0" + "version": "8.11.0" }, "event": { "original": "2023-06-26 13:45:50,748 INFO util.GSet: Computing capacity for map cachedBlocks" @@ -1030,7 +1030,7 @@ { "@timestamp": "2023-06-26T13:45:50.748Z", "ecs": { - "version": "8.0.0" + "version": "8.11.0" }, "event": { "original": "2023-06-26 13:45:50,748 INFO util.GSet: VM type = 64-bit" @@ -1049,7 +1049,7 @@ { "@timestamp": "2023-06-26T13:45:50.748Z", "ecs": { - "version": "8.0.0" + "version": "8.11.0" }, "event": { "original": "2023-06-26 13:45:50,748 INFO util.GSet: 0.25% max memory 864 MB = 2.2 MB" @@ -1068,7 +1068,7 @@ { "@timestamp": "2023-06-26T13:45:50.749Z", "ecs": { - "version": "8.0.0" + "version": "8.11.0" }, "event": { "original": "2023-06-26 13:45:50,749 INFO util.GSet: capacity = 2^18 = 262144 entries" @@ -1087,7 +1087,7 @@ { "@timestamp": "2023-06-26T13:45:50.764Z", "ecs": { - "version": "8.0.0" + "version": "8.11.0" }, "event": { "original": "2023-06-26 13:45:50,764 INFO metrics.TopMetrics: NNTop conf: dfs.namenode.top.window.num.buckets = 10" @@ -1106,7 +1106,7 @@ { "@timestamp": "2023-06-26T13:45:50.764Z", "ecs": { - "version": "8.0.0" + "version": "8.11.0" }, "event": { "original": "2023-06-26 13:45:50,764 INFO metrics.TopMetrics: NNTop conf: dfs.namenode.top.num.users = 10" @@ -1125,7 +1125,7 @@ { "@timestamp": "2023-06-26T13:45:50.764Z", "ecs": { - "version": "8.0.0" + "version": "8.11.0" }, "event": { "original": "2023-06-26 13:45:50,764 INFO metrics.TopMetrics: NNTop conf: dfs.namenode.top.windows.minutes = 1,5,25" @@ -1144,7 +1144,7 @@ { "@timestamp": "2023-06-26T13:45:50.768Z", "ecs": { - "version": "8.0.0" + "version": "8.11.0" }, "event": { "original": "2023-06-26 13:45:50,768 INFO namenode.FSNamesystem: Retry cache on namenode is enabled" @@ -1163,7 +1163,7 @@ { "@timestamp": "2023-06-26T13:45:50.768Z", "ecs": { - "version": "8.0.0" + "version": "8.11.0" }, "event": { "original": "2023-06-26 13:45:50,768 INFO namenode.FSNamesystem: Retry cache will use 0.03 of total heap and retry cache entry expiry time is 600000 millis" @@ -1182,7 +1182,7 @@ { "@timestamp": "2023-06-26T13:45:50.771Z", "ecs": { - "version": "8.0.0" + "version": "8.11.0" }, "event": { "original": "2023-06-26 13:45:50,771 INFO util.GSet: Computing capacity for map NameNodeRetryCache" @@ -1201,7 +1201,7 @@ { "@timestamp": "2023-06-26T13:45:50.771Z", "ecs": { - "version": "8.0.0" + "version": "8.11.0" }, "event": { "original": "2023-06-26 13:45:50,771 INFO util.GSet: VM type = 64-bit" @@ -1220,7 +1220,7 @@ { "@timestamp": "2023-06-26T13:45:50.771Z", "ecs": { - "version": "8.0.0" + "version": "8.11.0" }, "event": { "original": "2023-06-26 13:45:50,771 INFO util.GSet: 0.029999999329447746% max memory 864 MB = 265.4 KB" @@ -1239,7 +1239,7 @@ { "@timestamp": "2023-06-26T13:45:50.771Z", "ecs": { - "version": "8.0.0" + "version": "8.11.0" }, "event": { "original": "2023-06-26 13:45:50,771 INFO util.GSet: capacity = 2^15 = 32768 entries" @@ -1258,7 +1258,7 @@ { "@timestamp": "2023-06-26T13:45:50.774Z", "ecs": { - "version": "8.0.0" + "version": "8.11.0" }, "event": { "original": "2023-06-26 13:45:50,774 INFO namenode.FSNamesystem: Removal of Expired Lease on Open Files is enabled" @@ -1277,7 +1277,7 @@ { "@timestamp": "2023-06-26T13:45:50.811Z", "ecs": { - "version": "8.0.0" + "version": "8.11.0" }, "event": { "original": "2023-06-26 13:45:50,811 INFO namenode.FSImage: Allocated new BlockPoolId: BP-1979673447-172.31.25.102-1687787150800" @@ -1296,7 +1296,7 @@ { "@timestamp": "2023-06-26T13:45:50.824Z", "ecs": { - "version": "8.0.0" + "version": "8.11.0" }, "event": { "original": "2023-06-26 13:45:50,824 INFO common.Storage: Storage directory /mnt/namenode has been successfully formatted." @@ -1315,7 +1315,7 @@ { "@timestamp": "2023-06-26T13:45:50.826Z", "ecs": { - "version": "8.0.0" + "version": "8.11.0" }, "event": { "original": "2023-06-26 13:45:50,826 INFO common.Storage: Storage directory /mnt1/namenode has been successfully formatted." @@ -1334,7 +1334,7 @@ { "@timestamp": "2023-06-26T13:45:50.868Z", "ecs": { - "version": "8.0.0" + "version": "8.11.0" }, "event": { "original": "2023-06-26 13:45:50,868 INFO namenode.FSImageFormatProtobuf: Saving image file /mnt1/namenode/current/fsimage.ckpt_0000000000000000000 using no compression" @@ -1353,7 +1353,7 @@ { "@timestamp": "2023-06-26T13:45:50.868Z", "ecs": { - "version": "8.0.0" + "version": "8.11.0" }, "event": { "original": "2023-06-26 13:45:50,868 INFO namenode.FSImageFormatProtobuf: Saving image file /mnt/namenode/current/fsimage.ckpt_0000000000000000000 using no compression" @@ -1372,7 +1372,7 @@ { "@timestamp": "2023-06-26T13:45:51.228Z", "ecs": { - "version": "8.0.0" + "version": "8.11.0" }, "event": { "original": "2023-06-26 13:45:51,228 INFO namenode.FSImageFormatProtobuf: Image file /mnt1/namenode/current/fsimage.ckpt_0000000000000000000 of size 403 bytes saved in 0 seconds ." @@ -1391,7 +1391,7 @@ { "@timestamp": "2023-06-26T13:45:51.230Z", "ecs": { - "version": "8.0.0" + "version": "8.11.0" }, "event": { "original": "2023-06-26 13:45:51,230 INFO namenode.FSImageFormatProtobuf: Image file /mnt/namenode/current/fsimage.ckpt_0000000000000000000 of size 403 bytes saved in 0 seconds ." @@ -1410,7 +1410,7 @@ { "@timestamp": "2023-06-26T13:45:51.244Z", "ecs": { - "version": "8.0.0" + "version": "8.11.0" }, "event": { "original": "2023-06-26 13:45:51,244 INFO namenode.NNStorageRetentionManager: Going to retain 1 images with txid >= 0" @@ -1429,7 +1429,7 @@ { "@timestamp": "2023-06-26T13:45:51.301Z", "ecs": { - "version": "8.0.0" + "version": "8.11.0" }, "event": { "original": "2023-06-26 13:45:51,301 INFO namenode.FSNamesystem: Stopping services started for active state" @@ -1448,7 +1448,7 @@ { "@timestamp": "2023-06-26T13:45:51.301Z", "ecs": { - "version": "8.0.0" + "version": "8.11.0" }, "event": { "original": "2023-06-26 13:45:51,301 INFO namenode.FSNamesystem: Stopping services started for standby state" @@ -1467,7 +1467,7 @@ { "@timestamp": "2023-06-26T13:45:51.305Z", "ecs": { - "version": "8.0.0" + "version": "8.11.0" }, "event": { "original": "2023-06-26 13:45:51,305 INFO namenode.FSImage: FSImageSaver clean checkpoint: txid=0 when meet shutdown." @@ -1486,7 +1486,7 @@ { "@timestamp": "2023-06-26T13:45:51.306Z", "ecs": { - "version": "8.0.0" + "version": "8.11.0" }, "event": { "original": "2023-06-26 13:45:51,306 INFO namenode.FSImage: FSImageSaver clean checkpoint: txid=0 when meet shutdown." @@ -1505,7 +1505,7 @@ { "@timestamp": "2023-06-26T13:45:51.306Z", "ecs": { - "version": "8.0.0" + "version": "8.11.0" }, "event": { "original": "2023-06-26 13:45:51,306 INFO namenode.NameNode: SHUTDOWN_MSG: \n/************************************************************\nSHUTDOWN_MSG: Shutting down NameNode at ip-172-31-25-102.eu-central-1.compute.internal/172.31.25.102\n************************************************************/" diff --git a/packages/aws/data_stream/emr_logs/elasticsearch/ingest_pipeline/default.yml b/packages/aws/data_stream/emr_logs/elasticsearch/ingest_pipeline/default.yml index 55b24bc86c4..ab06cf2974f 100644 --- a/packages/aws/data_stream/emr_logs/elasticsearch/ingest_pipeline/default.yml +++ b/packages/aws/data_stream/emr_logs/elasticsearch/ingest_pipeline/default.yml @@ -3,7 +3,7 @@ description: "Pipeline for EMR logs" processors: - set: field: ecs.version - value: '8.0.0' + value: '8.11.0' - rename: field: message target_field: event.original diff --git a/packages/aws/data_stream/emr_logs/fields/ecs.yml b/packages/aws/data_stream/emr_logs/fields/ecs.yml deleted file mode 100644 index 22e4fc2780c..00000000000 --- a/packages/aws/data_stream/emr_logs/fields/ecs.yml +++ /dev/null @@ -1,65 +0,0 @@ -- external: ecs - name: ecs.version -- external: ecs - name: error.message -- external: ecs - name: message -- external: ecs - name: tags -- external: ecs - name: host.architecture -- external: ecs - name: host.domain -- external: ecs - name: host.hostname -- external: ecs - name: host.id -- external: ecs - name: host.ip -- external: ecs - name: host.mac -- external: ecs - name: host.name -- external: ecs - name: host.os.family -- external: ecs - name: host.os.kernel -- external: ecs - name: host.os.name -- external: ecs - name: host.os.platform -- external: ecs - name: host.os.version -- external: ecs - name: host.type -- external: ecs - name: cloud.account.id -- external: ecs - name: cloud.availability_zone -- external: ecs - name: cloud.instance.id -- external: ecs - name: cloud.instance.name -- external: ecs - name: cloud.machine.type -- external: ecs - name: cloud.provider -- external: ecs - name: cloud.region -- external: ecs - name: cloud.project.id -- external: ecs - name: container.id -- external: ecs - name: container.image.name -- external: ecs - name: container.labels -- external: ecs - name: container.name -- external: ecs - name: log.level -- external: ecs - name: log.file.path -- external: ecs - name: event.dataset - type: constant_keyword diff --git a/packages/aws/data_stream/emr_logs/fields/fields.yml b/packages/aws/data_stream/emr_logs/fields/fields.yml index 9c2b3e5d558..a0af2306dd6 100644 --- a/packages/aws/data_stream/emr_logs/fields/fields.yml +++ b/packages/aws/data_stream/emr_logs/fields/fields.yml @@ -17,9 +17,6 @@ type: flattened description: | AWS S3 object metadata values. -- name: process.name - type: keyword - description: Process name. - name: process.entrypoint type: keyword description: Process entrypoint. diff --git a/packages/aws/data_stream/emr_logs/sample_event.json b/packages/aws/data_stream/emr_logs/sample_event.json index 4cd70da2aeb..04dc644b003 100644 --- a/packages/aws/data_stream/emr_logs/sample_event.json +++ b/packages/aws/data_stream/emr_logs/sample_event.json @@ -27,7 +27,7 @@ "type": "logs" }, "ecs": { - "version": "8.0.0" + "version": "8.11.0" }, "elastic_agent": { "id": "acba78ef-1401-4689-977c-d8c2e5d6a8fa", diff --git a/packages/aws/data_stream/emr_metrics/fields/ecs.yml b/packages/aws/data_stream/emr_metrics/fields/ecs.yml index f91cecd5a3d..303e169c534 100644 --- a/packages/aws/data_stream/emr_metrics/fields/ecs.yml +++ b/packages/aws/data_stream/emr_metrics/fields/ecs.yml @@ -1,70 +1,9 @@ -- external: ecs - name: cloud - external: ecs name: cloud.account.id dimension: true -- external: ecs - name: cloud.account.name -- external: ecs - name: cloud.availability_zone -- external: ecs - name: cloud.instance.id -- external: ecs - name: cloud.instance.name -- external: ecs - name: cloud.project.id -- external: ecs - name: cloud.machine.type -- external: ecs - name: cloud.provider - external: ecs name: cloud.region dimension: true -- external: ecs - name: ecs.version -- external: ecs - name: error -- external: ecs - name: error.message -- external: ecs - name: service.type -- external: ecs - name: host.architecture -- external: ecs - name: host.domain -- external: ecs - name: host.hostname -- external: ecs - name: host.id -- external: ecs - name: host.ip -- external: ecs - name: host.mac -- external: ecs - name: host.name -- external: ecs - name: host.os.family -- external: ecs - name: host.os.kernel -- external: ecs - name: host.os.name -- external: ecs - name: host.os.platform -- external: ecs - name: host.os.version -- external: ecs - name: host.type -- external: ecs - name: container.id -- external: ecs - name: container.image.name -- external: ecs - name: container.labels -- external: ecs - name: container.name - name: agent.id external: ecs dimension: true -- external: ecs - name: event.dataset - type: constant_keyword diff --git a/packages/aws/data_stream/emr_metrics/sample_event.json b/packages/aws/data_stream/emr_metrics/sample_event.json index b9d58537612..fd862b66809 100644 --- a/packages/aws/data_stream/emr_metrics/sample_event.json +++ b/packages/aws/data_stream/emr_metrics/sample_event.json @@ -1,60 +1,60 @@ { "@timestamp": "2022-07-26T21:43:00.000Z", "agent": { - "name": "docker-fleet-agent", + "ephemeral_id": "cdaaaabb-be7e-432f-816b-bda019fd7c15", "id": "2d4b09d0-cdb6-445e-ac3f-6415f87b9864", + "name": "docker-fleet-agent", "type": "metricbeat", - "ephemeral_id": "cdaaaabb-be7e-432f-816b-bda019fd7c15", "version": "8.3.2" }, - "elastic_agent": { - "id": "2d4b09d0-cdb6-445e-ac3f-6415f87b9864", - "version": "8.3.2", - "snapshot": false - }, - "cloud": { - "provider": "aws", - "region": "eu-central-1", - "account": { - "name": "elastic-beats", - "id": "428152502467" - } - }, - "ecs": { - "version": "8.0.0" - }, - "service": { - "type": "aws" - }, - "data_stream": { - "namespace": "default", - "type": "metrics", - "dataset": "aws.emr_metrics" - }, - "metricset": { - "period": 300000, - "name": "cloudwatch" - }, "aws": { + "cloudwatch": { + "namespace": "AWS/ElasticMapReduce" + }, + "dimensions": { + "JobFlowId": "j-3LRBO17JBA7H9" + }, "elasticmapreduce": { "metrics": { "IsIdle": { "avg": 1 } } - }, - "cloudwatch": { - "namespace": "AWS/ElasticMapReduce" - }, - "dimensions": { - "JobFlowId": "j-3LRBO17JBA7H9" } }, + "cloud": { + "account": { + "id": "428152502467", + "name": "elastic-beats" + }, + "provider": "aws", + "region": "eu-central-1" + }, + "data_stream": { + "dataset": "aws.emr_metrics", + "namespace": "default", + "type": "metrics" + }, + "ecs": { + "version": "8.11.0" + }, + "elastic_agent": { + "id": "2d4b09d0-cdb6-445e-ac3f-6415f87b9864", + "snapshot": false, + "version": "8.3.2" + }, "event": { - "duration": 11576777300, "agent_id_status": "verified", + "dataset": "aws.emr_metrics", + "duration": 11576777300, "ingested": "2022-07-26T21:47:48Z", - "module": "aws", - "dataset": "aws.emr_metrics" + "module": "aws" + }, + "metricset": { + "name": "cloudwatch", + "period": 300000 + }, + "service": { + "type": "aws" } } \ No newline at end of file diff --git a/packages/aws/data_stream/firewall_logs/_dev/test/pipeline/test-firewall.log-expected.json b/packages/aws/data_stream/firewall_logs/_dev/test/pipeline/test-firewall.log-expected.json index fad4be2693e..969787d35c5 100644 --- a/packages/aws/data_stream/firewall_logs/_dev/test/pipeline/test-firewall.log-expected.json +++ b/packages/aws/data_stream/firewall_logs/_dev/test/pipeline/test-firewall.log-expected.json @@ -35,7 +35,7 @@ "port": 80 }, "ecs": { - "version": "8.0.0" + "version": "8.11.0" }, "event": { "category": [ @@ -164,7 +164,7 @@ "port": 5060 }, "ecs": { - "version": "8.0.0" + "version": "8.11.0" }, "event": { "category": [ diff --git a/packages/aws/data_stream/firewall_logs/elasticsearch/ingest_pipeline/default.yml b/packages/aws/data_stream/firewall_logs/elasticsearch/ingest_pipeline/default.yml index 2e90c9f022e..a322268a61b 100644 --- a/packages/aws/data_stream/firewall_logs/elasticsearch/ingest_pipeline/default.yml +++ b/packages/aws/data_stream/firewall_logs/elasticsearch/ingest_pipeline/default.yml @@ -4,7 +4,7 @@ processors: # General data - set: field: ecs.version - value: 8.0.0 + value: 8.11.0 - rename: field: message target_field: event.original diff --git a/packages/aws/data_stream/firewall_logs/fields/ecs.yml b/packages/aws/data_stream/firewall_logs/fields/ecs.yml deleted file mode 100644 index e50caaf839a..00000000000 --- a/packages/aws/data_stream/firewall_logs/fields/ecs.yml +++ /dev/null @@ -1,204 +0,0 @@ -- external: ecs - name: cloud.account.id -- external: ecs - name: cloud.region -- external: ecs - name: ecs.version -- external: ecs - name: error.message -- external: ecs - name: event.action -- external: ecs - name: event.category -- external: ecs - name: event.created -- external: ecs - name: event.ingested -- external: ecs - name: event.kind -- external: ecs - name: event.original -- external: ecs - name: event.provider -- external: ecs - name: event.type -- external: ecs - name: destination.bytes -- external: ecs - name: destination.as.number -- external: ecs - name: destination.as.organization.name -- external: ecs - name: destination.domain -- external: ecs - name: destination.geo.city_name -- external: ecs - name: destination.geo.continent_name -- external: ecs - name: destination.geo.country_iso_code -- external: ecs - name: destination.geo.country_name -- external: ecs - name: destination.geo.location - description: Longitude and latitude. - example: '{ "lon": -73.614830, "lat": 45.505918 }' - type: geo_point -- external: ecs - name: destination.geo.name -- external: ecs - name: destination.geo.region_iso_code -- external: ecs - name: destination.geo.region_name -- external: ecs - name: destination.ip -- external: ecs - name: destination.address -- external: ecs - name: destination.port -- external: ecs - name: group.id -- external: ecs - name: group.name -- external: ecs - name: http.request.method -- external: ecs - name: http.version -- external: ecs - name: message -- external: ecs - name: network.community_id -- external: ecs - name: network.protocol -- external: ecs - name: network.transport -- external: ecs - name: network.type -- external: ecs - name: observer.name -- external: ecs - name: observer.product -- external: ecs - name: observer.type -- external: ecs - name: observer.vendor -- external: ecs - name: related.hosts -- external: ecs - name: related.ip -- external: ecs - name: rule.id -- external: ecs - name: rule.name -- external: ecs - name: rule.category -- external: ecs - name: source.address -- external: ecs - name: source.as.number -- external: ecs - name: source.as.organization.name -- external: ecs - name: source.geo.city_name -- external: ecs - name: source.geo.continent_name -- external: ecs - name: source.geo.country_iso_code -- external: ecs - name: source.geo.country_name -- description: Longitude and latitude. - level: core - name: source.geo.location - type: geo_point -- external: ecs - name: source.geo.region_iso_code -- external: ecs - name: source.geo.region_name -- external: ecs - name: source.port -- external: ecs - name: source.ip -- external: ecs - name: tags -- external: ecs - name: url.original -- external: ecs - name: url.path -- external: ecs - name: url.domain -- external: ecs - name: url.scheme -- external: ecs - name: user.changes.name -- external: ecs - name: user.id -- external: ecs - name: user.name -- external: ecs - name: user.target.id -- external: ecs - name: user.target.name -- external: ecs - name: user_agent.device.name -- external: ecs - name: user_agent.name -- external: ecs - name: user_agent.original -- external: ecs - name: user_agent.os.full -- external: ecs - name: user_agent.os.name -- external: ecs - name: user_agent.os.version -- external: ecs - name: user_agent.version -- external: ecs - name: host.architecture -- external: ecs - name: host.domain -- external: ecs - name: host.hostname -- external: ecs - name: host.id -- external: ecs - name: host.ip -- external: ecs - name: host.mac -- external: ecs - name: host.name -- external: ecs - name: host.os.family -- external: ecs - name: host.os.kernel -- external: ecs - name: host.os.name -- external: ecs - name: host.os.platform -- external: ecs - name: host.os.version -- external: ecs - name: host.type -- external: ecs - name: cloud.availability_zone -- external: ecs - name: cloud.instance.id -- external: ecs - name: cloud.instance.name -- external: ecs - name: cloud.machine.type -- external: ecs - name: cloud.provider -- external: ecs - name: cloud.project.id -- external: ecs - name: container.id -- external: ecs - name: container.image.name -- external: ecs - name: container.labels -- external: ecs - name: container.name -- external: ecs - name: log.file.path -- external: ecs - name: event.dataset - type: constant_keyword diff --git a/packages/aws/data_stream/firewall_logs/sample_event.json b/packages/aws/data_stream/firewall_logs/sample_event.json index d644c062aa8..cedc9523fa8 100644 --- a/packages/aws/data_stream/firewall_logs/sample_event.json +++ b/packages/aws/data_stream/firewall_logs/sample_event.json @@ -55,7 +55,7 @@ "port": 80 }, "ecs": { - "version": "8.0.0" + "version": "8.11.0" }, "elastic_agent": { "id": "acba78ef-1401-4689-977c-d8c2e5d6a8fa", diff --git a/packages/aws/data_stream/firewall_metrics/fields/ecs.yml b/packages/aws/data_stream/firewall_metrics/fields/ecs.yml index 654c9b2e202..303e169c534 100644 --- a/packages/aws/data_stream/firewall_metrics/fields/ecs.yml +++ b/packages/aws/data_stream/firewall_metrics/fields/ecs.yml @@ -1,70 +1,9 @@ -- external: ecs - name: cloud - external: ecs name: cloud.account.id dimension: true -- external: ecs - name: cloud.account.name -- external: ecs - name: cloud.availability_zone -- external: ecs - name: cloud.instance.id -- external: ecs - name: cloud.machine.type -- external: ecs - name: cloud.provider - external: ecs name: cloud.region dimension: true -- external: ecs - name: ecs.version -- external: ecs - name: error -- external: ecs - name: error.message -- external: ecs - name: service.type -- external: ecs - name: host.architecture -- external: ecs - name: host.domain -- external: ecs - name: host.hostname -- external: ecs - name: host.id -- external: ecs - name: host.ip -- external: ecs - name: host.mac -- external: ecs - name: host.name -- external: ecs - name: host.os.family -- external: ecs - name: host.os.kernel -- external: ecs - name: host.os.name -- external: ecs - name: host.os.platform -- external: ecs - name: host.os.version -- external: ecs - name: host.type -- external: ecs - name: cloud.instance.name -- external: ecs - name: cloud.project.id -- external: ecs - name: container.id -- external: ecs - name: container.image.name -- external: ecs - name: container.labels -- external: ecs - name: container.name - name: agent.id external: ecs dimension: true -- external: ecs - name: event.dataset - type: constant_keyword diff --git a/packages/aws/data_stream/firewall_metrics/sample_event.json b/packages/aws/data_stream/firewall_metrics/sample_event.json index 5864b569a1a..3609f4ac837 100644 --- a/packages/aws/data_stream/firewall_metrics/sample_event.json +++ b/packages/aws/data_stream/firewall_metrics/sample_event.json @@ -1,64 +1,64 @@ { "@timestamp": "2020-05-28T17:58:27.154Z", - "service": { - "type": "aws" - }, - "ecs": { - "version": "8.0.0" + "agent": { + "ephemeral_id": "d3f31d10-7f16-4834-ae22-0df946c61f92", + "hostname": "docker-fleet-agent", + "id": "88c94c53-cbfe-4657-9a08-527b09d94cee", + "name": "docker-fleet-agent", + "type": "metricbeat", + "version": "7.15.0" }, "aws": { + "cloudwatch": { + "namespace": "AWS/NetworkFirewall" + }, + "dimensions": { + "AvailabilityZone": "us-east-2a", + "Engine": "Stateful", + "FirewallName": "AWSNetworkFirewall" + }, "networkfirewall": { "metrics": { - "PassedPackets": { - "sum": 0 - }, "DroppedPackets": { "sum": 4 }, + "PassedPackets": { + "sum": 0 + }, "ReceivedPackets": { "sum": 4 } } - }, - "cloudwatch": { - "namespace": "AWS/NetworkFirewall" - }, - "dimensions": { - "FirewallName": "AWSNetworkFirewall", - "AvailabilityZone": "us-east-2a", - "Engine": "Stateful" } }, - "event": { - "duration": 8925713800, - "agent_id_status": "verified", - "ingested": "2021-11-18T17:18:46Z", - "module": "aws", - "dataset": "aws.firewall_metrics" - }, - "metricset": { - "period": 60000, - "name": "cloudwatch" - }, "cloud": { - "provider": "aws", - "region": "us-east-2", "account": { - "name": "elastic-beats", - "id": "428152502467" - } + "id": "428152502467", + "name": "elastic-beats" + }, + "provider": "aws", + "region": "us-east-2" }, "data_stream": { + "dataset": "aws.firewall_metrics", "namespace": "default", - "type": "metrics", - "dataset": "aws.firewall_metrics" + "type": "metrics" }, - "agent": { - "hostname": "docker-fleet-agent", - "name": "docker-fleet-agent", - "id": "88c94c53-cbfe-4657-9a08-527b09d94cee", - "type": "metricbeat", - "ephemeral_id": "d3f31d10-7f16-4834-ae22-0df946c61f92", - "version": "7.15.0" + "ecs": { + "version": "8.11.0" + }, + "event": { + "agent_id_status": "verified", + "dataset": "aws.firewall_metrics", + "duration": 8925713800, + "ingested": "2021-11-18T17:18:46Z", + "module": "aws" + }, + "metricset": { + "name": "cloudwatch", + "period": 60000 + }, + "service": { + "type": "aws" } } \ No newline at end of file diff --git a/packages/aws/data_stream/guardduty/_dev/test/pipeline/test-guardduty.log-expected.json b/packages/aws/data_stream/guardduty/_dev/test/pipeline/test-guardduty.log-expected.json index e5cf7434be3..0f6f89a84ef 100644 --- a/packages/aws/data_stream/guardduty/_dev/test/pipeline/test-guardduty.log-expected.json +++ b/packages/aws/data_stream/guardduty/_dev/test/pipeline/test-guardduty.log-expected.json @@ -178,7 +178,7 @@ } }, "ecs": { - "version": "8.10.0" + "version": "8.11.0" }, "event": { "action": "DNS_REQUEST", @@ -335,7 +335,7 @@ } }, "ecs": { - "version": "8.10.0" + "version": "8.11.0" }, "event": { "action": "KUBERNETES_API_CALL", @@ -548,7 +548,7 @@ } }, "ecs": { - "version": "8.10.0" + "version": "8.11.0" }, "event": { "action": "KUBERNETES_API_CALL", @@ -745,7 +745,7 @@ } }, "ecs": { - "version": "8.10.0" + "version": "8.11.0" }, "event": { "action": "RDS_LOGIN_ATTEMPT", diff --git a/packages/aws/data_stream/guardduty/elasticsearch/ingest_pipeline/default.yml b/packages/aws/data_stream/guardduty/elasticsearch/ingest_pipeline/default.yml index 1bc3a9b224b..9e9836130b9 100644 --- a/packages/aws/data_stream/guardduty/elasticsearch/ingest_pipeline/default.yml +++ b/packages/aws/data_stream/guardduty/elasticsearch/ingest_pipeline/default.yml @@ -3,7 +3,7 @@ description: Pipeline for processing Amazon GuardDuty Findings logs. processors: - set: field: ecs.version - value: '8.10.0' + value: '8.11.0' - set: field: event.kind value: [event] diff --git a/packages/aws/data_stream/guardduty/fields/ecs.yml b/packages/aws/data_stream/guardduty/fields/ecs.yml deleted file mode 100644 index e7e0d87961d..00000000000 --- a/packages/aws/data_stream/guardduty/fields/ecs.yml +++ /dev/null @@ -1,145 +0,0 @@ -- external: ecs - name: cloud.account.id -- external: ecs - name: cloud.account.name -- external: ecs - name: cloud.availability_zone -- external: ecs - name: cloud.instance.id -- external: ecs - name: cloud.instance.name -- external: ecs - name: cloud.machine.type -- external: ecs - name: cloud.project.id -- external: ecs - name: cloud.provider -- external: ecs - name: cloud.region -- external: ecs - name: cloud.service.name -- external: ecs - name: container.id -- external: ecs - name: container.image.name -- external: ecs - name: container.labels -- external: ecs - name: container.name -- external: ecs - name: container.runtime -- external: ecs - name: destination.address -- external: ecs - name: dns.question.name -- external: ecs - name: ecs.version -- external: ecs - name: event.action -- external: ecs - name: event.created -- external: ecs - name: event.end -- external: ecs - name: event.id -- external: ecs - name: event.kind -- external: ecs - name: event.original -- external: ecs - name: event.provider -- external: ecs - name: event.severity -- external: ecs - name: event.start -- external: ecs - name: event.type -- external: ecs - name: file.hash.sha256 -- external: ecs - name: file.name -- external: ecs - name: file.path -- external: ecs - name: host.architecture -- external: ecs - name: host.domain -- external: ecs - name: host.hostname -- external: ecs - name: host.id -- external: ecs - name: host.ip -- external: ecs - name: host.mac -- external: ecs - name: host.name -- external: ecs - name: host.os.family -- external: ecs - name: host.os.kernel -- external: ecs - name: host.os.name -- external: ecs - name: host.os.platform -- external: ecs - name: host.os.version -- external: ecs - name: host.type -- external: ecs - name: message -- external: ecs - name: network.direction -- external: ecs - name: network.transport -- external: ecs - name: orchestrator.namespace -- external: ecs - name: orchestrator.resource.name -- external: ecs - name: orchestrator.resource.type -- external: ecs - name: related.hash -- external: ecs - name: related.hosts -- external: ecs - name: related.ip -- external: ecs - name: related.user -- external: ecs - name: rule.category -- external: ecs - name: rule.name -- external: ecs - name: rule.ruleset -- external: ecs - name: source.address -- external: ecs - name: source.as.number -- external: ecs - name: source.as.organization.name -- external: ecs - name: source.geo.city_name -- external: ecs - name: source.geo.country_iso_code -- external: ecs - name: source.geo.country_name -- external: ecs - name: source.geo.location -- external: ecs - name: source.ip -- external: ecs - name: source.port -- external: ecs - name: tags -- external: ecs - name: user.id -- external: ecs - name: user.name -- external: ecs - name: user.roles -- external: ecs - name: container.security_context.privileged -- external: ecs - name: event.dataset - type: constant_keyword diff --git a/packages/aws/data_stream/guardduty/sample_event.json b/packages/aws/data_stream/guardduty/sample_event.json index e27265c9a5e..f3f013e3c61 100644 --- a/packages/aws/data_stream/guardduty/sample_event.json +++ b/packages/aws/data_stream/guardduty/sample_event.json @@ -143,7 +143,7 @@ "type": "logs" }, "ecs": { - "version": "8.10.0" + "version": "8.11.0" }, "elastic_agent": { "id": "9e5875f3-d206-43b3-b24e-5a5096e50846", diff --git a/packages/aws/data_stream/inspector/_dev/test/pipeline/test-inspector.log-expected.json b/packages/aws/data_stream/inspector/_dev/test/pipeline/test-inspector.log-expected.json index 43ca2f87b3f..f6e8e340aec 100644 --- a/packages/aws/data_stream/inspector/_dev/test/pipeline/test-inspector.log-expected.json +++ b/packages/aws/data_stream/inspector/_dev/test/pipeline/test-inspector.log-expected.json @@ -166,7 +166,7 @@ ] }, "ecs": { - "version": "8.2.0" + "version": "8.11.0" }, "event": { "kind": "event", diff --git a/packages/aws/data_stream/inspector/elasticsearch/ingest_pipeline/default.yml b/packages/aws/data_stream/inspector/elasticsearch/ingest_pipeline/default.yml index 7bc529093af..ed30dcc0759 100644 --- a/packages/aws/data_stream/inspector/elasticsearch/ingest_pipeline/default.yml +++ b/packages/aws/data_stream/inspector/elasticsearch/ingest_pipeline/default.yml @@ -3,7 +3,7 @@ description: Pipeline for processing AWS Inspector Findings logs. processors: - set: field: ecs.version - value: '8.2.0' + value: '8.11.0' - set: field: event.kind value: event diff --git a/packages/aws/data_stream/inspector/fields/ecs.yml b/packages/aws/data_stream/inspector/fields/ecs.yml deleted file mode 100644 index 532790e821d..00000000000 --- a/packages/aws/data_stream/inspector/fields/ecs.yml +++ /dev/null @@ -1,89 +0,0 @@ -- external: ecs - name: cloud.account.id -- external: ecs - name: cloud.account.name -- external: ecs - name: cloud.availability_zone -- external: ecs - name: cloud.instance.id -- external: ecs - name: cloud.instance.name -- external: ecs - name: cloud.machine.type -- external: ecs - name: cloud.project.id -- external: ecs - name: cloud.provider -- external: ecs - name: cloud.region -- external: ecs - name: container.id -- external: ecs - name: container.image.name -- external: ecs - name: container.labels -- external: ecs - name: container.name -- external: ecs - name: ecs.version -- external: ecs - name: event.category -- external: ecs - name: event.created -- external: ecs - name: event.kind -- external: ecs - name: event.original -- external: ecs - name: event.severity -- external: ecs - name: event.type -- external: ecs - name: host.architecture -- external: ecs - name: host.domain -- external: ecs - name: host.hostname -- external: ecs - name: host.id -- external: ecs - name: host.ip -- external: ecs - name: host.mac -- external: ecs - name: host.name -- external: ecs - name: host.os.family -- external: ecs - name: host.os.kernel -- external: ecs - name: host.os.name -- external: ecs - name: host.os.platform -- external: ecs - name: host.os.version -- external: ecs - name: host.type -- external: ecs - name: message -- external: ecs - name: network.transport -- external: ecs - name: related.hash -- external: ecs - name: related.ip -- external: ecs - name: tags -- external: ecs - name: vulnerability.id -- external: ecs - name: vulnerability.reference -- external: ecs - name: vulnerability.score.base -- external: ecs - name: vulnerability.score.version -- external: ecs - name: vulnerability.severity -- external: ecs - name: event.dataset - type: constant_keyword diff --git a/packages/aws/data_stream/inspector/sample_event.json b/packages/aws/data_stream/inspector/sample_event.json index 5e1c1cc54c4..2116bc8251e 100644 --- a/packages/aws/data_stream/inspector/sample_event.json +++ b/packages/aws/data_stream/inspector/sample_event.json @@ -164,7 +164,7 @@ "type": "logs" }, "ecs": { - "version": "8.2.0" + "version": "8.11.0" }, "elastic_agent": { "id": "4a3373c9-b63f-4544-a929-761b42f50054", diff --git a/packages/aws/data_stream/kafka_metrics/fields/ecs.yml b/packages/aws/data_stream/kafka_metrics/fields/ecs.yml index 654c9b2e202..303e169c534 100644 --- a/packages/aws/data_stream/kafka_metrics/fields/ecs.yml +++ b/packages/aws/data_stream/kafka_metrics/fields/ecs.yml @@ -1,70 +1,9 @@ -- external: ecs - name: cloud - external: ecs name: cloud.account.id dimension: true -- external: ecs - name: cloud.account.name -- external: ecs - name: cloud.availability_zone -- external: ecs - name: cloud.instance.id -- external: ecs - name: cloud.machine.type -- external: ecs - name: cloud.provider - external: ecs name: cloud.region dimension: true -- external: ecs - name: ecs.version -- external: ecs - name: error -- external: ecs - name: error.message -- external: ecs - name: service.type -- external: ecs - name: host.architecture -- external: ecs - name: host.domain -- external: ecs - name: host.hostname -- external: ecs - name: host.id -- external: ecs - name: host.ip -- external: ecs - name: host.mac -- external: ecs - name: host.name -- external: ecs - name: host.os.family -- external: ecs - name: host.os.kernel -- external: ecs - name: host.os.name -- external: ecs - name: host.os.platform -- external: ecs - name: host.os.version -- external: ecs - name: host.type -- external: ecs - name: cloud.instance.name -- external: ecs - name: cloud.project.id -- external: ecs - name: container.id -- external: ecs - name: container.image.name -- external: ecs - name: container.labels -- external: ecs - name: container.name - name: agent.id external: ecs dimension: true -- external: ecs - name: event.dataset - type: constant_keyword diff --git a/packages/aws/data_stream/kafka_metrics/sample_event.json b/packages/aws/data_stream/kafka_metrics/sample_event.json index 5862e61e1c0..7d170c60ef5 100644 --- a/packages/aws/data_stream/kafka_metrics/sample_event.json +++ b/packages/aws/data_stream/kafka_metrics/sample_event.json @@ -44,7 +44,7 @@ "type": "metrics" }, "ecs": { - "version": "8.0.0" + "version": "8.11.0" }, "elastic_agent": { "id": "0395c9d5-9ac1-4ecc-bfd5-fc5376847519", diff --git a/packages/aws/data_stream/kinesis/fields/ecs.yml b/packages/aws/data_stream/kinesis/fields/ecs.yml index 80bcd50b690..303e169c534 100644 --- a/packages/aws/data_stream/kinesis/fields/ecs.yml +++ b/packages/aws/data_stream/kinesis/fields/ecs.yml @@ -1,70 +1,9 @@ -- external: ecs - name: cloud - external: ecs name: cloud.account.id dimension: true -- external: ecs - name: cloud.account.name -- external: ecs - name: cloud.availability_zone -- external: ecs - name: cloud.instance.id -- external: ecs - name: cloud.instance.name -- external: ecs - name: cloud.machine.type -- external: ecs - name: cloud.project.id -- external: ecs - name: cloud.provider - external: ecs name: cloud.region dimension: true -- external: ecs - name: container.image.name -- external: ecs - name: container.labels -- external: ecs - name: container.name -- external: ecs - name: ecs.version -- external: ecs - name: error -- external: ecs - name: error.message -- external: ecs - name: service.type -- external: ecs - name: host.architecture -- external: ecs - name: host.domain -- external: ecs - name: host.hostname -- external: ecs - name: host.id -- external: ecs - name: host.ip -- external: ecs - name: host.mac -- external: ecs - name: host.name -- external: ecs - name: host.os.family -- external: ecs - name: host.os.kernel -- external: ecs - name: host.os.name -- external: ecs - name: host.os.platform -- external: ecs - name: host.os.version -- external: ecs - name: host.type -- external: ecs - name: container.id - name: agent.id external: ecs dimension: true -- external: ecs - name: event.dataset - type: constant_keyword diff --git a/packages/aws/data_stream/kinesis/sample_event.json b/packages/aws/data_stream/kinesis/sample_event.json index dec427115e9..6119594e27f 100644 --- a/packages/aws/data_stream/kinesis/sample_event.json +++ b/packages/aws/data_stream/kinesis/sample_event.json @@ -1,40 +1,12 @@ { "@timestamp": "2022-07-27T20:56:00.000Z", "agent": { - "name": "docker-fleet-agent", - "id": "2d4b09d0-cdb6-445e-ac3f-6415f87b9864", "ephemeral_id": "51866723-6dfa-4a72-a68e-f439d5de7f53", + "id": "2d4b09d0-cdb6-445e-ac3f-6415f87b9864", + "name": "docker-fleet-agent", "type": "metricbeat", "version": "8.3.2" }, - "elastic_agent": { - "id": "2d4b09d0-cdb6-445e-ac3f-6415f87b9864", - "version": "8.3.2", - "snapshot": false - }, - "cloud": { - "provider": "aws", - "region": "us-east-1", - "account": { - "name": "elastic-beats", - "id": "428152502467" - } - }, - "ecs": { - "version": "8.0.0" - }, - "data_stream": { - "namespace": "default", - "type": "metrics", - "dataset": "aws.kinesis" - }, - "service": { - "type": "aws" - }, - "metricset": { - "period": 300000, - "name": "cloudwatch" - }, "aws": { "cloudwatch": { "namespace": "AWS/Kinesis" @@ -65,11 +37,39 @@ } } }, + "cloud": { + "account": { + "id": "428152502467", + "name": "elastic-beats" + }, + "provider": "aws", + "region": "us-east-1" + }, + "data_stream": { + "dataset": "aws.kinesis", + "namespace": "default", + "type": "metrics" + }, + "ecs": { + "version": "8.11.0" + }, + "elastic_agent": { + "id": "2d4b09d0-cdb6-445e-ac3f-6415f87b9864", + "snapshot": false, + "version": "8.3.2" + }, "event": { - "duration": 10483932100, "agent_id_status": "verified", + "dataset": "aws.kinesis", + "duration": 10483932100, "ingested": "2022-07-27T20:56:00.000Z", - "module": "aws", - "dataset": "aws.kinesis" + "module": "aws" + }, + "metricset": { + "name": "cloudwatch", + "period": 300000 + }, + "service": { + "type": "aws" } } \ No newline at end of file diff --git a/packages/aws/data_stream/lambda/fields/ecs.yml b/packages/aws/data_stream/lambda/fields/ecs.yml index 654c9b2e202..303e169c534 100644 --- a/packages/aws/data_stream/lambda/fields/ecs.yml +++ b/packages/aws/data_stream/lambda/fields/ecs.yml @@ -1,70 +1,9 @@ -- external: ecs - name: cloud - external: ecs name: cloud.account.id dimension: true -- external: ecs - name: cloud.account.name -- external: ecs - name: cloud.availability_zone -- external: ecs - name: cloud.instance.id -- external: ecs - name: cloud.machine.type -- external: ecs - name: cloud.provider - external: ecs name: cloud.region dimension: true -- external: ecs - name: ecs.version -- external: ecs - name: error -- external: ecs - name: error.message -- external: ecs - name: service.type -- external: ecs - name: host.architecture -- external: ecs - name: host.domain -- external: ecs - name: host.hostname -- external: ecs - name: host.id -- external: ecs - name: host.ip -- external: ecs - name: host.mac -- external: ecs - name: host.name -- external: ecs - name: host.os.family -- external: ecs - name: host.os.kernel -- external: ecs - name: host.os.name -- external: ecs - name: host.os.platform -- external: ecs - name: host.os.version -- external: ecs - name: host.type -- external: ecs - name: cloud.instance.name -- external: ecs - name: cloud.project.id -- external: ecs - name: container.id -- external: ecs - name: container.image.name -- external: ecs - name: container.labels -- external: ecs - name: container.name - name: agent.id external: ecs dimension: true -- external: ecs - name: event.dataset - type: constant_keyword diff --git a/packages/aws/data_stream/lambda/sample_event.json b/packages/aws/data_stream/lambda/sample_event.json index 377d69aa036..eb34d64b6c1 100644 --- a/packages/aws/data_stream/lambda/sample_event.json +++ b/packages/aws/data_stream/lambda/sample_event.json @@ -1,72 +1,72 @@ { "@timestamp": "2022-07-19T22:40:00.000Z", "agent": { - "name": "docker-fleet-agent", + "ephemeral_id": "ed2abfa1-df5e-4c3e-9c2b-143edcc0e111", "id": "2d4b09d0-cdb6-445e-ac3f-6415f87b9864", + "name": "docker-fleet-agent", "type": "metricbeat", - "ephemeral_id": "ed2abfa1-df5e-4c3e-9c2b-143edcc0e111", "version": "8.3.2" }, - "elastic_agent": { - "id": "2d4b09d0-cdb6-445e-ac3f-6415f87b9864", - "version": "8.3.2", - "snapshot": false - }, - "cloud": { - "provider": "aws", - "region": "eu-central-1", - "account": { - "name": "elastic-observability", - "id": "627286350134" - } - }, - "ecs": { - "version": "8.0.0" - }, - "service": { - "type": "aws" - }, - "data_stream": { - "namespace": "default", - "type": "metrics", - "dataset": "aws.lambda" - }, - "metricset": { - "period": 300000, - "name": "cloudwatch" - }, "aws": { + "cloudwatch": { + "namespace": "AWS/Lambda" + }, "lambda": { "metrics": { - "Errors": { - "avg": 0 - }, "ConcurrentExecutions": { "avg": 1 }, - "Invocations": { - "avg": 1 - }, - "UnreservedConcurrentExecutions": { - "avg": 1 - }, "Duration": { "avg": 130.97 }, + "Errors": { + "avg": 0 + }, + "Invocations": { + "avg": 1 + }, "Throttles": { "avg": 0 + }, + "UnreservedConcurrentExecutions": { + "avg": 1 } } - }, - "cloudwatch": { - "namespace": "AWS/Lambda" } }, + "cloud": { + "account": { + "id": "627286350134", + "name": "elastic-observability" + }, + "provider": "aws", + "region": "eu-central-1" + }, + "data_stream": { + "dataset": "aws.lambda", + "namespace": "default", + "type": "metrics" + }, + "ecs": { + "version": "8.11.0" + }, + "elastic_agent": { + "id": "2d4b09d0-cdb6-445e-ac3f-6415f87b9864", + "snapshot": false, + "version": "8.3.2" + }, "event": { - "duration": 11364562400, "agent_id_status": "verified", + "dataset": "aws.lambda", + "duration": 11364562400, "ingested": "2022-07-26T22:40:40Z", - "module": "aws", - "dataset": "aws.lambda" + "module": "aws" + }, + "metricset": { + "name": "cloudwatch", + "period": 300000 + }, + "service": { + "type": "aws" } } \ No newline at end of file diff --git a/packages/aws/data_stream/natgateway/fields/ecs.yml b/packages/aws/data_stream/natgateway/fields/ecs.yml index 654c9b2e202..303e169c534 100644 --- a/packages/aws/data_stream/natgateway/fields/ecs.yml +++ b/packages/aws/data_stream/natgateway/fields/ecs.yml @@ -1,70 +1,9 @@ -- external: ecs - name: cloud - external: ecs name: cloud.account.id dimension: true -- external: ecs - name: cloud.account.name -- external: ecs - name: cloud.availability_zone -- external: ecs - name: cloud.instance.id -- external: ecs - name: cloud.machine.type -- external: ecs - name: cloud.provider - external: ecs name: cloud.region dimension: true -- external: ecs - name: ecs.version -- external: ecs - name: error -- external: ecs - name: error.message -- external: ecs - name: service.type -- external: ecs - name: host.architecture -- external: ecs - name: host.domain -- external: ecs - name: host.hostname -- external: ecs - name: host.id -- external: ecs - name: host.ip -- external: ecs - name: host.mac -- external: ecs - name: host.name -- external: ecs - name: host.os.family -- external: ecs - name: host.os.kernel -- external: ecs - name: host.os.name -- external: ecs - name: host.os.platform -- external: ecs - name: host.os.version -- external: ecs - name: host.type -- external: ecs - name: cloud.instance.name -- external: ecs - name: cloud.project.id -- external: ecs - name: container.id -- external: ecs - name: container.image.name -- external: ecs - name: container.labels -- external: ecs - name: container.name - name: agent.id external: ecs dimension: true -- external: ecs - name: event.dataset - type: constant_keyword diff --git a/packages/aws/data_stream/natgateway/sample_event.json b/packages/aws/data_stream/natgateway/sample_event.json index 1d1c02b7847..07f280c60b6 100644 --- a/packages/aws/data_stream/natgateway/sample_event.json +++ b/packages/aws/data_stream/natgateway/sample_event.json @@ -1,116 +1,116 @@ { + "@timestamp": "2022-07-27T22:02:00.000Z", "agent": { - "name": "a3fc2d7bc1c5", - "id": "8940152e-2f20-4ad1-bc96-4db45cb7fc89", "ephemeral_id": "b7f3d3f4-137a-443f-90a7-ad2a5d81f81b", - "type": "metricbeat", - "version": "8.1.0" - }, - "elastic_agent": { "id": "8940152e-2f20-4ad1-bc96-4db45cb7fc89", - "version": "8.1.0", - "snapshot": false - }, - "cloud": { - "provider": "aws", - "region": "eu-west-1" - }, - "@timestamp": "2022-07-27T22:02:00.000Z", - "ecs": { - "version": "8.0.0" - }, - "service": { - "type": "aws" - }, - "data_stream": { - "namespace": "default", - "type": "metrics", - "dataset": "aws.natgateway" - }, - "host": { - "hostname": "a3fc2d7bc1c5", - "os": { - "kernel": "5.10.104-linuxkit", - "codename": "focal", - "name": "Ubuntu", - "type": "linux", - "family": "debian", - "version": "20.04.3 LTS (Focal Fossa)", - "platform": "ubuntu" - }, - "containerized": false, - "ip": [ - "172.20.0.7" - ], "name": "a3fc2d7bc1c5", - "mac": [ - "02-42-AC-14-00-07" - ], - "architecture": "aarch64" - }, - "metricset": { - "period": 180000, - "name": "cloudwatch" + "type": "metricbeat", + "version": "8.1.0" }, "aws": { "cloudwatch": { "namespace": "AWS/NATGateway" }, + "dimensions": { + "NatGatewayId": "nat-038389b5fc0734aa0" + }, "natgateway": { "metrics": { - "PacketsInFromSource": { - "sum": 421 - }, - "ErrorPortAllocation": { - "sum": 0 + "ActiveConnectionCount": { + "max": 0 }, - "PacketsOutToDestination": { - "sum": 421 + "BytesInFromDestination": { + "sum": 164752 }, - "PacketsOutToSource": { - "sum": 472 + "BytesInFromSource": { + "sum": 42505 }, "BytesOutToDestination": { "sum": 42505 }, - "ConnectionEstablishedCount": { - "sum": 23 + "BytesOutToSource": { + "sum": 164752 }, "ConnectionAttemptCount": { "sum": 23 }, - "PacketsInFromDestination": { - "sum": 472 + "ConnectionEstablishedCount": { + "sum": 23 }, - "BytesInFromDestination": { - "sum": 164752 + "ErrorPortAllocation": { + "sum": 0 + }, + "IdleTimeoutCount": { + "sum": 0 }, "PacketsDropCount": { "sum": 0 }, - "BytesInFromSource": { - "sum": 42505 + "PacketsInFromDestination": { + "sum": 472 }, - "BytesOutToSource": { - "sum": 164752 + "PacketsInFromSource": { + "sum": 421 }, - "IdleTimeoutCount": { - "sum": 0 + "PacketsOutToDestination": { + "sum": 421 }, - "ActiveConnectionCount": { - "max": 0 + "PacketsOutToSource": { + "sum": 472 } } - }, - "dimensions": { - "NatGatewayId": "nat-038389b5fc0734aa0" } }, + "cloud": { + "provider": "aws", + "region": "eu-west-1" + }, + "data_stream": { + "dataset": "aws.natgateway", + "namespace": "default", + "type": "metrics" + }, + "ecs": { + "version": "8.11.0" + }, + "elastic_agent": { + "id": "8940152e-2f20-4ad1-bc96-4db45cb7fc89", + "snapshot": false, + "version": "8.1.0" + }, "event": { - "duration": 612193833, "agent_id_status": "verified", + "dataset": "aws.natgateway", + "duration": 612193833, "ingested": "2022-07-27T22:05:27Z", - "module": "aws", - "dataset": "aws.natgateway" + "module": "aws" + }, + "host": { + "architecture": "aarch64", + "containerized": false, + "hostname": "a3fc2d7bc1c5", + "ip": [ + "172.20.0.7" + ], + "mac": [ + "02-42-AC-14-00-07" + ], + "name": "a3fc2d7bc1c5", + "os": { + "codename": "focal", + "family": "debian", + "kernel": "5.10.104-linuxkit", + "name": "Ubuntu", + "platform": "ubuntu", + "type": "linux", + "version": "20.04.3 LTS (Focal Fossa)" + } + }, + "metricset": { + "name": "cloudwatch", + "period": 180000 + }, + "service": { + "type": "aws" } } \ No newline at end of file diff --git a/packages/aws/data_stream/rds/fields/ecs.yml b/packages/aws/data_stream/rds/fields/ecs.yml index 654c9b2e202..303e169c534 100644 --- a/packages/aws/data_stream/rds/fields/ecs.yml +++ b/packages/aws/data_stream/rds/fields/ecs.yml @@ -1,70 +1,9 @@ -- external: ecs - name: cloud - external: ecs name: cloud.account.id dimension: true -- external: ecs - name: cloud.account.name -- external: ecs - name: cloud.availability_zone -- external: ecs - name: cloud.instance.id -- external: ecs - name: cloud.machine.type -- external: ecs - name: cloud.provider - external: ecs name: cloud.region dimension: true -- external: ecs - name: ecs.version -- external: ecs - name: error -- external: ecs - name: error.message -- external: ecs - name: service.type -- external: ecs - name: host.architecture -- external: ecs - name: host.domain -- external: ecs - name: host.hostname -- external: ecs - name: host.id -- external: ecs - name: host.ip -- external: ecs - name: host.mac -- external: ecs - name: host.name -- external: ecs - name: host.os.family -- external: ecs - name: host.os.kernel -- external: ecs - name: host.os.name -- external: ecs - name: host.os.platform -- external: ecs - name: host.os.version -- external: ecs - name: host.type -- external: ecs - name: cloud.instance.name -- external: ecs - name: cloud.project.id -- external: ecs - name: container.id -- external: ecs - name: container.image.name -- external: ecs - name: container.labels -- external: ecs - name: container.name - name: agent.id external: ecs dimension: true -- external: ecs - name: event.dataset - type: constant_keyword diff --git a/packages/aws/data_stream/rds/sample_event.json b/packages/aws/data_stream/rds/sample_event.json index bb8b53db3f6..6d46924b82b 100644 --- a/packages/aws/data_stream/rds/sample_event.json +++ b/packages/aws/data_stream/rds/sample_event.json @@ -1,49 +1,20 @@ { "@timestamp": "2022-06-03T15:28:00.000Z", - "ecs": { - "version": "8.0.0" - }, "agent": { - "name": "docker-fleet-agent", + "ephemeral_id": "c4161c81-1e2e-4e8b-a0be-15940cc13226", "id": "90bfb41e-b925-420f-973e-9c1115297278", + "name": "docker-fleet-agent", "type": "metricbeat", - "ephemeral_id": "c4161c81-1e2e-4e8b-a0be-15940cc13226", "version": "8.2.0" }, - "elastic_agent": { - "id": "90bfb41e-b925-420f-973e-9c1115297278", - "version": "8.2.0", - "snapshot": false - }, - "cloud": { - "provider": "aws", - "region": "eu-west-1", - "account": { - "name": "elastic-beats", - "id": "123456789" - } - }, - "data_stream": { - "namespace": "default", - "type": "metrics", - "dataset": "aws.rds" - }, - "service": { - "type": "aws" - }, - "metricset": { - "period": 60000, - "name": "cloudwatch" - }, "aws": { + "cloudwatch": { + "namespace": "AWS/RDS" + }, + "dimensions": { + "DatabaseClass": "db.r5.large" + }, "rds": { - "cache_hit_ratio": { - "buffer": 100, - "result_set": 0 - }, - "aurora_volume_left_total": { - "bytes": 70007366615040 - }, "aurora_bin_log_replica_lag": 0, "aurora_replica": { "lag": { @@ -56,127 +27,156 @@ "ms": 19.469999313354492 } }, - "latency": { - "dml": 0.09705000000000001, - "read": 0, - "select": 0.2412933510638298, - "commit": 3.536983333333333, - "insert": 0.09705000000000001, - "update": 0, - "ddl": 0, - "write": 0.0006218917818574514, - "delete": 0 - }, - "swap_usage": { - "bytes": 0 + "aurora_volume_left_total": { + "bytes": 70007366615040 }, - "transactions": { - "blocked": 0, - "active": 0 + "cache_hit_ratio": { + "buffer": 100, + "result_set": 0 }, - "queries": 7.737700770575286, "database_connections": 0, + "deadlocks": 0, + "engine_uptime": { + "sec": 53016926.5 + }, "free_local_storage": { "bytes": 28622428160 }, - "login_failures": 0, - "engine_uptime": { - "sec": 53016926.5 + "freeable_memory": { + "bytes": 4705378304 + }, + "latency": { + "commit": 3.536983333333333, + "ddl": 0, + "delete": 0, + "dml": 0.09705000000000001, + "insert": 0.09705000000000001, + "read": 0, + "select": 0.2412933510638298, + "update": 0, + "write": 0.0006218917818574514 }, + "login_failures": 0, "metrics": { - "Aurora_pq_request_not_chosen_below_min_rows": { + "AbortedClients": { "avg": 0 }, - "RowLockTime": { + "Aurora_pq_request_attempted": { "avg": 0 }, - "RollbackSegmentHistoryListLength": { - "avg": 53 + "Aurora_pq_request_executed": { + "avg": 0 }, - "SumBinaryLogSize": { + "Aurora_pq_request_failed": { "avg": 0 }, - "Aurora_pq_request_not_chosen_pq_high_buffer_pool_pct": { + "Aurora_pq_request_in_progress": { "avg": 0 }, - "StorageNetworkThroughput": { - "avg": 22950.537520958267 + "Aurora_pq_request_not_chosen": { + "avg": 0 }, - "Aurora_pq_request_not_chosen_few_pages_outside_buffer_pool": { + "Aurora_pq_request_not_chosen_below_min_rows": { "avg": 0 }, - "Aurora_pq_request_not_chosen_small_table": { + "Aurora_pq_request_not_chosen_few_pages_outside_buffer_pool": { "avg": 0 }, - "StorageNetworkReceiveThroughput": { - "avg": 7104.272100353031 + "Aurora_pq_request_not_chosen_long_trx": { + "avg": 0 }, - "AbortedClients": { + "Aurora_pq_request_not_chosen_pq_high_buffer_pool_pct": { "avg": 0 }, - "Aurora_pq_request_executed": { + "Aurora_pq_request_not_chosen_small_table": { "avg": 0 }, "Aurora_pq_request_not_chosen_unsupported_access": { "avg": 0 }, - "Aurora_pq_request_not_chosen_long_trx": { + "Aurora_pq_request_throttled": { "avg": 0 }, "ConnectionAttempts": { "avg": 0 }, - "Aurora_pq_request_failed": { - "avg": 0 - }, "NumBinaryLogFiles": { "avg": 0 }, - "Aurora_pq_request_not_chosen": { - "avg": 0 + "RollbackSegmentHistoryListLength": { + "avg": 53 }, - "Aurora_pq_request_in_progress": { + "RowLockTime": { "avg": 0 }, - "Aurora_pq_request_throttled": { - "avg": 0 + "StorageNetworkReceiveThroughput": { + "avg": 7104.272100353031 + }, + "StorageNetworkThroughput": { + "avg": 22950.537520958267 }, "StorageNetworkTransmitThroughput": { "avg": 15846.26542060524 }, - "Aurora_pq_request_attempted": { + "SumBinaryLogSize": { "avg": 0 } }, + "queries": 7.737700770575286, + "swap_usage": { + "bytes": 0 + }, "throughput": { - "dml": 0.2500125006250313, - "select": 2.9051419389878808, - "network_transmit": 0.7020888516985455, - "network_receive": 0.7020888516985455, "commit": 0.2500125006250313, - "insert": 0.2500125006250313, - "update": 0, - "delete": 0, "ddl": 0, - "network": 1.404177703397091 + "delete": 0, + "dml": 0.2500125006250313, + "insert": 0.2500125006250313, + "network": 1.404177703397091, + "network_receive": 0.7020888516985455, + "network_transmit": 0.7020888516985455, + "select": 2.9051419389878808, + "update": 0 }, - "deadlocks": 0, - "freeable_memory": { - "bytes": 4705378304 + "transactions": { + "active": 0, + "blocked": 0 } - }, - "cloudwatch": { - "namespace": "AWS/RDS" - }, - "dimensions": { - "DatabaseClass": "db.r5.large" } }, + "cloud": { + "account": { + "id": "123456789", + "name": "elastic-beats" + }, + "provider": "aws", + "region": "eu-west-1" + }, + "data_stream": { + "dataset": "aws.rds", + "namespace": "default", + "type": "metrics" + }, + "ecs": { + "version": "8.11.0" + }, + "elastic_agent": { + "id": "90bfb41e-b925-420f-973e-9c1115297278", + "snapshot": false, + "version": "8.2.0" + }, "event": { - "duration": 12570787900, "agent_id_status": "verified", + "dataset": "aws.rds", + "duration": 12570787900, "ingested": "2022-06-03T15:28:44Z", - "module": "aws", - "dataset": "aws.rds" + "module": "aws" + }, + "metricset": { + "name": "cloudwatch", + "period": 60000 + }, + "service": { + "type": "aws" } } \ No newline at end of file diff --git a/packages/aws/data_stream/redshift/fields/ecs.yml b/packages/aws/data_stream/redshift/fields/ecs.yml index feb4064addc..f87e83b130a 100644 --- a/packages/aws/data_stream/redshift/fields/ecs.yml +++ b/packages/aws/data_stream/redshift/fields/ecs.yml @@ -1,58 +1,13 @@ -- external: ecs - name: cloud - external: ecs name: cloud.account.id dimension: true -- external: ecs - name: cloud.account.name -- external: ecs - name: cloud.availability_zone -- external: ecs - name: cloud.instance.id -- external: ecs - name: cloud.machine.type -- external: ecs - name: cloud.provider - external: ecs name: cloud.region dimension: true -- external: ecs - name: ecs.version -- external: ecs - name: error -- external: ecs - name: error.message -- external: ecs - name: service.type - external: ecs name: container -- external: ecs - name: container.id -- external: ecs - name: container.name -- external: ecs - name: container.image.name -- external: ecs - name: container.labels - external: ecs name: host -- external: ecs - name: host.architecture -- external: ecs - name: host.domain -- external: ecs - name: host.id -- external: ecs - name: host.ip -- external: ecs - name: host.mac -- external: ecs - name: host.name -- external: ecs - name: host.type - name: agent.id external: ecs dimension: true -- external: ecs - name: event.dataset - type: constant_keyword diff --git a/packages/aws/data_stream/redshift/sample_event.json b/packages/aws/data_stream/redshift/sample_event.json index 64714fe2c8b..256afdd5e0c 100644 --- a/packages/aws/data_stream/redshift/sample_event.json +++ b/packages/aws/data_stream/redshift/sample_event.json @@ -87,7 +87,7 @@ "type": "metrics" }, "ecs": { - "version": "8.0.0" + "version": "8.11.0" }, "elastic_agent": { "id": "d745bccd-73a3-41b4-9fd0-4d9bac14f77b", diff --git a/packages/aws/data_stream/route53_public_logs/_dev/test/pipeline/test-route53.log-expected.json b/packages/aws/data_stream/route53_public_logs/_dev/test/pipeline/test-route53.log-expected.json index 2baf30729dc..aa2a729c826 100644 --- a/packages/aws/data_stream/route53_public_logs/_dev/test/pipeline/test-route53.log-expected.json +++ b/packages/aws/data_stream/route53_public_logs/_dev/test/pipeline/test-route53.log-expected.json @@ -21,7 +21,7 @@ "response_code": "NOERROR" }, "ecs": { - "version": "8.0.0" + "version": "8.11.0" }, "event": { "category": [ @@ -96,7 +96,7 @@ "response_code": "NOERROR" }, "ecs": { - "version": "8.0.0" + "version": "8.11.0" }, "event": { "category": [ @@ -171,7 +171,7 @@ "response_code": "NOERROR" }, "ecs": { - "version": "8.0.0" + "version": "8.11.0" }, "event": { "category": [ @@ -238,7 +238,7 @@ "response_code": "NXDOMAIN" }, "ecs": { - "version": "8.0.0" + "version": "8.11.0" }, "event": { "category": [ @@ -313,7 +313,7 @@ "response_code": "NOERROR" }, "ecs": { - "version": "8.0.0" + "version": "8.11.0" }, "event": { "category": [ @@ -388,7 +388,7 @@ "response_code": "NOERROR" }, "ecs": { - "version": "8.0.0" + "version": "8.11.0" }, "event": { "category": [ diff --git a/packages/aws/data_stream/route53_public_logs/elasticsearch/ingest_pipeline/default.yml b/packages/aws/data_stream/route53_public_logs/elasticsearch/ingest_pipeline/default.yml index 0268d3d5a2e..50ebf4ca5bb 100644 --- a/packages/aws/data_stream/route53_public_logs/elasticsearch/ingest_pipeline/default.yml +++ b/packages/aws/data_stream/route53_public_logs/elasticsearch/ingest_pipeline/default.yml @@ -4,7 +4,7 @@ description: Pipeline for AWS Route53 Logs processors: - set: field: ecs.version - value: '8.0.0' + value: '8.11.0' - set: field: cloud.provider value: aws diff --git a/packages/aws/data_stream/route53_public_logs/fields/beats.yml b/packages/aws/data_stream/route53_public_logs/fields/beats.yml index 3dde4d0b577..0b0328d86ab 100644 --- a/packages/aws/data_stream/route53_public_logs/fields/beats.yml +++ b/packages/aws/data_stream/route53_public_logs/fields/beats.yml @@ -1,9 +1,6 @@ - name: input.type type: keyword description: Type of Filebeat input. -- name: log.file.path - type: keyword - description: Path to the log file. - name: awscloudwatch.log_stream type: keyword description: AWS CloudWatch Log Stream name diff --git a/packages/aws/data_stream/route53_public_logs/fields/ecs.yml b/packages/aws/data_stream/route53_public_logs/fields/ecs.yml deleted file mode 100644 index d3bc48181d3..00000000000 --- a/packages/aws/data_stream/route53_public_logs/fields/ecs.yml +++ /dev/null @@ -1,109 +0,0 @@ -- external: ecs - name: ecs.version -- external: ecs - name: error.message -- external: ecs - name: message -- external: ecs - name: tags -- external: ecs - name: dns.question.name -- external: ecs - name: dns.question.type -- external: ecs - name: dns.question.registered_domain -- external: ecs - name: dns.question.subdomain -- external: ecs - name: dns.question.top_level_domain -- external: ecs - name: dns.response_code -- external: ecs - name: network.iana_number -- external: ecs - name: network.transport -- external: ecs - name: network.protocol -- external: ecs - name: network.type -- external: ecs - name: related.ip -- external: ecs - name: related.hosts -- name: source.geo.continent_name - external: ecs -- name: source.geo.country_iso_code - external: ecs -- external: ecs - name: cloud.account.id -- external: ecs - name: cloud.region -- external: ecs - name: source.address -- external: ecs - name: source.as.number -- external: ecs - name: source.as.organization.name -- external: ecs - name: source.geo.city_name -- external: ecs - name: source.geo.country_name -- description: Longitude and latitude. - level: core - name: source.geo.location - type: geo_point -- external: ecs - name: source.geo.region_iso_code -- external: ecs - name: source.geo.region_name -- external: ecs - name: source.ip -- external: ecs - name: host.architecture -- external: ecs - name: host.domain -- external: ecs - name: host.hostname -- external: ecs - name: host.id -- external: ecs - name: host.ip -- external: ecs - name: host.mac -- external: ecs - name: host.name -- external: ecs - name: host.os.family -- external: ecs - name: host.os.kernel -- external: ecs - name: host.os.name -- external: ecs - name: host.os.platform -- external: ecs - name: host.os.version -- external: ecs - name: host.type -- external: ecs - name: cloud.availability_zone -- external: ecs - name: cloud.instance.id -- external: ecs - name: cloud.instance.name -- external: ecs - name: cloud.machine.type -- external: ecs - name: cloud.provider -- external: ecs - name: cloud.project.id -- external: ecs - name: container.id -- external: ecs - name: container.image.name -- external: ecs - name: container.labels -- external: ecs - name: container.name -- external: ecs - name: event.dataset - type: constant_keyword diff --git a/packages/aws/data_stream/route53_public_logs/sample_event.json b/packages/aws/data_stream/route53_public_logs/sample_event.json index bb7a0ba256b..563088cfe90 100644 --- a/packages/aws/data_stream/route53_public_logs/sample_event.json +++ b/packages/aws/data_stream/route53_public_logs/sample_event.json @@ -1,62 +1,74 @@ { - "awscloudwatch": { - "log_group": "test", - "ingestion_time": "2021-12-06T02:18:20.000Z", - "log_stream": "test" - }, + "@timestamp": "2017-12-13T08:16:05.744Z", "agent": { - "name": "docker-fleet-agent", + "ephemeral_id": "1cf87179-f6b3-44b0-a46f-3aa6bc0f995f", "id": "c00f804f-7a02-441b-88f4-aeb9da6410d9", + "name": "docker-fleet-agent", "type": "filebeat", - "ephemeral_id": "1cf87179-f6b3-44b0-a46f-3aa6bc0f995f", "version": "8.0.0" }, - "elastic_agent": { - "id": "c00f804f-7a02-441b-88f4-aeb9da6410d9", - "version": "8.0.0", - "snapshot": true + "aws": { + "route53": { + "edge_location": "JFK5", + "hosted_zone_id": "Z123412341234" + } + }, + "awscloudwatch": { + "ingestion_time": "2021-12-06T02:18:20.000Z", + "log_group": "test", + "log_stream": "test" + }, + "cloud": { + "provider": "aws", + "region": "us-east-1" + }, + "data_stream": { + "dataset": "aws.route53_public_logs", + "namespace": "default", + "type": "logs" }, "dns": { - "response_code": "NOERROR", "question": { - "registered_domain": "example.com", - "top_level_domain": "com", "name": "txt.example.com", + "registered_domain": "example.com", "subdomain": "txt", + "top_level_domain": "com", "type": "TXT" - } - }, - "source": { - "as": { - "number": 721, - "organization": { - "name": "DoD Network Information Center" - } }, - "address": "55.36.5.7", - "ip": "55.36.5.7" + "response_code": "NOERROR" }, - "tags": [ - "preserve_original_event", - "forwarded", - "aws-route53-logs" - ], - "network": { - "protocol": "dns", - "transport": "udp", - "type": "ipv4", - "iana_number": "17" + "ecs": { + "version": "8.11.0" }, - "cloud": { - "provider": "aws", - "region": "us-east-1" + "elastic_agent": { + "id": "c00f804f-7a02-441b-88f4-aeb9da6410d9", + "snapshot": true, + "version": "8.0.0" + }, + "event": { + "agent_id_status": "verified", + "category": [ + "network" + ], + "dataset": "aws.route53_public_logs", + "id": "36545504503447201576705984279898091551471012413796646912", + "ingested": "2021-12-06T02:37:25Z", + "kind": "event", + "original": "1.0 2017-12-13T08:16:05.744Z Z123412341234 txt.example.com TXT NOERROR UDP JFK5 55.36.5.7 -", + "outcome": "success", + "type": [ + "protocol" + ] }, "input": { "type": "aws-cloudwatch" }, - "@timestamp": "2017-12-13T08:16:05.744Z", - "ecs": { - "version": "8.0.0" + "log.file.path": "test/test", + "network": { + "iana_number": "17", + "protocol": "dns", + "transport": "udp", + "type": "ipv4" }, "related": { "hosts": [ @@ -66,31 +78,19 @@ "55.36.5.7" ] }, - "data_stream": { - "namespace": "default", - "type": "logs", - "dataset": "aws.route53_public_logs" - }, - "log.file.path": "test/test", - "event": { - "agent_id_status": "verified", - "ingested": "2021-12-06T02:37:25Z", - "original": "1.0 2017-12-13T08:16:05.744Z Z123412341234 txt.example.com TXT NOERROR UDP JFK5 55.36.5.7 -", - "kind": "event", - "id": "36545504503447201576705984279898091551471012413796646912", - "category": [ - "network" - ], - "type": [ - "protocol" - ], - "dataset": "aws.route53_public_logs", - "outcome": "success" + "source": { + "address": "55.36.5.7", + "as": { + "number": 721, + "organization": { + "name": "DoD Network Information Center" + } + }, + "ip": "55.36.5.7" }, - "aws": { - "route53": { - "hosted_zone_id": "Z123412341234", - "edge_location": "JFK5" - } - } + "tags": [ + "preserve_original_event", + "forwarded", + "aws-route53-logs" + ] } \ No newline at end of file diff --git a/packages/aws/data_stream/route53_resolver_logs/_dev/test/pipeline/test-route53.log-expected.json b/packages/aws/data_stream/route53_resolver_logs/_dev/test/pipeline/test-route53.log-expected.json index e727e701d42..d8cf2714654 100644 --- a/packages/aws/data_stream/route53_resolver_logs/_dev/test/pipeline/test-route53.log-expected.json +++ b/packages/aws/data_stream/route53_resolver_logs/_dev/test/pipeline/test-route53.log-expected.json @@ -24,7 +24,7 @@ "response_code": "NXDOMAIN" }, "ecs": { - "version": "8.0.0" + "version": "8.11.0" }, "event": { "category": [ @@ -88,7 +88,7 @@ "response_code": "NOERROR" }, "ecs": { - "version": "8.0.0" + "version": "8.11.0" }, "event": { "category": [ @@ -148,7 +148,7 @@ "response_code": "NXDOMAIN" }, "ecs": { - "version": "8.0.0" + "version": "8.11.0" }, "event": { "category": [ @@ -205,7 +205,7 @@ "response_code": "NXDOMAIN" }, "ecs": { - "version": "8.0.0" + "version": "8.11.0" }, "event": { "category": [ @@ -262,7 +262,7 @@ "response_code": "NXDOMAIN" }, "ecs": { - "version": "8.0.0" + "version": "8.11.0" }, "event": { "category": [ @@ -319,7 +319,7 @@ "response_code": "NXDOMAIN" }, "ecs": { - "version": "8.0.0" + "version": "8.11.0" }, "event": { "category": [ @@ -376,7 +376,7 @@ "response_code": "NXDOMAIN" }, "ecs": { - "version": "8.0.0" + "version": "8.11.0" }, "event": { "category": [ @@ -433,7 +433,7 @@ "response_code": "NXDOMAIN" }, "ecs": { - "version": "8.0.0" + "version": "8.11.0" }, "event": { "category": [ @@ -490,7 +490,7 @@ "response_code": "NXDOMAIN" }, "ecs": { - "version": "8.0.0" + "version": "8.11.0" }, "event": { "category": [ @@ -547,7 +547,7 @@ "response_code": "NXDOMAIN" }, "ecs": { - "version": "8.0.0" + "version": "8.11.0" }, "event": { "category": [ @@ -620,7 +620,7 @@ "response_code": "NOERROR" }, "ecs": { - "version": "8.0.0" + "version": "8.11.0" }, "event": { "category": [ @@ -682,7 +682,7 @@ "response_code": "NOERROR" }, "ecs": { - "version": "8.0.0" + "version": "8.11.0" }, "event": { "category": [ @@ -755,7 +755,7 @@ "response_code": "NOERROR" }, "ecs": { - "version": "8.0.0" + "version": "8.11.0" }, "event": { "category": [ @@ -824,7 +824,7 @@ "response_code": "NOERROR" }, "ecs": { - "version": "8.0.0" + "version": "8.11.0" }, "event": { "category": [ @@ -892,7 +892,7 @@ "response_code": "NOERROR" }, "ecs": { - "version": "8.0.0" + "version": "8.11.0" }, "event": { "category": [ @@ -960,7 +960,7 @@ "response_code": "NOERROR" }, "ecs": { - "version": "8.0.0" + "version": "8.11.0" }, "event": { "category": [ @@ -1021,7 +1021,7 @@ "response_code": "NOERROR" }, "ecs": { - "version": "8.0.0" + "version": "8.11.0" }, "event": { "category": [ @@ -1088,7 +1088,7 @@ "response_code": "NOERROR" }, "ecs": { - "version": "8.0.0" + "version": "8.11.0" }, "event": { "category": [ @@ -1157,7 +1157,7 @@ "response_code": "NOERROR" }, "ecs": { - "version": "8.0.0" + "version": "8.11.0" }, "event": { "category": [ @@ -1231,7 +1231,7 @@ "response_code": "NOERROR" }, "ecs": { - "version": "8.0.0" + "version": "8.11.0" }, "event": { "category": [ @@ -1290,7 +1290,7 @@ "response_code": "NXDOMAIN" }, "ecs": { - "version": "8.0.0" + "version": "8.11.0" }, "event": { "category": [ @@ -1348,7 +1348,7 @@ "response_code": "NOERROR" }, "ecs": { - "version": "8.0.0" + "version": "8.11.0" }, "event": { "category": [ @@ -1430,7 +1430,7 @@ "response_code": "NOERROR" }, "ecs": { - "version": "8.0.0" + "version": "8.11.0" }, "event": { "category": [ @@ -1494,7 +1494,7 @@ "response_code": "NOERROR" }, "ecs": { - "version": "8.0.0" + "version": "8.11.0" }, "event": { "category": [ @@ -1576,7 +1576,7 @@ "response_code": "NOERROR" }, "ecs": { - "version": "8.0.0" + "version": "8.11.0" }, "event": { "category": [ @@ -1662,7 +1662,7 @@ "response_code": "NOERROR" }, "ecs": { - "version": "8.0.0" + "version": "8.11.0" }, "event": { "category": [ @@ -1748,7 +1748,7 @@ "response_code": "NOERROR" }, "ecs": { - "version": "8.0.0" + "version": "8.11.0" }, "event": { "category": [ @@ -1809,7 +1809,7 @@ "response_code": "NXDOMAIN" }, "ecs": { - "version": "8.0.0" + "version": "8.11.0" }, "event": { "category": [ @@ -1874,7 +1874,7 @@ "response_code": "NOERROR" }, "ecs": { - "version": "8.0.0" + "version": "8.11.0" }, "event": { "category": [ @@ -1939,7 +1939,7 @@ "response_code": "NOERROR" }, "ecs": { - "version": "8.0.0" + "version": "8.11.0" }, "event": { "category": [ @@ -2007,7 +2007,7 @@ "response_code": "NOERROR" }, "ecs": { - "version": "8.0.0" + "version": "8.11.0" }, "event": { "category": [ @@ -2085,7 +2085,7 @@ "response_code": "NOERROR" }, "ecs": { - "version": "8.0.0" + "version": "8.11.0" }, "event": { "category": [ @@ -2166,7 +2166,7 @@ "response_code": "NOERROR" }, "ecs": { - "version": "8.0.0" + "version": "8.11.0" }, "event": { "category": [ @@ -2247,7 +2247,7 @@ "response_code": "NOERROR" }, "ecs": { - "version": "8.0.0" + "version": "8.11.0" }, "event": { "category": [ diff --git a/packages/aws/data_stream/route53_resolver_logs/elasticsearch/ingest_pipeline/default.yml b/packages/aws/data_stream/route53_resolver_logs/elasticsearch/ingest_pipeline/default.yml index 050b5aec0f2..d92ab2516b1 100644 --- a/packages/aws/data_stream/route53_resolver_logs/elasticsearch/ingest_pipeline/default.yml +++ b/packages/aws/data_stream/route53_resolver_logs/elasticsearch/ingest_pipeline/default.yml @@ -4,7 +4,7 @@ description: Pipeline for AWS Route53 Resolver Logs processors: - set: field: ecs.version - value: '8.0.0' + value: '8.11.0' - rename: field: message target_field: event.original diff --git a/packages/aws/data_stream/route53_resolver_logs/fields/beats.yml b/packages/aws/data_stream/route53_resolver_logs/fields/beats.yml index 3dde4d0b577..0b0328d86ab 100644 --- a/packages/aws/data_stream/route53_resolver_logs/fields/beats.yml +++ b/packages/aws/data_stream/route53_resolver_logs/fields/beats.yml @@ -1,9 +1,6 @@ - name: input.type type: keyword description: Type of Filebeat input. -- name: log.file.path - type: keyword - description: Path to the log file. - name: awscloudwatch.log_stream type: keyword description: AWS CloudWatch Log Stream name diff --git a/packages/aws/data_stream/route53_resolver_logs/fields/ecs.yml b/packages/aws/data_stream/route53_resolver_logs/fields/ecs.yml deleted file mode 100644 index e3345d966c2..00000000000 --- a/packages/aws/data_stream/route53_resolver_logs/fields/ecs.yml +++ /dev/null @@ -1,126 +0,0 @@ -- external: ecs - name: ecs.version -- external: ecs - name: error.message -- external: ecs - name: message -- external: ecs - name: tags -- external: ecs - name: dns.question.name -- external: ecs - name: dns.question.type -- external: ecs - name: dns.question.registered_domain -- external: ecs - name: dns.question.subdomain -- external: ecs - name: dns.question.top_level_domain -- external: ecs - name: dns.response_code -- external: ecs - name: dns.answers - type: group -- external: ecs - name: dns.answers.class -- external: ecs - name: dns.answers.data -- external: ecs - name: dns.answers.name -- external: ecs - name: dns.answers.ttl -- external: ecs - name: dns.answers.type -- external: ecs - name: dns.question.class -- external: ecs - name: network.iana_number -- external: ecs - name: network.transport -- external: ecs - name: network.protocol -- external: ecs - name: network.type -- external: ecs - name: related.ip -- external: ecs - name: related.hosts -- external: ecs - name: source.port -- external: ecs - name: source.ip -- external: ecs - name: source.as.organization.name -- name: source.geo.country_iso_code - external: ecs -- name: source.geo.region_iso_code - external: ecs -- name: source.geo.region_name - external: ecs -- external: ecs - name: cloud.account.id -- external: ecs - name: cloud.region -- external: ecs - name: source.address -- external: ecs - name: source.as.number -- external: ecs - name: source.geo.city_name -- external: ecs - name: source.geo.continent_name -- external: ecs - name: source.geo.country_name -- description: Longitude and latitude. - level: core - name: source.geo.location - type: geo_point -- external: ecs - name: host.architecture -- external: ecs - name: host.domain -- external: ecs - name: host.hostname -- external: ecs - name: host.id -- external: ecs - name: host.ip -- external: ecs - name: host.mac -- external: ecs - name: host.name -- external: ecs - name: host.os.family -- external: ecs - name: host.os.kernel -- external: ecs - name: host.os.name -- external: ecs - name: host.os.platform -- external: ecs - name: host.os.version -- external: ecs - name: host.type -- external: ecs - name: cloud.availability_zone -- external: ecs - name: cloud.instance.id -- external: ecs - name: cloud.instance.name -- external: ecs - name: cloud.machine.type -- external: ecs - name: cloud.provider -- external: ecs - name: cloud.project.id -- external: ecs - name: container.id -- external: ecs - name: container.image.name -- external: ecs - name: container.labels -- external: ecs - name: container.name -- external: ecs - name: event.dataset - type: constant_keyword diff --git a/packages/aws/data_stream/route53_resolver_logs/sample_event.json b/packages/aws/data_stream/route53_resolver_logs/sample_event.json index 50acf5eb98f..2f37578e71b 100644 --- a/packages/aws/data_stream/route53_resolver_logs/sample_event.json +++ b/packages/aws/data_stream/route53_resolver_logs/sample_event.json @@ -62,7 +62,7 @@ "response_code": "NOERROR" }, "ecs": { - "version": "8.0.0" + "version": "8.11.0" }, "elastic_agent": { "id": "acba78ef-1401-4689-977c-d8c2e5d6a8fa", diff --git a/packages/aws/data_stream/s3_daily_storage/fields/ecs.yml b/packages/aws/data_stream/s3_daily_storage/fields/ecs.yml index 654c9b2e202..303e169c534 100644 --- a/packages/aws/data_stream/s3_daily_storage/fields/ecs.yml +++ b/packages/aws/data_stream/s3_daily_storage/fields/ecs.yml @@ -1,70 +1,9 @@ -- external: ecs - name: cloud - external: ecs name: cloud.account.id dimension: true -- external: ecs - name: cloud.account.name -- external: ecs - name: cloud.availability_zone -- external: ecs - name: cloud.instance.id -- external: ecs - name: cloud.machine.type -- external: ecs - name: cloud.provider - external: ecs name: cloud.region dimension: true -- external: ecs - name: ecs.version -- external: ecs - name: error -- external: ecs - name: error.message -- external: ecs - name: service.type -- external: ecs - name: host.architecture -- external: ecs - name: host.domain -- external: ecs - name: host.hostname -- external: ecs - name: host.id -- external: ecs - name: host.ip -- external: ecs - name: host.mac -- external: ecs - name: host.name -- external: ecs - name: host.os.family -- external: ecs - name: host.os.kernel -- external: ecs - name: host.os.name -- external: ecs - name: host.os.platform -- external: ecs - name: host.os.version -- external: ecs - name: host.type -- external: ecs - name: cloud.instance.name -- external: ecs - name: cloud.project.id -- external: ecs - name: container.id -- external: ecs - name: container.image.name -- external: ecs - name: container.labels -- external: ecs - name: container.name - name: agent.id external: ecs dimension: true -- external: ecs - name: event.dataset - type: constant_keyword diff --git a/packages/aws/data_stream/s3_daily_storage/sample_event.json b/packages/aws/data_stream/s3_daily_storage/sample_event.json index d603330d7d3..050305bd92e 100644 --- a/packages/aws/data_stream/s3_daily_storage/sample_event.json +++ b/packages/aws/data_stream/s3_daily_storage/sample_event.json @@ -1,65 +1,65 @@ { "@timestamp": "2022-07-25T19:02:00.000Z", "agent": { - "name": "docker-fleet-agent", - "id": "2d4b09d0-cdb6-445e-ac3f-6415f87b9864", "ephemeral_id": "9ef87976-bec2-4a74-9876-4e76d42035bb", + "id": "2d4b09d0-cdb6-445e-ac3f-6415f87b9864", + "name": "docker-fleet-agent", "type": "metricbeat", "version": "8.3.2" }, - "elastic_agent": { - "id": "2d4b09d0-cdb6-445e-ac3f-6415f87b9864", - "version": "8.3.2", - "snapshot": false - }, - "cloud": { - "provider": "aws", - "region": "eu-central-1", - "account": { - "name": "elastic-beats", - "id": "428152502467" - } - }, - "ecs": { - "version": "8.0.0" - }, - "service": { - "type": "aws" - }, - "data_stream": { - "namespace": "default", - "type": "metrics", - "dataset": "aws.s3_daily_storage" - }, - "metricset": { - "period": 86400000, - "name": "cloudwatch" - }, "aws": { + "cloudwatch": { + "namespace": "AWS/S3" + }, + "dimensions": { + "StorageType": "StandardStorage" + }, "s3": { "bucket": { "name": "filebeat-aws-elb-test" } }, - "cloudwatch": { - "namespace": "AWS/S3" - }, "s3_daily_storage": { "bucket": { "size": { "bytes": 469407687 } } - }, - "dimensions": { - "StorageType": "StandardStorage" } }, + "cloud": { + "account": { + "id": "428152502467", + "name": "elastic-beats" + }, + "provider": "aws", + "region": "eu-central-1" + }, + "data_stream": { + "dataset": "aws.s3_daily_storage", + "namespace": "default", + "type": "metrics" + }, + "ecs": { + "version": "8.11.0" + }, + "elastic_agent": { + "id": "2d4b09d0-cdb6-445e-ac3f-6415f87b9864", + "snapshot": false, + "version": "8.3.2" + }, "event": { - "duration": 9553539400, "agent_id_status": "verified", + "dataset": "aws.s3_daily_storage", + "duration": 9553539400, "ingested": "2022-07-26T19:02:17Z", - "module": "aws", - "dataset": "aws.s3_daily_storage" + "module": "aws" + }, + "metricset": { + "name": "cloudwatch", + "period": 86400000 + }, + "service": { + "type": "aws" } } \ No newline at end of file diff --git a/packages/aws/data_stream/s3_request/fields/ecs.yml b/packages/aws/data_stream/s3_request/fields/ecs.yml index 654c9b2e202..303e169c534 100644 --- a/packages/aws/data_stream/s3_request/fields/ecs.yml +++ b/packages/aws/data_stream/s3_request/fields/ecs.yml @@ -1,70 +1,9 @@ -- external: ecs - name: cloud - external: ecs name: cloud.account.id dimension: true -- external: ecs - name: cloud.account.name -- external: ecs - name: cloud.availability_zone -- external: ecs - name: cloud.instance.id -- external: ecs - name: cloud.machine.type -- external: ecs - name: cloud.provider - external: ecs name: cloud.region dimension: true -- external: ecs - name: ecs.version -- external: ecs - name: error -- external: ecs - name: error.message -- external: ecs - name: service.type -- external: ecs - name: host.architecture -- external: ecs - name: host.domain -- external: ecs - name: host.hostname -- external: ecs - name: host.id -- external: ecs - name: host.ip -- external: ecs - name: host.mac -- external: ecs - name: host.name -- external: ecs - name: host.os.family -- external: ecs - name: host.os.kernel -- external: ecs - name: host.os.name -- external: ecs - name: host.os.platform -- external: ecs - name: host.os.version -- external: ecs - name: host.type -- external: ecs - name: cloud.instance.name -- external: ecs - name: cloud.project.id -- external: ecs - name: container.id -- external: ecs - name: container.image.name -- external: ecs - name: container.labels -- external: ecs - name: container.name - name: agent.id external: ecs dimension: true -- external: ecs - name: event.dataset - type: constant_keyword diff --git a/packages/aws/data_stream/s3_request/sample_event.json b/packages/aws/data_stream/s3_request/sample_event.json index 996d7560eaa..47526b90fc5 100644 --- a/packages/aws/data_stream/s3_request/sample_event.json +++ b/packages/aws/data_stream/s3_request/sample_event.json @@ -1,50 +1,32 @@ { "@timestamp": "2022-07-26T20:10:00.000Z", "agent": { - "name": "docker-fleet-agent", + "ephemeral_id": "287cb701-3031-45be-a8c1-4c4860603d9b", "id": "2d4b09d0-cdb6-445e-ac3f-6415f87b9864", + "name": "docker-fleet-agent", "type": "metricbeat", - "ephemeral_id": "287cb701-3031-45be-a8c1-4c4860603d9b", "version": "8.3.2" }, - "elastic_agent": { - "id": "2d4b09d0-cdb6-445e-ac3f-6415f87b9864", - "version": "8.3.2", - "snapshot": false - }, - "cloud": { - "provider": "aws", - "region": "us-east-1", - "account": { - "name": "elastic-beats", - "id": "428152502467" - } - }, - "ecs": { - "version": "8.0.0" - }, - "service": { - "type": "aws" - }, - "data_stream": { - "namespace": "default", - "type": "metrics", - "dataset": "aws.s3_request" - }, - "metricset": { - "period": 60000, - "name": "cloudwatch" - }, "aws": { + "cloudwatch": { + "namespace": "AWS/S3" + }, + "dimensions": { + "FilterId": "AllItems" + }, "s3": { "bucket": { "name": "vpc-flow-logs-ks" } }, - "cloudwatch": { - "namespace": "AWS/S3" - }, "s3_request": { + "downloaded": { + "bytes": 400 + }, + "errors": { + "4xx": 1, + "5xx": 0 + }, "latency": { "total_request": { "ms": 32 @@ -53,24 +35,42 @@ "requests": { "head": 1, "total": 1 - }, - "downloaded": { - "bytes": 400 - }, - "errors": { - "4xx": 1, - "5xx": 0 } - }, - "dimensions": { - "FilterId": "AllItems" } }, + "cloud": { + "account": { + "id": "428152502467", + "name": "elastic-beats" + }, + "provider": "aws", + "region": "us-east-1" + }, + "data_stream": { + "dataset": "aws.s3_request", + "namespace": "default", + "type": "metrics" + }, + "ecs": { + "version": "8.11.0" + }, + "elastic_agent": { + "id": "2d4b09d0-cdb6-445e-ac3f-6415f87b9864", + "snapshot": false, + "version": "8.3.2" + }, "event": { - "duration": 9552028500, "agent_id_status": "verified", + "dataset": "aws.s3_request", + "duration": 9552028500, "ingested": "2022-07-26T20:16:31Z", - "module": "aws", - "dataset": "aws.s3_request" + "module": "aws" + }, + "metricset": { + "name": "cloudwatch", + "period": 60000 + }, + "service": { + "type": "aws" } } \ No newline at end of file diff --git a/packages/aws/data_stream/s3_storage_lens/fields/ecs.yml b/packages/aws/data_stream/s3_storage_lens/fields/ecs.yml index 654c9b2e202..303e169c534 100644 --- a/packages/aws/data_stream/s3_storage_lens/fields/ecs.yml +++ b/packages/aws/data_stream/s3_storage_lens/fields/ecs.yml @@ -1,70 +1,9 @@ -- external: ecs - name: cloud - external: ecs name: cloud.account.id dimension: true -- external: ecs - name: cloud.account.name -- external: ecs - name: cloud.availability_zone -- external: ecs - name: cloud.instance.id -- external: ecs - name: cloud.machine.type -- external: ecs - name: cloud.provider - external: ecs name: cloud.region dimension: true -- external: ecs - name: ecs.version -- external: ecs - name: error -- external: ecs - name: error.message -- external: ecs - name: service.type -- external: ecs - name: host.architecture -- external: ecs - name: host.domain -- external: ecs - name: host.hostname -- external: ecs - name: host.id -- external: ecs - name: host.ip -- external: ecs - name: host.mac -- external: ecs - name: host.name -- external: ecs - name: host.os.family -- external: ecs - name: host.os.kernel -- external: ecs - name: host.os.name -- external: ecs - name: host.os.platform -- external: ecs - name: host.os.version -- external: ecs - name: host.type -- external: ecs - name: cloud.instance.name -- external: ecs - name: cloud.project.id -- external: ecs - name: container.id -- external: ecs - name: container.image.name -- external: ecs - name: container.labels -- external: ecs - name: container.name - name: agent.id external: ecs dimension: true -- external: ecs - name: event.dataset - type: constant_keyword diff --git a/packages/aws/data_stream/s3_storage_lens/sample_event.json b/packages/aws/data_stream/s3_storage_lens/sample_event.json index 3b9757e8d17..be7a65ac5a8 100644 --- a/packages/aws/data_stream/s3_storage_lens/sample_event.json +++ b/packages/aws/data_stream/s3_storage_lens/sample_event.json @@ -1,138 +1,138 @@ { "@timestamp": "2021-11-07T20:38:00.000Z", - "ecs": { - "version": "8.0.0" - }, - "data_stream": { - "namespace": "default", - "type": "metrics", - "dataset": "aws.s3_storage_lens" - }, - "service": { - "type": "aws" - }, - "cloud": { - "provider": "aws", - "region": "us-east-1", - "account": { - "name": "elastic-beats", - "id": "428152502467" - } - }, - "metricset": { - "period": 86400000, - "name": "cloudwatch" - }, - "event": { - "duration": 22973251900, - "agent_id_status": "verified", - "ingested": "2021-11-08T20:38:37Z", - "module": "aws", - "dataset": "aws.s3_storage_lens" - }, "aws": { + "cloudwatch": { + "namespace": "AWS/S3/Storage-Lens" + }, + "dimensions": { + "aws_account_number": "428152502467", + "aws_region": "eu-central-1", + "bucket_name": "filebeat-aws-elb-test", + "configuration_id": "default-account-dashboard", + "metrics_version": "1.0", + "record_type": "BUCKET", + "storage_class": "STANDARD" + }, "s3_storage_lens": { "metrics": { - "NonCurrentVersionStorageBytes": { + "4xxErrors": { "avg": 0 }, - "DeleteMarkerObjectCount": { + "5xxErrors": { "avg": 0 }, - "GetRequests": { - "avg": 0 + "AllRequests": { + "avg": 145 }, - "SelectReturnedBytes": { + "BytesDownloaded": { "avg": 0 }, - "ObjectCount": { + "BytesUploaded": { + "avg": 82537 + }, + "CurrentVersionObjectCount": { "avg": 164195 }, - "HeadRequests": { - "avg": 0 + "CurrentVersionStorageBytes": { + "avg": 154238334 }, - "ListRequests": { + "DeleteMarkerObjectCount": { "avg": 0 }, "DeleteRequests": { "avg": 0 }, - "SelectRequests": { - "avg": 0 + "EncryptedObjectCount": { + "avg": 164191 }, - "5xxErrors": { + "EncryptedStorageBytes": { + "avg": 154237917 + }, + "GetRequests": { "avg": 0 }, - "BytesDownloaded": { + "HeadRequests": { "avg": 0 }, - "BytesUploaded": { - "avg": 82537 + "IncompleteMultipartUploadObjectCount": { + "avg": 0 }, - "CurrentVersionStorageBytes": { - "avg": 154238334 + "IncompleteMultipartUploadStorageBytes": { + "avg": 0 }, - "StorageBytes": { - "avg": 154238334 + "ListRequests": { + "avg": 0 }, - "ObjectLockEnabledStorageBytes": { + "NonCurrentVersionObjectCount": { "avg": 0 }, - "4xxErrors": { + "NonCurrentVersionStorageBytes": { "avg": 0 }, - "PutRequests": { - "avg": 145 + "ObjectCount": { + "avg": 164195 }, "ObjectLockEnabledObjectCount": { "avg": 0 }, - "EncryptedObjectCount": { - "avg": 164191 - }, - "CurrentVersionObjectCount": { - "avg": 164195 - }, - "IncompleteMultipartUploadObjectCount": { + "ObjectLockEnabledStorageBytes": { "avg": 0 }, - "ReplicatedObjectCount": { + "PostRequests": { "avg": 0 }, - "AllRequests": { + "PutRequests": { "avg": 145 }, - "PostRequests": { + "ReplicatedObjectCount": { "avg": 0 }, - "IncompleteMultipartUploadStorageBytes": { + "ReplicatedStorageBytes": { "avg": 0 }, - "NonCurrentVersionObjectCount": { + "SelectRequests": { "avg": 0 }, - "ReplicatedStorageBytes": { + "SelectReturnedBytes": { "avg": 0 }, - "EncryptedStorageBytes": { - "avg": 154237917 - }, "SelectScannedBytes": { "avg": 0 + }, + "StorageBytes": { + "avg": 154238334 } } - }, - "cloudwatch": { - "namespace": "AWS/S3/Storage-Lens" - }, - "dimensions": { - "metrics_version": "1.0", - "storage_class": "STANDARD", - "aws_region": "eu-central-1", - "bucket_name": "filebeat-aws-elb-test", - "aws_account_number": "428152502467", - "configuration_id": "default-account-dashboard", - "record_type": "BUCKET" } + }, + "cloud": { + "account": { + "id": "428152502467", + "name": "elastic-beats" + }, + "provider": "aws", + "region": "us-east-1" + }, + "data_stream": { + "dataset": "aws.s3_storage_lens", + "namespace": "default", + "type": "metrics" + }, + "ecs": { + "version": "8.11.0" + }, + "event": { + "agent_id_status": "verified", + "dataset": "aws.s3_storage_lens", + "duration": 22973251900, + "ingested": "2021-11-08T20:38:37Z", + "module": "aws" + }, + "metricset": { + "name": "cloudwatch", + "period": 86400000 + }, + "service": { + "type": "aws" } } \ No newline at end of file diff --git a/packages/aws/data_stream/s3access/_dev/test/pipeline/test-s3-server-access.log-expected.json b/packages/aws/data_stream/s3access/_dev/test/pipeline/test-s3-server-access.log-expected.json index 34ddf79d268..ccba9840760 100644 --- a/packages/aws/data_stream/s3access/_dev/test/pipeline/test-s3-server-access.log-expected.json +++ b/packages/aws/data_stream/s3access/_dev/test/pipeline/test-s3-server-access.log-expected.json @@ -47,11 +47,13 @@ "region": "ap-southeast-1" }, "ecs": { - "version": "8.0.0" + "version": "8.11.0" }, "event": { "action": "REST.GET.LOCATION", - "category": "web", + "category": [ + "web" + ], "duration": 17000000, "id": "44EE8651683CB4DA", "kind": "event", @@ -167,11 +169,13 @@ "region": "ap-southeast-1" }, "ecs": { - "version": "8.0.0" + "version": "8.11.0" }, "event": { "action": "REST.GET.LOCATION", - "category": "web", + "category": [ + "web" + ], "duration": 3000000, "id": "E26222010BCC32B6", "kind": "event", @@ -288,11 +292,13 @@ "region": "ap-southeast-1" }, "ecs": { - "version": "8.0.0" + "version": "8.11.0" }, "event": { "action": "REST.GET.BUCKET", - "category": "web", + "category": [ + "web" + ], "duration": 2000000, "id": "4DD6D17D1C5C401C", "kind": "event", @@ -408,11 +414,13 @@ "region": "ap-southeast-1" }, "ecs": { - "version": "8.0.0" + "version": "8.11.0" }, "event": { "action": "REST.GET.LOCATION", - "category": "web", + "category": [ + "web" + ], "duration": 4000000, "id": "706992E2F3CC3C3D", "kind": "event", @@ -526,11 +534,13 @@ "region": "eu-central-1" }, "ecs": { - "version": "8.0.0" + "version": "8.11.0" }, "event": { "action": "BATCH.DELETE.OBJECT", - "category": "web", + "category": [ + "web" + ], "id": "8CD7A4A71E2E5C9E", "kind": "event", "original": "36c1f05b76016b78528454e6e0c60e2b7ff7aa20c0a5e4c748276e5b0a2debd2 jsoriano-s3-test [10/Sep/2019:15:11:07 +0000] 89.160.20.156 arn:aws:iam::123456:user/test@elastic.co 8CD7A4A71E2E5C9E BATCH.DELETE.OBJECT jolokia-war-1.5.0.war - 204 - - 344017 - - - - - IeDW5I3wefFxU8iHOcAzi5qr+O+1bdRlcQ0AO2WGjFh7JwYM6qCoKq+1TrUshrXMlBxPFtg97Vk= SigV4 ECDHE-RSA-AES128-SHA AuthHeader s3.eu-central-1.amazonaws.com TLSv1.2", @@ -618,11 +628,13 @@ "region": "ap-southeast-1" }, "ecs": { - "version": "8.0.0" + "version": "8.11.0" }, "event": { "action": "BATCH.DELETE.OBJECT", - "category": "web", + "category": [ + "web" + ], "id": "6CE38F1312D32BDD", "kind": "event", "original": "36c1f05b76016b78528454e6e0c60e2b7ff7aa20c0a5e4c748276e5b0a2debd2 test-s3-ks [19/Sep/2019:17:06:39 +0000] 89.160.20.156 arn:aws:iam::123456:user/test@elastic.co 6CE38F1312D32BDD BATCH.DELETE.OBJECT Screen+Shot+2019-09-09+at+9.08.44+AM.png - 204 - - 57138 - - - - - LwRa4w6DbuU48GKQiH3jDbjfTyLCbwasFBsdttugRQ+9lH4jK8lT91+HhGZKMYI3sPyKuQ9LvU0= SigV4 ECDHE-RSA-AES128-SHA AuthHeader s3.ap-southeast-1.amazonaws.com TLSv1.2", @@ -698,11 +710,13 @@ "region": "us-gov-west-1" }, "ecs": { - "version": "8.0.0" + "version": "8.11.0" }, "event": { "action": "REST.PUT.OBJECT", - "category": "web", + "category": [ + "web" + ], "duration": 103000000, "id": "MVGXZXEVN3IG9S24", "kind": "event", diff --git a/packages/aws/data_stream/s3access/elasticsearch/ingest_pipeline/default.yml b/packages/aws/data_stream/s3access/elasticsearch/ingest_pipeline/default.yml index 5dfba312d61..3aab741b663 100644 --- a/packages/aws/data_stream/s3access/elasticsearch/ingest_pipeline/default.yml +++ b/packages/aws/data_stream/s3access/elasticsearch/ingest_pipeline/default.yml @@ -4,13 +4,13 @@ description: "Pipeline for s3 server access logs" processors: - set: field: ecs.version - value: '8.0.0' + value: '8.11.0' - set: field: event.category - value: web + value: ["web"] - append: field: event.type - value: access + value: ["access"] - rename: field: message target_field: event.original diff --git a/packages/aws/data_stream/s3access/fields/ecs.yml b/packages/aws/data_stream/s3access/fields/ecs.yml deleted file mode 100644 index 3f786c0f385..00000000000 --- a/packages/aws/data_stream/s3access/fields/ecs.yml +++ /dev/null @@ -1,143 +0,0 @@ -- external: ecs - name: client.address -- external: ecs - name: client.ip -- external: ecs - name: client.user.id -- external: ecs - name: ecs.version -- external: ecs - name: error.message -- external: ecs - name: event.action -- external: ecs - name: event.code -- external: ecs - name: event.duration -- external: ecs - name: event.id -- external: ecs - name: event.kind -- external: ecs - name: event.outcome -- external: ecs - name: client.geo.city_name -- external: ecs - name: client.geo.continent_name -- external: ecs - name: client.geo.country_iso_code -- external: ecs - name: client.geo.country_name -- description: Longitude and latitude. - level: core - name: client.geo.location - type: geo_point -- external: ecs - name: client.geo.region_iso_code -- external: ecs - name: client.geo.region_name -- external: ecs - name: http.request.method -- external: ecs - name: http.request.referrer -- external: ecs - name: http.response.body.bytes -- external: ecs - name: http.response.status_code -- external: ecs - name: http.version -- external: ecs - name: related.ip -- external: ecs - name: related.user -- external: ecs - name: tags -- external: ecs - name: tls.cipher -- external: ecs - name: tls.version -- external: ecs - name: tls.version_protocol -- external: ecs - name: url.domain -- external: ecs - name: url.extension -- external: ecs - name: url.original -- external: ecs - name: url.path -- external: ecs - name: url.query -- external: ecs - name: url.scheme -- external: ecs - name: user_agent.device.name -- external: ecs - name: user_agent.name -- external: ecs - name: user_agent.original -- external: ecs - name: user_agent.os.full -- external: ecs - name: user_agent.os.name -- external: ecs - name: user_agent.os.version -- external: ecs - name: user_agent.version -- external: ecs - name: cloud.account.id -- external: ecs - name: cloud.account.name -- external: ecs - name: cloud.availability_zone -- external: ecs - name: cloud.instance.id -- external: ecs - name: cloud.instance.name -- external: ecs - name: cloud.machine.type -- external: ecs - name: cloud.provider -- external: ecs - name: cloud.project.id -- external: ecs - name: cloud.region -- external: ecs - name: container.id -- external: ecs - name: container.image.name -- external: ecs - name: container.labels -- external: ecs - name: container.name -- external: ecs - name: host.architecture -- external: ecs - name: host.domain -- external: ecs - name: host.hostname -- external: ecs - name: host.id -- external: ecs - name: host.ip -- external: ecs - name: host.mac -- external: ecs - name: host.name -- external: ecs - name: host.os.family -- external: ecs - name: host.os.kernel -- external: ecs - name: host.os.name -- external: ecs - name: host.os.platform -- external: ecs - name: host.os.version -- external: ecs - name: host.type -- external: ecs - name: log.file.path -- external: ecs - name: event.dataset - type: constant_keyword diff --git a/packages/aws/data_stream/s3access/sample_event.json b/packages/aws/data_stream/s3access/sample_event.json index aec1efd14b3..3311423cd6e 100644 --- a/packages/aws/data_stream/s3access/sample_event.json +++ b/packages/aws/data_stream/s3access/sample_event.json @@ -66,7 +66,7 @@ "type": "logs" }, "ecs": { - "version": "8.0.0" + "version": "8.11.0" }, "elastic_agent": { "id": "acba78ef-1401-4689-977c-d8c2e5d6a8fa", @@ -76,7 +76,9 @@ "event": { "action": "REST.GET.LOCATION", "agent_id_status": "verified", - "category": "web", + "category": [ + "web" + ], "dataset": "aws.s3access", "duration": 17000000, "id": "44EE8651683CB4DA", diff --git a/packages/aws/data_stream/securityhub_findings/_dev/test/pipeline/test-securityhub-findings.log-expected.json b/packages/aws/data_stream/securityhub_findings/_dev/test/pipeline/test-securityhub-findings.log-expected.json index 1b61cd59157..43af1b05bc3 100644 --- a/packages/aws/data_stream/securityhub_findings/_dev/test/pipeline/test-securityhub-findings.log-expected.json +++ b/packages/aws/data_stream/securityhub_findings/_dev/test/pipeline/test-securityhub-findings.log-expected.json @@ -368,7 +368,7 @@ "port": 80 }, "ecs": { - "version": "8.2.0" + "version": "8.11.0" }, "event": { "action": "port_probe", @@ -811,7 +811,7 @@ "port": 80 }, "ecs": { - "version": "8.2.0" + "version": "8.11.0" }, "event": { "action": "port_probe", @@ -1001,7 +1001,7 @@ } }, "ecs": { - "version": "8.2.0" + "version": "8.11.0" }, "event": { "id": "xxxx", @@ -1115,7 +1115,7 @@ } }, "ecs": { - "version": "8.2.0" + "version": "8.11.0" }, "event": { "id": "xxx", diff --git a/packages/aws/data_stream/securityhub_findings/elasticsearch/ingest_pipeline/default.yml b/packages/aws/data_stream/securityhub_findings/elasticsearch/ingest_pipeline/default.yml index 0b8bd462594..2dafd11a833 100644 --- a/packages/aws/data_stream/securityhub_findings/elasticsearch/ingest_pipeline/default.yml +++ b/packages/aws/data_stream/securityhub_findings/elasticsearch/ingest_pipeline/default.yml @@ -3,7 +3,7 @@ description: Pipeline for processing AWS Security Hub Findings logs. processors: - set: field: ecs.version - value: '8.2.0' + value: '8.11.0' - set: field: event.kind value: event diff --git a/packages/aws/data_stream/securityhub_findings/fields/ecs.yml b/packages/aws/data_stream/securityhub_findings/fields/ecs.yml deleted file mode 100644 index 88ece74b907..00000000000 --- a/packages/aws/data_stream/securityhub_findings/fields/ecs.yml +++ /dev/null @@ -1,153 +0,0 @@ -- external: ecs - name: destination.domain -- external: ecs - name: destination.ip -- external: ecs - name: destination.port -- external: ecs - name: ecs.version -- external: ecs - name: event.action -- external: ecs - name: event.category -- external: ecs - name: event.created -- external: ecs - name: event.id -- external: ecs - name: event.kind -- external: ecs - name: event.original -- external: ecs - name: event.type -- external: ecs - name: network.direction -- external: ecs - name: network.protocol -- external: ecs - name: organization.name -- external: ecs - name: process.end -- external: ecs - name: process.executable -- external: ecs - name: process.name -- external: ecs - name: process.parent.pid -- external: ecs - name: process.pid -- external: ecs - name: process.start -- external: ecs - name: related.ip -- external: ecs - name: source.domain -- external: ecs - name: source.ip -- external: ecs - name: source.mac -- external: ecs - name: source.port -- external: ecs - name: tags -- external: ecs - name: threat.indicator.last_seen -- external: ecs - name: threat.indicator.type -- external: ecs - name: threat.enrichments -- external: ecs - name: threat.enrichments.indicator.file.hash.md5 -- external: ecs - name: threat.enrichments.indicator.file.hash.sha1 -- external: ecs - name: threat.enrichments.indicator.file.hash.sha256 -- external: ecs - name: threat.enrichments.indicator.file.hash.sha512 -- external: ecs - name: url.domain -- external: ecs - name: url.extension -- external: ecs - name: url.fragment -- external: ecs - name: url.full -- external: ecs - name: url.original -- external: ecs - name: url.password -- external: ecs - name: url.path -- external: ecs - name: url.port -- external: ecs - name: url.query -- external: ecs - name: url.scheme -- external: ecs - name: url.username -- external: ecs - name: vulnerability.id -- external: ecs - name: vulnerability.reference -- external: ecs - name: vulnerability.scanner.vendor -- external: ecs - name: vulnerability.score.base -- external: ecs - name: vulnerability.score.version -- external: ecs - name: host.architecture -- external: ecs - name: host.domain -- external: ecs - name: host.hostname -- external: ecs - name: host.id -- external: ecs - name: host.ip -- external: ecs - name: host.mac -- external: ecs - name: host.name -- external: ecs - name: host.os.family -- external: ecs - name: host.os.kernel -- external: ecs - name: host.os.name -- external: ecs - name: host.os.platform -- external: ecs - name: host.os.version -- external: ecs - name: host.type -- external: ecs - name: cloud.account.id -- external: ecs - name: cloud.account.name -- external: ecs - name: cloud.instance.name -- external: ecs - name: cloud.availability_zone -- external: ecs - name: cloud.instance.id -- external: ecs - name: cloud.machine.type -- external: ecs - name: cloud.project.id -- external: ecs - name: cloud.provider -- external: ecs - name: cloud.region -- external: ecs - name: container.id -- external: ecs - name: container.image.name -- external: ecs - name: container.labels -- external: ecs - name: container.name -- external: ecs - name: event.dataset - type: constant_keyword diff --git a/packages/aws/data_stream/securityhub_findings/sample_event.json b/packages/aws/data_stream/securityhub_findings/sample_event.json index 0fd640c0e11..6a33e31d0fb 100644 --- a/packages/aws/data_stream/securityhub_findings/sample_event.json +++ b/packages/aws/data_stream/securityhub_findings/sample_event.json @@ -338,7 +338,7 @@ "port": 80 }, "ecs": { - "version": "8.2.0" + "version": "8.11.0" }, "elastic_agent": { "id": "eea1c0db-3657-4195-add3-da25a54834e7", diff --git a/packages/aws/data_stream/securityhub_insights/_dev/test/pipeline/test-securityhub-insights.log-expected.json b/packages/aws/data_stream/securityhub_insights/_dev/test/pipeline/test-securityhub-insights.log-expected.json index 3eb8c13c1c4..b1641d5c89f 100644 --- a/packages/aws/data_stream/securityhub_insights/_dev/test/pipeline/test-securityhub-insights.log-expected.json +++ b/packages/aws/data_stream/securityhub_insights/_dev/test/pipeline/test-securityhub-insights.log-expected.json @@ -711,7 +711,7 @@ } }, "ecs": { - "version": "8.2.0" + "version": "8.11.0" }, "event": { "kind": "event", diff --git a/packages/aws/data_stream/securityhub_insights/elasticsearch/ingest_pipeline/default.yml b/packages/aws/data_stream/securityhub_insights/elasticsearch/ingest_pipeline/default.yml index 43888c7671b..8937a9f5dfc 100644 --- a/packages/aws/data_stream/securityhub_insights/elasticsearch/ingest_pipeline/default.yml +++ b/packages/aws/data_stream/securityhub_insights/elasticsearch/ingest_pipeline/default.yml @@ -3,7 +3,7 @@ description: Pipeline for processing AWS Security Hub Insights logs. processors: - set: field: ecs.version - value: '8.2.0' + value: '8.11.0' - set: field: event.kind value: event diff --git a/packages/aws/data_stream/securityhub_insights/fields/ecs.yml b/packages/aws/data_stream/securityhub_insights/fields/ecs.yml deleted file mode 100644 index dfb31b445e2..00000000000 --- a/packages/aws/data_stream/securityhub_insights/fields/ecs.yml +++ /dev/null @@ -1,69 +0,0 @@ -- external: ecs - name: ecs.version -- external: ecs - name: event.category -- external: ecs - name: event.created -- external: ecs - name: event.kind -- external: ecs - name: event.original -- external: ecs - name: event.type -- external: ecs - name: tags -- external: ecs - name: host.architecture -- external: ecs - name: host.domain -- external: ecs - name: host.hostname -- external: ecs - name: host.id -- external: ecs - name: host.ip -- external: ecs - name: host.mac -- external: ecs - name: host.name -- external: ecs - name: host.os.family -- external: ecs - name: host.os.kernel -- external: ecs - name: host.os.name -- external: ecs - name: host.os.platform -- external: ecs - name: host.os.version -- external: ecs - name: host.type -- external: ecs - name: cloud.account.id -- external: ecs - name: cloud.account.name -- external: ecs - name: cloud.availability_zone -- external: ecs - name: cloud.instance.id -- external: ecs - name: cloud.instance.name -- external: ecs - name: cloud.machine.type -- external: ecs - name: cloud.project.id -- external: ecs - name: cloud.provider -- external: ecs - name: cloud.region -- external: ecs - name: container.id -- external: ecs - name: container.image.name -- external: ecs - name: container.labels -- external: ecs - name: container.name -- external: ecs - name: event.dataset - type: constant_keyword diff --git a/packages/aws/data_stream/securityhub_insights/sample_event.json b/packages/aws/data_stream/securityhub_insights/sample_event.json index 35beb7eb20d..4fb34b7066e 100644 --- a/packages/aws/data_stream/securityhub_insights/sample_event.json +++ b/packages/aws/data_stream/securityhub_insights/sample_event.json @@ -722,7 +722,7 @@ "type": "logs" }, "ecs": { - "version": "8.2.0" + "version": "8.11.0" }, "elastic_agent": { "id": "eea1c0db-3657-4195-add3-da25a54834e7", diff --git a/packages/aws/data_stream/sns/fields/ecs.yml b/packages/aws/data_stream/sns/fields/ecs.yml index e355dab5237..303e169c534 100644 --- a/packages/aws/data_stream/sns/fields/ecs.yml +++ b/packages/aws/data_stream/sns/fields/ecs.yml @@ -1,70 +1,9 @@ -- external: ecs - name: cloud - external: ecs name: cloud.account.id dimension: true -- external: ecs - name: cloud.account.name -- external: ecs - name: cloud.availability_zone -- external: ecs - name: cloud.instance.name -- external: ecs - name: cloud.project.id -- external: ecs - name: cloud.instance.id -- external: ecs - name: cloud.machine.type -- external: ecs - name: cloud.provider - external: ecs name: cloud.region dimension: true -- external: ecs - name: ecs.version -- external: ecs - name: error -- external: ecs - name: error.message -- external: ecs - name: service.type -- external: ecs - name: host.architecture -- external: ecs - name: host.domain -- external: ecs - name: host.hostname -- external: ecs - name: host.id -- external: ecs - name: host.ip -- external: ecs - name: host.mac -- external: ecs - name: host.name -- external: ecs - name: host.os.family -- external: ecs - name: host.os.kernel -- external: ecs - name: host.os.name -- external: ecs - name: host.os.platform -- external: ecs - name: host.os.version -- external: ecs - name: host.type -- external: ecs - name: container.id -- external: ecs - name: container.image.name -- external: ecs - name: container.labels -- external: ecs - name: container.name - name: agent.id external: ecs dimension: true -- external: ecs - name: event.dataset - type: constant_keyword diff --git a/packages/aws/data_stream/sns/sample_event.json b/packages/aws/data_stream/sns/sample_event.json index 6b42816eac3..e115eefab3c 100644 --- a/packages/aws/data_stream/sns/sample_event.json +++ b/packages/aws/data_stream/sns/sample_event.json @@ -1,69 +1,69 @@ { "@timestamp": "2022-07-26T21:56:00.000Z", "agent": { - "name": "docker-fleet-agent", - "id": "2d4b09d0-cdb6-445e-ac3f-6415f87b9864", "ephemeral_id": "51866723-6dfa-4a72-a68e-f439d5de7f53", + "id": "2d4b09d0-cdb6-445e-ac3f-6415f87b9864", + "name": "docker-fleet-agent", "type": "metricbeat", "version": "8.3.2" }, - "elastic_agent": { - "id": "2d4b09d0-cdb6-445e-ac3f-6415f87b9864", - "version": "8.3.2", - "snapshot": false - }, - "cloud": { - "provider": "aws", - "region": "us-east-1", - "account": { - "name": "elastic-beats", - "id": "428152502467" - } - }, - "ecs": { - "version": "8.0.0" - }, - "data_stream": { - "namespace": "default", - "type": "metrics", - "dataset": "aws.sns" - }, - "service": { - "type": "aws" - }, - "metricset": { - "period": 300000, - "name": "cloudwatch" - }, "aws": { + "cloudwatch": { + "namespace": "AWS/SNS" + }, + "dimensions": { + "TopicName": "vpc-flow-logs-sns-topic" + }, "sns": { "metrics": { - "NumberOfNotificationsDelivered": { - "sum": 5 - }, "NumberOfMessagesPublished": { "sum": 6 }, - "PublishSize": { - "avg": 905 + "NumberOfNotificationsDelivered": { + "sum": 5 }, "NumberOfNotificationsFailed": { "sum": 0 + }, + "PublishSize": { + "avg": 905 } } - }, - "cloudwatch": { - "namespace": "AWS/SNS" - }, - "dimensions": { - "TopicName": "vpc-flow-logs-sns-topic" } }, + "cloud": { + "account": { + "id": "428152502467", + "name": "elastic-beats" + }, + "provider": "aws", + "region": "us-east-1" + }, + "data_stream": { + "dataset": "aws.sns", + "namespace": "default", + "type": "metrics" + }, + "ecs": { + "version": "8.11.0" + }, + "elastic_agent": { + "id": "2d4b09d0-cdb6-445e-ac3f-6415f87b9864", + "snapshot": false, + "version": "8.3.2" + }, "event": { - "duration": 10483932100, "agent_id_status": "verified", + "dataset": "aws.sns", + "duration": 10483932100, "ingested": "2022-07-26T22:01:00Z", - "module": "aws", - "dataset": "aws.sns" + "module": "aws" + }, + "metricset": { + "name": "cloudwatch", + "period": 300000 + }, + "service": { + "type": "aws" } } \ No newline at end of file diff --git a/packages/aws/data_stream/sqs/fields/ecs.yml b/packages/aws/data_stream/sqs/fields/ecs.yml index f91cecd5a3d..303e169c534 100644 --- a/packages/aws/data_stream/sqs/fields/ecs.yml +++ b/packages/aws/data_stream/sqs/fields/ecs.yml @@ -1,70 +1,9 @@ -- external: ecs - name: cloud - external: ecs name: cloud.account.id dimension: true -- external: ecs - name: cloud.account.name -- external: ecs - name: cloud.availability_zone -- external: ecs - name: cloud.instance.id -- external: ecs - name: cloud.instance.name -- external: ecs - name: cloud.project.id -- external: ecs - name: cloud.machine.type -- external: ecs - name: cloud.provider - external: ecs name: cloud.region dimension: true -- external: ecs - name: ecs.version -- external: ecs - name: error -- external: ecs - name: error.message -- external: ecs - name: service.type -- external: ecs - name: host.architecture -- external: ecs - name: host.domain -- external: ecs - name: host.hostname -- external: ecs - name: host.id -- external: ecs - name: host.ip -- external: ecs - name: host.mac -- external: ecs - name: host.name -- external: ecs - name: host.os.family -- external: ecs - name: host.os.kernel -- external: ecs - name: host.os.name -- external: ecs - name: host.os.platform -- external: ecs - name: host.os.version -- external: ecs - name: host.type -- external: ecs - name: container.id -- external: ecs - name: container.image.name -- external: ecs - name: container.labels -- external: ecs - name: container.name - name: agent.id external: ecs dimension: true -- external: ecs - name: event.dataset - type: constant_keyword diff --git a/packages/aws/data_stream/sqs/sample_event.json b/packages/aws/data_stream/sqs/sample_event.json index 8e3191cdf0c..6162522ebe4 100644 --- a/packages/aws/data_stream/sqs/sample_event.json +++ b/packages/aws/data_stream/sqs/sample_event.json @@ -1,76 +1,76 @@ { "@timestamp": "2022-07-26T21:43:00.000Z", "agent": { - "name": "docker-fleet-agent", + "ephemeral_id": "cdaaaabb-be7e-432f-816b-bda019fd7c15", "id": "2d4b09d0-cdb6-445e-ac3f-6415f87b9864", + "name": "docker-fleet-agent", "type": "metricbeat", - "ephemeral_id": "cdaaaabb-be7e-432f-816b-bda019fd7c15", "version": "8.3.2" }, - "elastic_agent": { - "id": "2d4b09d0-cdb6-445e-ac3f-6415f87b9864", - "version": "8.3.2", - "snapshot": false - }, - "cloud": { - "provider": "aws", - "region": "eu-central-1", - "account": { - "name": "elastic-beats", - "id": "428152502467" - } - }, - "ecs": { - "version": "8.0.0" - }, - "service": { - "type": "aws" - }, - "data_stream": { - "namespace": "default", - "type": "metrics", - "dataset": "aws.sqs" - }, - "metricset": { - "period": 300000, - "name": "cloudwatch" - }, "aws": { + "cloudwatch": { + "namespace": "AWS/SQS" + }, + "dimensions": { + "QueueName": "filebeat-aws-elb-test" + }, "sqs": { + "empty_receives": 0, "messages": { - "visible": 1518.4, + "delayed": 0, "deleted": 0, "not_visible": 0, - "delayed": 0, "received": 0, - "sent": 0.16666666666666666 - }, - "empty_receives": 0, - "sent_message_size": { - "bytes": 1002 + "sent": 0.16666666666666666, + "visible": 1518.4 }, "oldest_message_age": { "sec": 345605.6 }, "queue": { "name": "filebeat-aws-elb-test" + }, + "sent_message_size": { + "bytes": 1002 } }, - "cloudwatch": { - "namespace": "AWS/SQS" - }, - "dimensions": { - "QueueName": "filebeat-aws-elb-test" - }, "tags": { "created-by": "kaiyan" } }, + "cloud": { + "account": { + "id": "428152502467", + "name": "elastic-beats" + }, + "provider": "aws", + "region": "eu-central-1" + }, + "data_stream": { + "dataset": "aws.sqs", + "namespace": "default", + "type": "metrics" + }, + "ecs": { + "version": "8.11.0" + }, + "elastic_agent": { + "id": "2d4b09d0-cdb6-445e-ac3f-6415f87b9864", + "snapshot": false, + "version": "8.3.2" + }, "event": { - "duration": 11576777300, "agent_id_status": "verified", + "dataset": "aws.sqs", + "duration": 11576777300, "ingested": "2022-07-26T21:47:48Z", - "module": "aws", - "dataset": "aws.sqs" + "module": "aws" + }, + "metricset": { + "name": "cloudwatch", + "period": 300000 + }, + "service": { + "type": "aws" } } \ No newline at end of file diff --git a/packages/aws/data_stream/transitgateway/fields/ecs.yml b/packages/aws/data_stream/transitgateway/fields/ecs.yml index e355dab5237..303e169c534 100644 --- a/packages/aws/data_stream/transitgateway/fields/ecs.yml +++ b/packages/aws/data_stream/transitgateway/fields/ecs.yml @@ -1,70 +1,9 @@ -- external: ecs - name: cloud - external: ecs name: cloud.account.id dimension: true -- external: ecs - name: cloud.account.name -- external: ecs - name: cloud.availability_zone -- external: ecs - name: cloud.instance.name -- external: ecs - name: cloud.project.id -- external: ecs - name: cloud.instance.id -- external: ecs - name: cloud.machine.type -- external: ecs - name: cloud.provider - external: ecs name: cloud.region dimension: true -- external: ecs - name: ecs.version -- external: ecs - name: error -- external: ecs - name: error.message -- external: ecs - name: service.type -- external: ecs - name: host.architecture -- external: ecs - name: host.domain -- external: ecs - name: host.hostname -- external: ecs - name: host.id -- external: ecs - name: host.ip -- external: ecs - name: host.mac -- external: ecs - name: host.name -- external: ecs - name: host.os.family -- external: ecs - name: host.os.kernel -- external: ecs - name: host.os.name -- external: ecs - name: host.os.platform -- external: ecs - name: host.os.version -- external: ecs - name: host.type -- external: ecs - name: container.id -- external: ecs - name: container.image.name -- external: ecs - name: container.labels -- external: ecs - name: container.name - name: agent.id external: ecs dimension: true -- external: ecs - name: event.dataset - type: constant_keyword diff --git a/packages/aws/data_stream/transitgateway/sample_event.json b/packages/aws/data_stream/transitgateway/sample_event.json index 9730f63daa3..05901a83e40 100644 --- a/packages/aws/data_stream/transitgateway/sample_event.json +++ b/packages/aws/data_stream/transitgateway/sample_event.json @@ -1,102 +1,102 @@ { + "@timestamp": "2022-07-26T21:58:00.000Z", "agent": { - "name": "a20ad158868c", - "id": "ac8c5411-b1d9-486a-baf7-a719744b13e5", "ephemeral_id": "d43b281f-9a3e-48be-a7b2-e70c0d0b9acd", - "type": "metricbeat", - "version": "8.1.0" - }, - "elastic_agent": { "id": "ac8c5411-b1d9-486a-baf7-a719744b13e5", - "version": "8.1.0", - "snapshot": false - }, - "cloud": { - "provider": "aws", - "region": "eu-west-1", - "account": { - "name": "elastic-observability", - "id": "627286350134" - } - }, - "@timestamp": "2022-07-26T21:58:00.000Z", - "ecs": { - "version": "8.0.0" - }, - "data_stream": { - "namespace": "default", - "type": "metrics", - "dataset": "aws.transitgateway" - }, - "service": { - "type": "aws" - }, - "host": { - "hostname": "a20ad158868c", - "os": { - "kernel": "5.10.104-linuxkit", - "codename": "focal", - "name": "Ubuntu", - "type": "linux", - "family": "debian", - "version": "20.04.3 LTS (Focal Fossa)", - "platform": "ubuntu" - }, - "containerized": false, - "ip": [ - "172.20.0.7" - ], "name": "a20ad158868c", - "mac": [ - "02-42-AC-14-00-07" - ], - "architecture": "aarch64" - }, - "metricset": { - "period": 60000, - "name": "cloudwatch" + "type": "metricbeat", + "version": "8.1.0" }, "aws": { "cloudwatch": { "namespace": "AWS/TransitGateway" }, + "dimensions": { + "TransitGateway": "tgw-04653af6191a63891" + }, "transitgateway": { "metrics": { - "PacketsOut": { + "BytesDropCountBlackhole": { "sum": 0 }, "BytesDropCountNoRoute": { "sum": 0 }, - "PacketDropCountNoRoute": { + "BytesIn": { "sum": 0 }, "BytesOut": { "sum": 0 }, - "BytesIn": { + "PacketDropCountBlackhole": { "sum": 0 }, - "PacketsIn": { + "PacketDropCountNoRoute": { "sum": 0 }, - "BytesDropCountBlackhole": { + "PacketsIn": { "sum": 0 }, - "PacketDropCountBlackhole": { + "PacketsOut": { "sum": 0 } } - }, - "dimensions": { - "TransitGateway": "tgw-04653af6191a63891" } }, + "cloud": { + "account": { + "id": "627286350134", + "name": "elastic-observability" + }, + "provider": "aws", + "region": "eu-west-1" + }, + "data_stream": { + "dataset": "aws.transitgateway", + "namespace": "default", + "type": "metrics" + }, + "ecs": { + "version": "8.11.0" + }, + "elastic_agent": { + "id": "ac8c5411-b1d9-486a-baf7-a719744b13e5", + "snapshot": false, + "version": "8.1.0" + }, "event": { - "duration": 1614567042, "agent_id_status": "verified", + "dataset": "aws.transitgateway", + "duration": 1614567042, "ingested": "2022-07-26T21:59:04Z", - "module": "aws", - "dataset": "aws.transitgateway" + "module": "aws" + }, + "host": { + "architecture": "aarch64", + "containerized": false, + "hostname": "a20ad158868c", + "ip": [ + "172.20.0.7" + ], + "mac": [ + "02-42-AC-14-00-07" + ], + "name": "a20ad158868c", + "os": { + "codename": "focal", + "family": "debian", + "kernel": "5.10.104-linuxkit", + "name": "Ubuntu", + "platform": "ubuntu", + "type": "linux", + "version": "20.04.3 LTS (Focal Fossa)" + } + }, + "metricset": { + "name": "cloudwatch", + "period": 60000 + }, + "service": { + "type": "aws" } } \ No newline at end of file diff --git a/packages/aws/data_stream/usage/fields/ecs.yml b/packages/aws/data_stream/usage/fields/ecs.yml index e355dab5237..303e169c534 100644 --- a/packages/aws/data_stream/usage/fields/ecs.yml +++ b/packages/aws/data_stream/usage/fields/ecs.yml @@ -1,70 +1,9 @@ -- external: ecs - name: cloud - external: ecs name: cloud.account.id dimension: true -- external: ecs - name: cloud.account.name -- external: ecs - name: cloud.availability_zone -- external: ecs - name: cloud.instance.name -- external: ecs - name: cloud.project.id -- external: ecs - name: cloud.instance.id -- external: ecs - name: cloud.machine.type -- external: ecs - name: cloud.provider - external: ecs name: cloud.region dimension: true -- external: ecs - name: ecs.version -- external: ecs - name: error -- external: ecs - name: error.message -- external: ecs - name: service.type -- external: ecs - name: host.architecture -- external: ecs - name: host.domain -- external: ecs - name: host.hostname -- external: ecs - name: host.id -- external: ecs - name: host.ip -- external: ecs - name: host.mac -- external: ecs - name: host.name -- external: ecs - name: host.os.family -- external: ecs - name: host.os.kernel -- external: ecs - name: host.os.name -- external: ecs - name: host.os.platform -- external: ecs - name: host.os.version -- external: ecs - name: host.type -- external: ecs - name: container.id -- external: ecs - name: container.image.name -- external: ecs - name: container.labels -- external: ecs - name: container.name - name: agent.id external: ecs dimension: true -- external: ecs - name: event.dataset - type: constant_keyword diff --git a/packages/aws/data_stream/usage/sample_event.json b/packages/aws/data_stream/usage/sample_event.json index aec21f75536..2af1e0d08c0 100644 --- a/packages/aws/data_stream/usage/sample_event.json +++ b/packages/aws/data_stream/usage/sample_event.json @@ -1,63 +1,63 @@ { "@timestamp": "2022-07-25T20:50:00.000Z", "agent": { - "name": "docker-fleet-agent", + "ephemeral_id": "6bab70d4-84d9-411d-887c-f144d4244e78", "id": "2d4b09d0-cdb6-445e-ac3f-6415f87b9864", + "name": "docker-fleet-agent", "type": "metricbeat", - "ephemeral_id": "6bab70d4-84d9-411d-887c-f144d4244e78", "version": "8.3.2" }, - "elastic_agent": { - "id": "2d4b09d0-cdb6-445e-ac3f-6415f87b9864", - "version": "8.3.2", - "snapshot": false - }, - "cloud": { - "provider": "aws", - "region": "eu-north-1", - "account": { - "name": "elastic-beats", - "id": "428152502467" - } - }, - "ecs": { - "version": "8.0.0" - }, - "service": { - "type": "aws" - }, - "data_stream": { - "namespace": "default", - "type": "metrics", - "dataset": "aws.usage" - }, - "metricset": { - "period": 60000, - "name": "cloudwatch" - }, "aws": { - "usage": { - "metrics": { - "CallCount": { - "sum": 1 - } - } - }, "cloudwatch": { "namespace": "AWS/Usage" }, "dimensions": { - "Type": "API", + "Class": "None", "Resource": "ListMetrics", "Service": "CloudWatch", - "Class": "None" + "Type": "API" + }, + "usage": { + "metrics": { + "CallCount": { + "sum": 1 + } + } } }, + "cloud": { + "account": { + "id": "428152502467", + "name": "elastic-beats" + }, + "provider": "aws", + "region": "eu-north-1" + }, + "data_stream": { + "dataset": "aws.usage", + "namespace": "default", + "type": "metrics" + }, + "ecs": { + "version": "8.11.0" + }, + "elastic_agent": { + "id": "2d4b09d0-cdb6-445e-ac3f-6415f87b9864", + "snapshot": false, + "version": "8.3.2" + }, "event": { - "duration": 1432082500, "agent_id_status": "verified", + "dataset": "aws.usage", + "duration": 1432082500, "ingested": "2022-07-25T20:51:19Z", - "module": "aws", - "dataset": "aws.usage" + "module": "aws" + }, + "metricset": { + "name": "cloudwatch", + "period": 60000 + }, + "service": { + "type": "aws" } } \ No newline at end of file diff --git a/packages/aws/data_stream/vpcflow/_dev/test/pipeline/test-extra-samples.log-expected.json b/packages/aws/data_stream/vpcflow/_dev/test/pipeline/test-extra-samples.log-expected.json index 73d5f783e99..9e112d0bfd0 100644 --- a/packages/aws/data_stream/vpcflow/_dev/test/pipeline/test-extra-samples.log-expected.json +++ b/packages/aws/data_stream/vpcflow/_dev/test/pipeline/test-extra-samples.log-expected.json @@ -32,7 +32,7 @@ "port": 22 }, "ecs": { - "version": "8.0.0" + "version": "8.11.0" }, "event": { "category": [ @@ -98,7 +98,7 @@ "provider": "aws" }, "ecs": { - "version": "8.0.0" + "version": "8.11.0" }, "event": { "category": [ @@ -133,7 +133,7 @@ "provider": "aws" }, "ecs": { - "version": "8.0.0" + "version": "8.11.0" }, "event": { "category": [ @@ -192,7 +192,7 @@ "port": 22 }, "ecs": { - "version": "8.0.0" + "version": "8.11.0" }, "event": { "category": [ @@ -291,7 +291,7 @@ "port": 3389 }, "ecs": { - "version": "8.0.0" + "version": "8.11.0" }, "event": { "category": [ @@ -372,7 +372,7 @@ "port": 0 }, "ecs": { - "version": "8.0.0" + "version": "8.11.0" }, "event": { "category": [ @@ -471,7 +471,7 @@ "port": 0 }, "ecs": { - "version": "8.0.0" + "version": "8.11.0" }, "event": { "category": [ diff --git a/packages/aws/data_stream/vpcflow/_dev/test/pipeline/test-tcp-flag-sequence.log-expected.json b/packages/aws/data_stream/vpcflow/_dev/test/pipeline/test-tcp-flag-sequence.log-expected.json index 5b47f22a27f..d7d092f63a2 100644 --- a/packages/aws/data_stream/vpcflow/_dev/test/pipeline/test-tcp-flag-sequence.log-expected.json +++ b/packages/aws/data_stream/vpcflow/_dev/test/pipeline/test-tcp-flag-sequence.log-expected.json @@ -36,7 +36,7 @@ "port": 5001 }, "ecs": { - "version": "8.0.0" + "version": "8.11.0" }, "event": { "category": [ @@ -117,7 +117,7 @@ "provider": "aws" }, "ecs": { - "version": "8.0.0" + "version": "8.11.0" }, "event": { "category": [ @@ -158,7 +158,7 @@ "provider": "aws" }, "ecs": { - "version": "8.0.0" + "version": "8.11.0" }, "event": { "category": [ diff --git a/packages/aws/data_stream/vpcflow/_dev/test/pipeline/test-v5-all-fields.log-expected.json b/packages/aws/data_stream/vpcflow/_dev/test/pipeline/test-v5-all-fields.log-expected.json index aab712c0ee9..9e82baf7f28 100644 --- a/packages/aws/data_stream/vpcflow/_dev/test/pipeline/test-v5-all-fields.log-expected.json +++ b/packages/aws/data_stream/vpcflow/_dev/test/pipeline/test-v5-all-fields.log-expected.json @@ -42,7 +42,7 @@ "port": 33004 }, "ecs": { - "version": "8.0.0" + "version": "8.11.0" }, "event": { "category": [ diff --git a/packages/aws/data_stream/vpcflow/_dev/test/pipeline/test-with-message-field.log-expected.json b/packages/aws/data_stream/vpcflow/_dev/test/pipeline/test-with-message-field.log-expected.json index bc9b1592f33..fcadc8fbc74 100644 --- a/packages/aws/data_stream/vpcflow/_dev/test/pipeline/test-with-message-field.log-expected.json +++ b/packages/aws/data_stream/vpcflow/_dev/test/pipeline/test-with-message-field.log-expected.json @@ -17,7 +17,7 @@ "provider": "aws" }, "ecs": { - "version": "8.0.0" + "version": "8.11.0" }, "event": { "category": [ diff --git a/packages/aws/data_stream/vpcflow/elasticsearch/ingest_pipeline/default.yml b/packages/aws/data_stream/vpcflow/elasticsearch/ingest_pipeline/default.yml index d607c64eb3b..08fa43547c3 100644 --- a/packages/aws/data_stream/vpcflow/elasticsearch/ingest_pipeline/default.yml +++ b/packages/aws/data_stream/vpcflow/elasticsearch/ingest_pipeline/default.yml @@ -4,7 +4,7 @@ description: Pipeline for AWS VPC Flow Logs processors: - set: field: ecs.version - value: '8.0.0' + value: '8.11.0' - dot_expander: field: "*" - rename: diff --git a/packages/aws/data_stream/vpcflow/fields/ecs.yml b/packages/aws/data_stream/vpcflow/fields/ecs.yml deleted file mode 100644 index 553ef9532ff..00000000000 --- a/packages/aws/data_stream/vpcflow/fields/ecs.yml +++ /dev/null @@ -1,143 +0,0 @@ -- name: cloud.account.id - external: ecs -- external: ecs - name: cloud.availability_zone -- external: ecs - name: cloud.instance.name -- external: ecs - name: cloud.project.id -- external: ecs - name: cloud.machine.type -- name: cloud.instance.id - external: ecs -- name: cloud.provider - external: ecs -- external: ecs - name: cloud.region -- name: destination.address - external: ecs -- name: destination.as.number - external: ecs -- name: destination.as.organization.name - external: ecs -- name: destination.geo.city_name - external: ecs -- name: destination.geo.continent_name - external: ecs -- name: destination.geo.country_iso_code - external: ecs -- name: destination.geo.country_name - external: ecs -- name: destination.geo.location - external: ecs -- name: destination.geo.region_iso_code - external: ecs -- name: destination.geo.region_name - external: ecs -- name: destination.ip - external: ecs -- name: destination.port - external: ecs -- name: ecs.version - external: ecs -- name: error.message - external: ecs -- name: event.category - external: ecs -- name: event.end - external: ecs -- name: event.kind - external: ecs -- name: event.original - external: ecs -- name: event.outcome - external: ecs -- name: event.start - external: ecs -- name: event.type - external: ecs -- name: network.bytes - external: ecs -- name: network.community_id - external: ecs -- name: network.iana_number - external: ecs -- name: network.packets - external: ecs -- name: network.transport - external: ecs -- name: network.type - external: ecs -- name: network.direction - external: ecs -- name: related.ip - external: ecs -- name: source.address - external: ecs -- name: source.as.number - external: ecs -- name: source.as.organization.name - external: ecs -- name: source.bytes - external: ecs -- name: source.geo.city_name - external: ecs -- name: source.geo.continent_name - external: ecs -- name: source.geo.country_iso_code - external: ecs -- name: source.geo.country_name - external: ecs -- name: source.geo.location - external: ecs -- name: source.geo.region_iso_code - external: ecs -- name: source.geo.region_name - external: ecs -- name: source.ip - external: ecs -- name: source.packets - external: ecs -- name: source.port - external: ecs -- name: tags - external: ecs -- external: ecs - name: host.architecture -- external: ecs - name: host.domain -- external: ecs - name: host.hostname -- external: ecs - name: host.id -- external: ecs - name: host.ip -- external: ecs - name: host.mac -- external: ecs - name: host.name -- external: ecs - name: host.os.family -- external: ecs - name: host.os.kernel -- external: ecs - name: host.os.name -- external: ecs - name: host.os.platform -- external: ecs - name: host.os.version -- external: ecs - name: host.type -- external: ecs - name: container.id -- external: ecs - name: container.image.name -- external: ecs - name: container.labels -- external: ecs - name: container.name -- external: ecs - name: log.file.path -- external: ecs - name: event.dataset - type: constant_keyword diff --git a/packages/aws/data_stream/vpcflow/sample_event.json b/packages/aws/data_stream/vpcflow/sample_event.json index 82d3e7c23b6..1ab87d5dff1 100644 --- a/packages/aws/data_stream/vpcflow/sample_event.json +++ b/packages/aws/data_stream/vpcflow/sample_event.json @@ -61,7 +61,7 @@ "port": 22 }, "ecs": { - "version": "8.0.0" + "version": "8.11.0" }, "elastic_agent": { "id": "acba78ef-1401-4689-977c-d8c2e5d6a8fa", diff --git a/packages/aws/data_stream/vpn/fields/ecs.yml b/packages/aws/data_stream/vpn/fields/ecs.yml index f91cecd5a3d..303e169c534 100644 --- a/packages/aws/data_stream/vpn/fields/ecs.yml +++ b/packages/aws/data_stream/vpn/fields/ecs.yml @@ -1,70 +1,9 @@ -- external: ecs - name: cloud - external: ecs name: cloud.account.id dimension: true -- external: ecs - name: cloud.account.name -- external: ecs - name: cloud.availability_zone -- external: ecs - name: cloud.instance.id -- external: ecs - name: cloud.instance.name -- external: ecs - name: cloud.project.id -- external: ecs - name: cloud.machine.type -- external: ecs - name: cloud.provider - external: ecs name: cloud.region dimension: true -- external: ecs - name: ecs.version -- external: ecs - name: error -- external: ecs - name: error.message -- external: ecs - name: service.type -- external: ecs - name: host.architecture -- external: ecs - name: host.domain -- external: ecs - name: host.hostname -- external: ecs - name: host.id -- external: ecs - name: host.ip -- external: ecs - name: host.mac -- external: ecs - name: host.name -- external: ecs - name: host.os.family -- external: ecs - name: host.os.kernel -- external: ecs - name: host.os.name -- external: ecs - name: host.os.platform -- external: ecs - name: host.os.version -- external: ecs - name: host.type -- external: ecs - name: container.id -- external: ecs - name: container.image.name -- external: ecs - name: container.labels -- external: ecs - name: container.name - name: agent.id external: ecs dimension: true -- external: ecs - name: event.dataset - type: constant_keyword diff --git a/packages/aws/data_stream/vpn/sample_event.json b/packages/aws/data_stream/vpn/sample_event.json index a5f331f9c5e..aeeda003725 100644 --- a/packages/aws/data_stream/vpn/sample_event.json +++ b/packages/aws/data_stream/vpn/sample_event.json @@ -1,51 +1,51 @@ { "@timestamp": "2020-05-28T17:58:27.154Z", - "service": { - "type": "aws" - }, - "ecs": { - "version": "1.5.0" + "agent": { + "ephemeral_id": "17803f33-b617-4ce9-a9ac-e218c02aeb4b", + "id": "12f376ef-5186-4e8b-a175-70f1140a8f30", + "name": "MacBook-Elastic.local", + "type": "metricbeat", + "version": "8.0.0" }, "aws": { + "cloudwatch": { + "namespace": "AWS/VPN" + }, "vpn": { "metrics": { - "TunnelState": { - "avg": 0 - }, "TunnelDataIn": { "sum": 0 }, "TunnelDataOut": { "sum": 0 + }, + "TunnelState": { + "avg": 0 } } - }, - "cloudwatch": { - "namespace": "AWS/VPN" } }, + "cloud": { + "account": { + "id": "428152502467", + "name": "elastic-beats" + }, + "provider": "aws", + "region": "us-west-2" + }, + "ecs": { + "version": "8.11.0" + }, "event": { "dataset": "aws.vpn", - "module": "aws", - "duration": 10418157072 + "duration": 10418157072, + "module": "aws" }, "metricset": { - "period": 60000, - "name": "vpn" - }, - "cloud": { - "region": "us-west-2", - "account": { - "name": "elastic-beats", - "id": "428152502467" - }, - "provider": "aws" + "name": "vpn", + "period": 60000 }, - "agent": { - "version": "8.0.0", - "ephemeral_id": "17803f33-b617-4ce9-a9ac-e218c02aeb4b", - "id": "12f376ef-5186-4e8b-a175-70f1140a8f30", - "name": "MacBook-Elastic.local", - "type": "metricbeat" + "service": { + "type": "aws" } } \ No newline at end of file diff --git a/packages/aws/data_stream/waf/_dev/test/pipeline/test-waf.log-expected.json b/packages/aws/data_stream/waf/_dev/test/pipeline/test-waf.log-expected.json index 7d4ef0c9d79..f0852ca8657 100644 --- a/packages/aws/data_stream/waf/_dev/test/pipeline/test-waf.log-expected.json +++ b/packages/aws/data_stream/waf/_dev/test/pipeline/test-waf.log-expected.json @@ -38,11 +38,14 @@ } }, "ecs": { - "version": "8.0.0" + "version": "8.11.0" }, "event": { "action": "BLOCK", - "category": "web", + "category": [ + "web", + "network" + ], "kind": "event", "original": "{\"timestamp\":1576280412771,\"formatVersion\":1,\"webaclId\":\"arn:aws:wafv2:ap-southeast-2:EXAMPLE12345:regional/webacl/STMTest/1EXAMPLE-2ARN-3ARN-4ARN-123456EXAMPLE\",\"terminatingRuleId\":\"STMTest_SQLi_XSS\",\"terminatingRuleType\":\"REGULAR\",\"action\":\"BLOCK\",\"terminatingRuleMatchDetails\":[{\"conditionType\":\"SQL_INJECTION\",\"location\":\"HEADER\",\"matchedData\":[\"10\",\"AND\",\"1\"]}],\"httpSourceName\":\"-\",\"httpSourceId\":\"-\",\"ruleGroupList\":[],\"rateBasedRuleList\":[],\"nonTerminatingMatchingRules\":[],\"httpRequest\":{\"clientIp\":\"89.160.20.156\",\"country\":\"AU\",\"headers\":[{\"name\":\"Host\",\"value\":\"localhost:1989\"},{\"name\":\"User-Agent\",\"value\":\"curl/7.61.1\"},{\"name\":\"Accept\",\"value\":\"*/*\"},{\"name\":\"x-stm-test\",\"value\":\"10 AND 1=1\"}],\"uri\":\"/foo\",\"args\":\"\",\"httpVersion\":\"HTTP/1.1\",\"httpMethod\":\"GET\",\"requestId\":\"rid\"},\"labels\":[{\"name\":\"value\"}]}", "type": [ @@ -142,11 +145,14 @@ } }, "ecs": { - "version": "8.0.0" + "version": "8.11.0" }, "event": { "action": "ALLOW", - "category": "web", + "category": [ + "web", + "network" + ], "kind": "event", "original": "{\"timestamp\":1592357192516,\"formatVersion\":1,\"webaclId\":\"arn:aws:wafv2:us-east-1:123456789012:global/webacl/hello-world/5933d6d9-9dde-js82-v8aw-9ck28nv9\",\"terminatingRuleId\":\"Default_Action\",\"terminatingRuleType\":\"REGULAR\",\"action\":\"ALLOW\",\"terminatingRuleMatchDetails\":[],\"httpSourceName\":\"-\",\"httpSourceId\":\"-\",\"ruleGroupList\":[],\"rateBasedRuleList\":[],\"nonTerminatingMatchingRules\":[{\"ruleId\":\"TestRule\",\"action\":\"COUNT\",\"ruleMatchDetails\":[{\"conditionType\":\"SQL_INJECTION\",\"location\":\"HEADER\",\"matchedData\":[\"10\",\"and\",\"1\"]}]}],\"httpRequest\":{\"clientIp\":\"89.160.20.156\",\"country\":\"US\",\"headers\":[{\"name\":\"Host\",\"value\":\"localhost:1989\"},{\"name\":\"User-Agent\",\"value\":\"curl/7.61.1\"},{\"name\":\"Accept\",\"value\":\"*/*\"},{\"name\":\"foo\",\"value\":\"10 AND 1=1\"}],\"uri\":\"/foo\",\"args\":\"\",\"httpVersion\":\"HTTP/1.1\",\"httpMethod\":\"GET\",\"requestId\":\"rid\"},\"labels\":[{\"name\":\"value\"}]}", "type": [ @@ -266,11 +272,14 @@ } }, "ecs": { - "version": "8.0.0" + "version": "8.11.0" }, "event": { "action": "BLOCK", - "category": "web", + "category": [ + "web", + "network" + ], "kind": "event", "original": "{\"timestamp\":1592361810888,\"formatVersion\":1,\"webaclId\":\"arn:aws:wafv2:us-east-1:123456789012:global/webacl/hello-world/5933d6d9-9dde-js82-v8aw-9ck28nv9\",\"terminatingRuleId\":\"RG-Reference\",\"terminatingRuleType\":\"GROUP\",\"action\":\"BLOCK\",\"terminatingRuleMatchDetails\":[{\"conditionType\":\"XSS\",\"location\":\"HEADER\",\"matchedData\":[\"<\",\"frameset\"]}],\"httpSourceName\":\"-\",\"httpSourceId\":\"-\",\"ruleGroupList\":[{\"ruleGroupId\":\"arn:aws:wafv2:us-east-1:123456789012:global/rulegroup/hello-world/c05lb698-1f11-4m41-aef4-99a506d53f4b\",\"terminatingRule\":{\"ruleId\":\"RuleA-XSS\",\"action\":\"BLOCK\",\"ruleMatchDetails\":null},\"nonTerminatingMatchingRules\":[{\"ruleId\":\"RuleB-SQLi\",\"action\":\"COUNT\",\"ruleMatchDetails\":[{\"conditionType\":\"SQL_INJECTION\",\"location\":\"HEADER\",\"matchedData\":[\"10\",\"and\",\"1\"]}]}],\"excludedRules\":null}],\"rateBasedRuleList\":[],\"nonTerminatingMatchingRules\":[],\"httpRequest\":{\"clientIp\":\"89.160.20.156\",\"country\":\"US\",\"headers\":[{\"name\":\"Host\",\"value\":\"localhost:1989\"},{\"name\":\"User-Agent\",\"value\":\"curl/7.61.1\"},{\"name\":\"Accept\",\"value\":\"*/*\"},{\"name\":\"xssfoo\",\"value\":\"\"},{\"name\":\"bar\",\"value\":\"10 AND 1=1\"}],\"uri\":\"/foo\",\"args\":\"\",\"httpVersion\":\"HTTP/1.1\",\"httpMethod\":\"GET\",\"requestId\":\"rid\"},\"labels\":[{\"name\":\"value\"}]}", "type": [ @@ -360,11 +369,14 @@ } }, "ecs": { - "version": "8.0.0" + "version": "8.11.0" }, "event": { "action": "BLOCK", - "category": "web", + "category": [ + "web", + "network" + ], "kind": "event", "original": "{\"timestamp\":1576280412771,\"formatVersion\":1,\"webaclId\":\"arn:aws:wafv2:ap-southeast-2:12345:regional/webacl/test/111\",\"terminatingRuleId\":\"STMTest_SQLi_XSS\",\"terminatingRuleType\":\"REGULAR\",\"action\":\"BLOCK\",\"terminatingRuleMatchDetails\":[{\"conditionType\":\"SQL_INJECTION\",\"location\":\"UNKNOWN\",\"matchedData\":[\"10\",\"AND\",\"1\"]}],\"httpSourceName\":\"ALB\",\"httpSourceId\":\"alb\",\"ruleGroupList\":[],\"rateBasedRuleList\":[],\"nonTerminatingMatchingRules\":[],\"requestHeadersInserted\":null,\"responseCodeSent\":null,\"httpRequest\":{\"clientIp\":\"89.160.20.156\",\"country\":\"AU\",\"headers\":[],\"uri\":\"\",\"args\":\"\",\"httpVersion\":\"HTTP/1.1\",\"httpMethod\":\"POST\",\"requestId\":\"null\"},\"labels\":[{\"name\":\"value\"}]}", "type": [ diff --git a/packages/aws/data_stream/waf/elasticsearch/ingest_pipeline/default.yml b/packages/aws/data_stream/waf/elasticsearch/ingest_pipeline/default.yml index f754281e763..5e7c23ab514 100644 --- a/packages/aws/data_stream/waf/elasticsearch/ingest_pipeline/default.yml +++ b/packages/aws/data_stream/waf/elasticsearch/ingest_pipeline/default.yml @@ -3,13 +3,13 @@ description: "Pipeline for WAF logs" processors: - set: field: ecs.version - value: '8.0.0' + value: '8.11.0' - set: field: event.category - value: web + value: ["web", "network"] - append: field: event.type - value: access + value: ["access"] - rename: field: message target_field: event.original diff --git a/packages/aws/data_stream/waf/fields/ecs.yml b/packages/aws/data_stream/waf/fields/ecs.yml deleted file mode 100644 index 2873106116f..00000000000 --- a/packages/aws/data_stream/waf/fields/ecs.yml +++ /dev/null @@ -1,115 +0,0 @@ -- external: ecs - name: source.address -- external: ecs - name: source.ip -- external: ecs - name: ecs.version -- external: ecs - name: error.message -- external: ecs - name: event.action -- external: ecs - name: event.id -- external: ecs - name: event.kind -- external: ecs - name: event.outcome -- external: ecs - name: http.request.method -- external: ecs - name: http.version -- external: ecs - name: http.request.id -- external: ecs - name: related.ip -- external: ecs - name: tags -- external: ecs - name: url.path -- external: ecs - name: url.query -- external: ecs - name: source.as.number -- external: ecs - name: source.as.organization.name -- external: ecs - name: source.geo.city_name -- external: ecs - name: source.geo.continent_name -- external: ecs - name: source.geo.country_iso_code -- external: ecs - name: source.geo.country_name -- description: Longitude and latitude. - level: core - name: source.geo.location - type: geo_point -- external: ecs - name: source.geo.region_iso_code -- external: ecs - name: source.geo.region_name -- external: ecs - name: network.protocol -- external: ecs - name: network.transport -- external: ecs - name: rule.id -- external: ecs - name: rule.ruleset -- external: ecs - name: cloud.account.id -- external: ecs - name: cloud.account.name -- external: ecs - name: cloud.availability_zone -- external: ecs - name: cloud.instance.id -- external: ecs - name: cloud.instance.name -- external: ecs - name: cloud.project.id -- external: ecs - name: cloud.machine.type -- external: ecs - name: cloud.provider -- external: ecs - name: cloud.region -- external: ecs - name: container.id -- external: ecs - name: container.image.name -- external: ecs - name: container.labels -- external: ecs - name: container.name -- external: ecs - name: host.architecture -- external: ecs - name: host.domain -- external: ecs - name: host.hostname -- external: ecs - name: host.id -- external: ecs - name: host.ip -- external: ecs - name: host.mac -- external: ecs - name: host.name -- external: ecs - name: host.os.family -- external: ecs - name: host.os.kernel -- external: ecs - name: host.os.name -- external: ecs - name: host.os.platform -- external: ecs - name: host.os.version -- external: ecs - name: host.type -- external: ecs - name: log.file.path -- external: ecs - name: event.dataset - type: constant_keyword diff --git a/packages/aws/data_stream/waf/sample_event.json b/packages/aws/data_stream/waf/sample_event.json index 8ac674b1481..ea6b4c26c75 100644 --- a/packages/aws/data_stream/waf/sample_event.json +++ b/packages/aws/data_stream/waf/sample_event.json @@ -57,7 +57,7 @@ "type": "logs" }, "ecs": { - "version": "8.0.0" + "version": "8.11.0" }, "elastic_agent": { "id": "acba78ef-1401-4689-977c-d8c2e5d6a8fa", @@ -67,7 +67,10 @@ "event": { "action": "BLOCK", "agent_id_status": "verified", - "category": "web", + "category": [ + "web", + "network" + ], "dataset": "aws.waf", "ingested": "2023-11-08T08:24:54Z", "kind": "event", diff --git a/packages/aws/docs/apigateway.md b/packages/aws/docs/apigateway.md index 2a3edd839ed..f242656b583 100644 --- a/packages/aws/docs/apigateway.md +++ b/packages/aws/docs/apigateway.md @@ -67,63 +67,14 @@ An example event for `apigateway` looks as following: ```json { + "@timestamp": "2023-05-08T16:30:00.000Z", "agent": { - "name": "docker-fleet-agent", + "ephemeral_id": "dfa418e2-1fe7-4039-9e44-bec39fa60341", "id": "fe8366bc-f3f8-4901-acce-b2c6788cf21f", + "name": "docker-fleet-agent", "type": "metricbeat", - "ephemeral_id": "dfa418e2-1fe7-4039-9e44-bec39fa60341", "version": "8.6.2" }, - "@timestamp": "2023-05-08T16:30:00.000Z", - "ecs": { - "version": "8.0.0" - }, - "data_stream": { - "namespace": "default", - "type": "metrics", - "dataset": "aws.apigateway_metrics" - }, - "service": { - "type": "aws" - }, - "host": { - "hostname": "docker-fleet-agent", - "os": { - "kernel": "5.15.90.1-microsoft-standard-WSL2", - "codename": "focal", - "name": "Ubuntu", - "family": "debian", - "type": "linux", - "version": "20.04.5 LTS (Focal Fossa)", - "platform": "ubuntu" - }, - "containerized": false, - "ip": [ - "172.18.0.7" - ], - "name": "docker-fleet-agent", - "id": "f91b175388d423fca58155815dfc2279", - "mac": [ - "02-42-AC-12-00-07" - ], - "architecture": "x86_64" - }, - "elastic_agent": { - "id": "fe8336bc-f3f1-4901-ac0a-b266788cf21f", - "version": "8.6.2", - "snapshot": false - }, - "metricset": { - "period": 300000, - "name": "cloudwatch" - }, - "event": { - "duration": 10830411419, - "agent_id_status": "verified", - "ingested": "2023-05-08T16:39:47Z", - "module": "aws", - "dataset": "aws.apigateway_metrics" - }, "aws": { "apigateway": { "metrics": { @@ -133,12 +84,12 @@ An example event for `apigateway` looks as following: "5xx": { "sum": 0 }, - "DataProcessed": { - "avg": 48460 - }, "Count": { "sum": 2 }, + "DataProcessed": { + "avg": 48460 + }, "IntegrationLatency": { "avg": 85.5 }, @@ -153,10 +104,63 @@ An example event for `apigateway` looks as following: "dimensions": { "ApiId": "6am7mj7jqx" } + }, + "data_stream": { + "dataset": "aws.apigateway_metrics", + "namespace": "default", + "type": "metrics" + }, + "ecs": { + "version": "8.11.0" + }, + "elastic_agent": { + "id": "fe8336bc-f3f1-4901-ac0a-b266788cf21f", + "snapshot": false, + "version": "8.6.2" + }, + "event": { + "agent_id_status": "verified", + "dataset": "aws.apigateway_metrics", + "duration": 10830411419, + "ingested": "2023-05-08T16:39:47Z", + "module": "aws" + }, + "host": { + "architecture": "x86_64", + "containerized": false, + "hostname": "docker-fleet-agent", + "id": "f91b175388d423fca58155815dfc2279", + "ip": [ + "172.18.0.7" + ], + "mac": [ + "02-42-AC-12-00-07" + ], + "name": "docker-fleet-agent", + "os": { + "codename": "focal", + "family": "debian", + "kernel": "5.15.90.1-microsoft-standard-WSL2", + "name": "Ubuntu", + "platform": "ubuntu", + "type": "linux", + "version": "20.04.5 LTS (Focal Fossa)" + } + }, + "metricset": { + "name": "cloudwatch", + "period": 300000 + }, + "service": { + "type": "aws" } } ``` +**ECS Field Reference** + +Please refer to the following [document](https://www.elastic.co/guide/en/ecs/current/ecs-field-reference.html) for detailed information on ECS fields. + **Exported fields** | Field | Description | Type | Unit | Metric Type | @@ -186,47 +190,16 @@ An example event for `apigateway` looks as following: | aws.dimensions.Route | Routes define the path and HTTP methods that clients can use to access different functionalities of the API. | keyword | | | | aws.dimensions.Stage | It represents a specific version of the API that is accessible to clients. A stage allows you to manage different environments or versions of your API, such as development, testing, and production. | keyword | | | | aws.tags | Tag key-value pairs from AWS resources. | flattened | | | -| cloud | Fields related to the cloud or infrastructure the events are coming from. | group | | | | cloud.account.id | The cloud account or organization id used to identify different entities in a multi-tenant environment. Examples: AWS account id, Google Cloud ORG Id, or other unique identifier. | keyword | | | -| cloud.account.name | The cloud account name or alias used to identify different entities in a multi-tenant environment. Examples: AWS account name, Google Cloud ORG display name. | keyword | | | -| cloud.availability_zone | Availability zone in which this host, resource, or service is located. | keyword | | | | cloud.image.id | Image ID for the cloud instance. | keyword | | | -| cloud.instance.id | Instance ID of the host machine. | keyword | | | -| cloud.instance.name | Instance name of the host machine. | keyword | | | -| cloud.machine.type | Machine type of the host machine. | keyword | | | -| cloud.project.id | The cloud project identifier. Examples: Google Cloud Project id, Azure Project id. | keyword | | | -| cloud.provider | Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. | keyword | | | | cloud.region | Region in which this host, resource, or service is located. | keyword | | | -| container.id | Unique container id. | keyword | | | -| container.image.name | Name of the image the container was built on. | keyword | | | -| container.labels | Image labels. | object | | | -| container.name | Container name. | keyword | | | | data_stream.dataset | Data stream dataset. | constant_keyword | | | | data_stream.namespace | Data stream namespace. | constant_keyword | | | | data_stream.type | Data stream type. | constant_keyword | | | -| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | | | -| error | These fields can represent errors of any kind. Use them for errors that happen while fetching events or in cases where the event itself contains an error. | group | | | -| error.message | Error message. | match_only_text | | | -| event.dataset | Name of the dataset. If an event source publishes more than one type of log or events (e.g. access log, error log), the dataset is used to specify which one the event comes from. It's recommended but not required to start the dataset name with the module name, followed by a dot, then the dataset name. | constant_keyword | | | | event.module | Event module | constant_keyword | | | -| host.architecture | Operating system architecture. | keyword | | | | host.containerized | If the host is a container. | boolean | | | -| host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | | | -| host.hostname | Hostname of the host. It normally contains what the `hostname` command returns on the host machine. | keyword | | | -| host.id | Unique host id. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`. | keyword | | | -| host.ip | Host ip addresses. | ip | | | -| host.mac | Host MAC addresses. The notation format from RFC 7042 is suggested: Each octet (that is, 8-bit byte) is represented by two [uppercase] hexadecimal digits giving the value of the octet as an unsigned integer. Successive octets are separated by a hyphen. | keyword | | | -| host.name | Name of the host. It can contain what hostname returns on Unix systems, the fully qualified domain name (FQDN), or a name specified by the user. The recommended value is the lowercase FQDN of the host. | keyword | | | | host.os.build | OS build information. | keyword | | | | host.os.codename | OS codename, if any. | keyword | | | -| host.os.family | OS family (such as redhat, debian, freebsd, windows). | keyword | | | -| host.os.kernel | Operating system kernel version as a raw string. | keyword | | | -| host.os.name | Operating system name, without the version. | keyword | | | -| host.os.name.text | Multi-field of `host.os.name`. | match_only_text | | | -| host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword | | | -| host.os.version | Operating system version as a raw string. | keyword | | | -| host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | | | -| service.type | The type of the service data is collected from. The type can be used to group and correlate logs and metrics from one service type. Example: If logs or metrics are collected from Elasticsearch, `service.type` would be `elasticsearch`. | keyword | | | ## Logs reference @@ -274,7 +247,7 @@ An example event for `apigateway` looks as following: "type": "logs" }, "ecs": { - "version": "8.0.0" + "version": "8.11.0" }, "elastic_agent": { "id": "acba78ef-1401-4689-977c-d8c2e5d6a8fa", @@ -304,6 +277,10 @@ An example event for `apigateway` looks as following: } ``` +**ECS Field Reference** + +Please refer to the following [document](https://www.elastic.co/guide/en/ecs/current/ecs-field-reference.html) for detailed information on ECS fields. + **Exported fields** | Field | Description | Type | @@ -329,45 +306,13 @@ An example event for `apigateway` looks as following: | aws.s3.bucket.name | Name of a S3 bucket. | keyword | | aws.s3.metadata | AWS S3 object metadata values. | flattened | | aws.s3.object.key | Name of the S3 object that this log retrieved from. | keyword | -| cloud.account.id | The cloud account or organization id used to identify different entities in a multi-tenant environment. Examples: AWS account id, Google Cloud ORG Id, or other unique identifier. | keyword | -| cloud.availability_zone | Availability zone in which this host, resource, or service is located. | keyword | | cloud.image.id | Image ID for the cloud instance. | keyword | -| cloud.instance.id | Instance ID of the host machine. | keyword | -| cloud.instance.name | Instance name of the host machine. | keyword | -| cloud.machine.type | Machine type of the host machine. | keyword | -| cloud.project.id | The cloud project identifier. Examples: Google Cloud Project id, Azure Project id. | keyword | -| cloud.provider | Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. | keyword | -| cloud.region | Region in which this host, resource, or service is located. | keyword | -| container.id | Unique container id. | keyword | -| container.image.name | Name of the image the container was built on. | keyword | -| container.labels | Image labels. | object | -| container.name | Container name. | keyword | | data_stream.dataset | Data stream dataset. | constant_keyword | | data_stream.namespace | Data stream namespace. | constant_keyword | | data_stream.type | Data stream type. | constant_keyword | -| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | -| error.message | Error message. | match_only_text | -| event.dataset | Name of the dataset. If an event source publishes more than one type of log or events (e.g. access log, error log), the dataset is used to specify which one the event comes from. It's recommended but not required to start the dataset name with the module name, followed by a dot, then the dataset name. | constant_keyword | | event.module | Event module | constant_keyword | -| host.architecture | Operating system architecture. | keyword | | host.containerized | If the host is a container. | boolean | -| host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | -| host.hostname | Hostname of the host. It normally contains what the `hostname` command returns on the host machine. | keyword | -| host.id | Unique host id. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`. | keyword | -| host.ip | Host ip addresses. | ip | -| host.mac | Host MAC addresses. The notation format from RFC 7042 is suggested: Each octet (that is, 8-bit byte) is represented by two [uppercase] hexadecimal digits giving the value of the octet as an unsigned integer. Successive octets are separated by a hyphen. | keyword | -| host.name | Name of the host. It can contain what hostname returns on Unix systems, the fully qualified domain name (FQDN), or a name specified by the user. The recommended value is the lowercase FQDN of the host. | keyword | | host.os.build | OS build information. | keyword | | host.os.codename | OS codename, if any. | keyword | -| host.os.family | OS family (such as redhat, debian, freebsd, windows). | keyword | -| host.os.kernel | Operating system kernel version as a raw string. | keyword | -| host.os.name | Operating system name, without the version. | keyword | -| host.os.name.text | Multi-field of `host.os.name`. | match_only_text | -| host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword | -| host.os.version | Operating system version as a raw string. | keyword | -| host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | | input.type | Input type | keyword | -| log.file.path | Full path to the log file this event came from, including the file name. It should include the drive letter, when appropriate. If the event wasn't read from a log file, do not populate this field. | keyword | | log.offset | Log offset | long | -| message | For log events the message field contains the log message, optimized for viewing in a log viewer. For structured logs without an original message field, other fields can be concatenated to form a human-readable summary of the event. If multiple messages exist, they can be combined into one message. | match_only_text | -| tags | List of keywords used to tag each event. | keyword | diff --git a/packages/aws/docs/billing.md b/packages/aws/docs/billing.md index 1e8e96f55e2..698e4323978 100644 --- a/packages/aws/docs/billing.md +++ b/packages/aws/docs/billing.md @@ -54,31 +54,15 @@ An example event for `billing` looks as following: ```json { "@timestamp": "2020-05-28T17:17:06.212Z", - "cloud": { - "provider": "aws", - "region": "us-east-1", - "account": { - "id": "428152502467", - "name": "elastic-beats" - } - }, - "event": { - "dataset": "aws.billing", - "module": "aws", - "duration": 1938760247 - }, - "metricset": { - "name": "billing", - "period": 43200000 - }, - "ecs": { - "version": "1.5.0" + "agent": { + "ephemeral_id": "17803f33-b617-4ce9-a9ac-e218c02aeb4b", + "id": "12f376ef-5186-4e8b-a175-70f1140a8f30", + "name": "MacBook-Elastic.local", + "type": "metricbeat", + "version": "8.0.0" }, "aws": { "billing": { - "Currency": "USD", - "EstimatedCharges": 39.26, - "ServiceName": "AmazonEKS", "AmortizedCost": { "amount": 51.6, "unit": "USD" @@ -87,10 +71,13 @@ An example event for `billing` looks as following: "amount": 51.6, "unit": "USD" }, + "Currency": "USD", + "EstimatedCharges": 39.26, "NormalizedUsageAmount": { "amount": 672, "unit": "N/A" }, + "ServiceName": "AmazonEKS", "UnblendedCost": { "amount": 51.6, "unit": "USD" @@ -101,19 +88,36 @@ An example event for `billing` looks as following: } } }, + "cloud": { + "account": { + "id": "428152502467", + "name": "elastic-beats" + }, + "provider": "aws", + "region": "us-east-1" + }, + "ecs": { + "version": "8.11.0" + }, + "event": { + "dataset": "aws.billing", + "duration": 1938760247, + "module": "aws" + }, + "metricset": { + "name": "billing", + "period": 43200000 + }, "service": { "type": "aws" - }, - "agent": { - "id": "12f376ef-5186-4e8b-a175-70f1140a8f30", - "name": "MacBook-Elastic.local", - "type": "metricbeat", - "version": "8.0.0", - "ephemeral_id": "17803f33-b617-4ce9-a9ac-e218c02aeb4b" } } ``` +**ECS Field Reference** + +Please refer to the following [document](https://www.elastic.co/guide/en/ecs/current/ecs-field-reference.html) for detailed information on ECS fields. + **Exported fields** | Field | Description | Type | Metric Type | @@ -143,45 +147,13 @@ An example event for `billing` looks as following: | aws.linked_account.id | ID used to identify linked account. | keyword | | | aws.linked_account.name | Name or alias used to identify linked account. | keyword | | | aws.tags | Tag key value pairs from aws resources. | flattened | | -| cloud | Fields related to the cloud or infrastructure the events are coming from. | group | | | cloud.account.id | The cloud account or organization id used to identify different entities in a multi-tenant environment. Examples: AWS account id, Google Cloud ORG Id, or other unique identifier. | keyword | | -| cloud.account.name | The cloud account name or alias used to identify different entities in a multi-tenant environment. Examples: AWS account name, Google Cloud ORG display name. | keyword | | -| cloud.availability_zone | Availability zone in which this host, resource, or service is located. | keyword | | | cloud.image.id | Image ID for the cloud instance. | keyword | | -| cloud.instance.id | Instance ID of the host machine. | keyword | | -| cloud.instance.name | Instance name of the host machine. | keyword | | -| cloud.machine.type | Machine type of the host machine. | keyword | | -| cloud.project.id | The cloud project identifier. Examples: Google Cloud Project id, Azure Project id. | keyword | | -| cloud.provider | Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. | keyword | | -| cloud.region | Region in which this host, resource, or service is located. | keyword | | -| container.id | Unique container id. | keyword | | -| container.image.name | Name of the image the container was built on. | keyword | | -| container.labels | Image labels. | object | | -| container.name | Container name. | keyword | | | data_stream.dataset | Data stream dataset. | constant_keyword | | | data_stream.namespace | Data stream namespace. | constant_keyword | | | data_stream.type | Data stream type. | constant_keyword | | -| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | | -| error | These fields can represent errors of any kind. Use them for errors that happen while fetching events or in cases where the event itself contains an error. | group | | -| error.message | Error message. | match_only_text | | -| event.dataset | Name of the dataset. If an event source publishes more than one type of log or events (e.g. access log, error log), the dataset is used to specify which one the event comes from. It's recommended but not required to start the dataset name with the module name, followed by a dot, then the dataset name. | constant_keyword | | | event.module | Event module | constant_keyword | | -| host.architecture | Operating system architecture. | keyword | | | host.containerized | If the host is a container. | boolean | | -| host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | | -| host.hostname | Hostname of the host. It normally contains what the `hostname` command returns on the host machine. | keyword | | -| host.id | Unique host id. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`. | keyword | | -| host.ip | Host ip addresses. | ip | | -| host.mac | Host MAC addresses. The notation format from RFC 7042 is suggested: Each octet (that is, 8-bit byte) is represented by two [uppercase] hexadecimal digits giving the value of the octet as an unsigned integer. Successive octets are separated by a hyphen. | keyword | | -| host.name | Name of the host. It can contain what hostname returns on Unix systems, the fully qualified domain name (FQDN), or a name specified by the user. The recommended value is the lowercase FQDN of the host. | keyword | | | host.os.build | OS build information. | keyword | | | host.os.codename | OS codename, if any. | keyword | | -| host.os.family | OS family (such as redhat, debian, freebsd, windows). | keyword | | -| host.os.kernel | Operating system kernel version as a raw string. | keyword | | -| host.os.name | Operating system name, without the version. | keyword | | -| host.os.name.text | Multi-field of `host.os.name`. | match_only_text | | -| host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword | | -| host.os.version | Operating system version as a raw string. | keyword | | -| host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | | -| service.type | The type of the service data is collected from. The type can be used to group and correlate logs and metrics from one service type. Example: If logs or metrics are collected from Elasticsearch, `service.type` would be `elasticsearch`. | keyword | | diff --git a/packages/aws/docs/cloudfront.md b/packages/aws/docs/cloudfront.md index 671d63f33df..db54ce3f875 100644 --- a/packages/aws/docs/cloudfront.md +++ b/packages/aws/docs/cloudfront.md @@ -46,6 +46,10 @@ For step-by-step instructions on how to set up an integration, see the The `cloudfront` data stream collects standard logs (also called access logs) from AWS CloudFront. CloudFront standard logs provide detailed records about every request that’s made to a distribution. +**ECS Field Reference** + +Please refer to the following [document](https://www.elastic.co/guide/en/ecs/current/ecs-field-reference.html) for detailed information on ECS fields. + **Exported fields** | Field | Description | Type | @@ -62,101 +66,16 @@ CloudFront standard logs provide detailed records about every request that’s m | aws.s3.bucket.arn | The AWS S3 bucket ARN. | keyword | | aws.s3.bucket.name | The AWS S3 bucket name. | keyword | | aws.s3.object.key | The AWS S3 Object key. | keyword | -| cloud.account.id | The cloud account or organization id used to identify different entities in a multi-tenant environment. Examples: AWS account id, Google Cloud ORG Id, or other unique identifier. | keyword | -| cloud.availability_zone | Availability zone in which this host, resource, or service is located. | keyword | | cloud.image.id | Image ID for the cloud instance. | keyword | -| cloud.instance.id | Instance ID of the host machine. | keyword | -| cloud.instance.name | Instance name of the host machine. | keyword | -| cloud.machine.type | Machine type of the host machine. | keyword | -| cloud.project.id | The cloud project identifier. Examples: Google Cloud Project id, Azure Project id. | keyword | -| cloud.provider | Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. | keyword | -| cloud.region | Region in which this host, resource, or service is located. | keyword | -| container.id | Unique container id. | keyword | -| container.image.name | Name of the image the container was built on. | keyword | -| container.labels | Image labels. | object | -| container.name | Container name. | keyword | | data_stream.dataset | Data stream dataset. | constant_keyword | | data_stream.namespace | Data stream namespace. | constant_keyword | | data_stream.type | Data stream type. | constant_keyword | -| destination.address | Some event destination addresses are defined ambiguously. The event will sometimes list an IP, a domain or a unix socket. You should always store the raw address in the `.address` field. Then it should be duplicated to `.ip` or `.domain`, depending on which one it is. | keyword | -| destination.domain | The domain name of the destination system. This value may be a host name, a fully qualified domain name, or another host naming format. The value may derive from the original event or be added from enrichment. | keyword | -| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | -| error.message | Error message. | match_only_text | -| event.dataset | Name of the dataset. If an event source publishes more than one type of log or events (e.g. access log, error log), the dataset is used to specify which one the event comes from. It's recommended but not required to start the dataset name with the module name, followed by a dot, then the dataset name. | constant_keyword | -| event.duration | Duration of the event in nanoseconds. If `event.start` and `event.end` are known this value should be the difference between the end and start time. | long | | event.module | Event module | constant_keyword | -| host.architecture | Operating system architecture. | keyword | | host.containerized | If the host is a container. | boolean | -| host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | -| host.hostname | Hostname of the host. It normally contains what the `hostname` command returns on the host machine. | keyword | -| host.id | Unique host id. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`. | keyword | -| host.ip | Host ip addresses. | ip | -| host.mac | Host MAC addresses. The notation format from RFC 7042 is suggested: Each octet (that is, 8-bit byte) is represented by two [uppercase] hexadecimal digits giving the value of the octet as an unsigned integer. Successive octets are separated by a hyphen. | keyword | -| host.name | Name of the host. It can contain what hostname returns on Unix systems, the fully qualified domain name (FQDN), or a name specified by the user. The recommended value is the lowercase FQDN of the host. | keyword | | host.os.build | OS build information. | keyword | | host.os.codename | OS codename, if any. | keyword | -| host.os.family | OS family (such as redhat, debian, freebsd, windows). | keyword | -| host.os.kernel | Operating system kernel version as a raw string. | keyword | -| host.os.name | Operating system name, without the version. | keyword | -| host.os.name.text | Multi-field of `host.os.name`. | match_only_text | -| host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword | -| host.os.version | Operating system version as a raw string. | keyword | -| host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | -| http.request.bytes | Total size in bytes of the request (body and headers). | long | -| http.request.id | A unique identifier for each HTTP request to correlate logs between clients and servers in transactions. The id may be contained in a non-standard HTTP header, such as `X-Request-ID` or `X-Correlation-ID`. | keyword | -| http.request.method | HTTP request method. The value should retain its casing from the original event. For example, `GET`, `get`, and `GeT` are all considered valid values for this field. | keyword | -| http.request.referrer | Referrer for this HTTP request. | keyword | -| http.response.body.bytes | Size in bytes of the response body. | long | -| http.response.bytes | Total size in bytes of the response (body and headers). | long | -| http.response.status_code | HTTP response status code. | long | -| http.version | HTTP version. | keyword | | input.type | Input type | keyword | -| log.file.path | Full path to the log file this event came from, including the file name. It should include the drive letter, when appropriate. If the event wasn't read from a log file, do not populate this field. | keyword | | log.offset | Log offset | long | -| network.forwarded_ip | Host IP address when the source IP address is the proxy. | ip | -| network.protocol | In the OSI Model this would be the Application Layer protocol. For example, `http`, `dns`, or `ssh`. The field value must be normalized to lowercase for querying. | keyword | -| network.type | In the OSI Model this would be the Network Layer. ipv4, ipv6, ipsec, pim, etc The field value must be normalized to lowercase for querying. | keyword | -| related.hosts | All hostnames or other host identifiers seen on your event. Example identifiers include FQDNs, domain names, workstation names, or aliases. | keyword | -| related.ip | All of the IPs seen on your event. | ip | -| source.address | Some event source addresses are defined ambiguously. The event will sometimes list an IP, a domain or a unix socket. You should always store the raw address in the `.address` field. Then it should be duplicated to `.ip` or `.domain`, depending on which one it is. | keyword | -| source.as.number | Unique number allocated to the autonomous system. The autonomous system number (ASN) uniquely identifies each network on the Internet. | long | -| source.as.organization.name | Organization name. | keyword | -| source.as.organization.name.text | Multi-field of `source.as.organization.name`. | match_only_text | -| source.geo.city_name | City name. | keyword | -| source.geo.continent_name | Name of the continent. | keyword | -| source.geo.country_iso_code | Country ISO code. | keyword | -| source.geo.country_name | Country name. | keyword | -| source.geo.location | Longitude and latitude. | geo_point | -| source.geo.region_iso_code | Region ISO code. | keyword | -| source.geo.region_name | Region name. | keyword | -| source.ip | IP address of the source (IPv4 or IPv6). | ip | -| source.port | Port of the source. | long | -| tags | List of keywords used to tag each event. | keyword | -| tls.cipher | String indicating the cipher used during the current connection. | keyword | -| tls.version | Numeric part of the version parsed from the original string. | keyword | -| tls.version_protocol | Normalized lowercase protocol name parsed from original string. | keyword | -| url.domain | Domain of the url, such as "www.elastic.co". In some cases a URL may refer to an IP and/or port directly, without a domain name. In this case, the IP address would go to the `domain` field. If the URL contains a literal IPv6 address enclosed by `[` and `]` (IETF RFC 2732), the `[` and `]` characters should also be captured in the `domain` field. | keyword | -| url.extension | The field contains the file extension from the original request url, excluding the leading dot. The file extension is only set if it exists, as not every url has a file extension. The leading period must not be included. For example, the value must be "png", not ".png". Note that when the file name has multiple extensions (example.tar.gz), only the last one should be captured ("gz", not "tar.gz"). | keyword | -| url.full | If full URLs are important to your use case, they should be stored in `url.full`, whether this field is reconstructed or present in the event source. | wildcard | -| url.full.text | Multi-field of `url.full`. | match_only_text | -| url.original | Unmodified original url as seen in the event source. Note that in network monitoring, the observed URL may be a full URL, whereas in access logs, the URL is often just represented as a path. This field is meant to represent the URL as it was observed, complete or not. | wildcard | -| url.original.text | Multi-field of `url.original`. | match_only_text | -| url.path | Path of the request, such as "/search". | wildcard | -| url.query | The query field describes the query string of the request, such as "q=elasticsearch". The `?` is excluded from the query string. If a URL contains no `?`, there is no query field. If there is a `?` but no query, the query field exists with an empty string. The `exists` query can be used to differentiate between the two cases. | keyword | -| url.registered_domain | The highest registered url domain, stripped of the subdomain. For example, the registered domain for "foo.example.com" is "example.com". This value can be determined precisely with a list like the public suffix list (http://publicsuffix.org). Trying to approximate this by simply taking the last two labels will not work well for TLDs such as "co.uk". | keyword | -| url.scheme | Scheme of the request, such as "https". Note: The `:` is not part of the scheme. | keyword | -| url.subdomain | The subdomain portion of a fully qualified domain name includes all of the names except the host name under the registered_domain. In a partially qualified domain, or if the the qualification level of the full name cannot be determined, subdomain contains all of the names below the registered domain. For example the subdomain portion of "www.east.mydomain.co.uk" is "east". If the domain has multiple levels of subdomain, such as "sub2.sub1.example.com", the subdomain field should contain "sub2.sub1", with no trailing period. | keyword | -| url.top_level_domain | The effective top level domain (eTLD), also known as the domain suffix, is the last part of the domain name. For example, the top level domain for example.com is "com". This value can be determined precisely with a list like the public suffix list (http://publicsuffix.org). Trying to approximate this by simply taking the last label will not work well for effective TLDs such as "co.uk". | keyword | -| user_agent.device.name | Name of the device. | keyword | -| user_agent.name | Name of the user agent. | keyword | -| user_agent.original | Unparsed user_agent string. | keyword | -| user_agent.original.text | Multi-field of `user_agent.original`. | match_only_text | -| user_agent.os.full | Operating system name, including the version or code name. | keyword | -| user_agent.os.full.text | Multi-field of `user_agent.os.full`. | match_only_text | -| user_agent.os.name | Operating system name, without the version. | keyword | -| user_agent.os.name.text | Multi-field of `user_agent.os.name`. | match_only_text | -| user_agent.os.version | Operating system version as a raw string. | keyword | -| user_agent.version | Version of the user agent. | keyword | An example event for `cloudfront` looks as following: @@ -205,7 +124,7 @@ An example event for `cloudfront` looks as following: "domain": "d111111abcdef8.cloudfront.net" }, "ecs": { - "version": "8.0.0" + "version": "8.11.0" }, "elastic_agent": { "id": "acba78ef-1401-4689-977c-d8c2e5d6a8fa", @@ -214,7 +133,9 @@ An example event for `cloudfront` looks as following: }, "event": { "agent_id_status": "verified", - "category": "web", + "category": [ + "web" + ], "dataset": "aws.cloudfront_logs", "id": "SOX4xwn4XV6Q4rgb7XiVGOHms_BGlTAC4KyHmureZmBNrjGdRLiNIQ==", "ingested": "2023-11-03T13:01:05Z", diff --git a/packages/aws/docs/cloudtrail.md b/packages/aws/docs/cloudtrail.md index f84ec6f6a33..6f98e063986 100644 --- a/packages/aws/docs/cloudtrail.md +++ b/packages/aws/docs/cloudtrail.md @@ -68,6 +68,10 @@ files to a specific Amazon S3 bucket. of the CloudTrail Digest S3 Objects you'd like to read. If blank, CloudTrail Digest logs will be skipped. +**ECS Field Reference** + +Please refer to the following [document](https://www.elastic.co/guide/en/ecs/current/ecs-field-reference.html) for detailed information on ECS fields. + **Exported fields** | Field | Description | Type | @@ -119,98 +123,16 @@ If blank, CloudTrail Digest logs will be skipped. | aws.s3.bucket.name | Name of a S3 bucket. | keyword | | aws.s3.metadata | AWS S3 object metadata values. | flattened | | aws.s3.object.key | Name of the S3 object that this log retrieved from. | keyword | -| cloud.account.id | The cloud account or organization id used to identify different entities in a multi-tenant environment. Examples: AWS account id, Google Cloud ORG Id, or other unique identifier. | keyword | -| cloud.availability_zone | Availability zone in which this host, resource, or service is located. | keyword | | cloud.image.id | Image ID for the cloud instance. | keyword | -| cloud.instance.id | Instance ID of the host machine. | keyword | -| cloud.instance.name | Instance name of the host machine. | keyword | -| cloud.machine.type | Machine type of the host machine. | keyword | -| cloud.project.id | The cloud project identifier. Examples: Google Cloud Project id, Azure Project id. | keyword | -| cloud.provider | Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. | keyword | -| cloud.region | Region in which this host, resource, or service is located. | keyword | -| container.id | Unique container id. | keyword | -| container.image.name | Name of the image the container was built on. | keyword | -| container.labels | Image labels. | object | -| container.name | Container name. | keyword | | data_stream.dataset | Data stream dataset. | constant_keyword | | data_stream.namespace | Data stream namespace. | constant_keyword | | data_stream.type | Data stream type. | constant_keyword | -| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | -| error.message | Error message. | match_only_text | -| event.action | The action captured by the event. This describes the information in the event. It is more specific than `event.category`. Examples are `group-add`, `process-started`, `file-created`. The value is normally defined by the implementer. | keyword | -| event.created | `event.created` contains the date/time when the event was first read by an agent, or by your pipeline. This field is distinct from `@timestamp` in that `@timestamp` typically contain the time extracted from the original event. In most situations, these two timestamps will be slightly different. The difference can be used to calculate the delay between your source generating an event, and the time when your agent first processed it. This can be used to monitor your agent's or pipeline's ability to keep up with your event source. In case the two timestamps are identical, `@timestamp` should be used. | date | -| event.dataset | Name of the dataset. If an event source publishes more than one type of log or events (e.g. access log, error log), the dataset is used to specify which one the event comes from. It's recommended but not required to start the dataset name with the module name, followed by a dot, then the dataset name. | constant_keyword | -| event.ingested | Timestamp when an event arrived in the central data store. This is different from `@timestamp`, which is when the event originally occurred. It's also different from `event.created`, which is meant to capture the first time an agent saw the event. In normal conditions, assuming no tampering, the timestamps should chronologically look like this: `@timestamp` \< `event.created` \< `event.ingested`. | date | -| event.kind | This is one of four ECS Categorization Fields, and indicates the highest level in the ECS category hierarchy. `event.kind` gives high-level information about what type of information the event contains, without being specific to the contents of the event. For example, values of this field distinguish alert events from metric events. The value of this field can be used to inform how these kinds of events should be handled. They may warrant different retention, different access control, it may also help understand whether the data is coming in at a regular interval or not. | keyword | | event.module | Event module | constant_keyword | -| event.original | Raw text message of entire event. Used to demonstrate log integrity or where the full log message (before splitting it up in multiple parts) may be required, e.g. for reindex. This field is not indexed and doc_values are disabled. It cannot be searched, but it can be retrieved from `_source`. If users wish to override this and index this field, please see `Field data types` in the `Elasticsearch Reference`. | keyword | -| event.provider | Source of the event. Event transports such as Syslog or the Windows Event Log typically mention the source of an event. It can be the name of the software that generated the event (e.g. Sysmon, httpd), or of a subsystem of the operating system (kernel, Microsoft-Windows-Security-Auditing). | keyword | -| event.type | This is one of four ECS Categorization Fields, and indicates the third level in the ECS category hierarchy. `event.type` represents a categorization "sub-bucket" that, when used along with the `event.category` field values, enables filtering events down to a level appropriate for single visualization. This field is an array. This will allow proper categorization of some events that fall in multiple event types. | keyword | -| file.hash.md5 | MD5 hash. | keyword | -| file.hash.sha1 | SHA1 hash. | keyword | -| file.hash.sha256 | SHA256 hash. | keyword | -| file.hash.sha512 | SHA512 hash. | keyword | -| file.path | Full path to the file, including the file name. It should include the drive letter, when appropriate. | keyword | -| file.path.text | Multi-field of `file.path`. | match_only_text | -| group.id | Unique identifier for the group on the system/platform. | keyword | -| group.name | Name of the group. | keyword | -| host.architecture | Operating system architecture. | keyword | | host.containerized | If the host is a container. | boolean | -| host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | -| host.hostname | Hostname of the host. It normally contains what the `hostname` command returns on the host machine. | keyword | -| host.id | Unique host id. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`. | keyword | -| host.ip | Host ip addresses. | ip | -| host.mac | Host MAC addresses. The notation format from RFC 7042 is suggested: Each octet (that is, 8-bit byte) is represented by two [uppercase] hexadecimal digits giving the value of the octet as an unsigned integer. Successive octets are separated by a hyphen. | keyword | -| host.name | Name of the host. It can contain what hostname returns on Unix systems, the fully qualified domain name (FQDN), or a name specified by the user. The recommended value is the lowercase FQDN of the host. | keyword | | host.os.build | OS build information. | keyword | | host.os.codename | OS codename, if any. | keyword | -| host.os.family | OS family (such as redhat, debian, freebsd, windows). | keyword | -| host.os.kernel | Operating system kernel version as a raw string. | keyword | -| host.os.name | Operating system name, without the version. | keyword | -| host.os.name.text | Multi-field of `host.os.name`. | match_only_text | -| host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword | -| host.os.version | Operating system version as a raw string. | keyword | -| host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | | input.type | Input type | keyword | -| log.file.path | Full path to the log file this event came from, including the file name. It should include the drive letter, when appropriate. If the event wasn't read from a log file, do not populate this field. | keyword | | log.offset | Log offset | long | -| related.hash | All the hashes seen on your event. Populating this field, then using it to search for hashes can help in situations where you're unsure what the hash algorithm is (and therefore which key name to search). | keyword | -| related.user | All the user names or other user identifiers seen on the event. | keyword | -| source.address | Some event source addresses are defined ambiguously. The event will sometimes list an IP, a domain or a unix socket. You should always store the raw address in the `.address` field. Then it should be duplicated to `.ip` or `.domain`, depending on which one it is. | keyword | -| source.as.number | Unique number allocated to the autonomous system. The autonomous system number (ASN) uniquely identifies each network on the Internet. | long | -| source.as.organization.name | Organization name. | keyword | -| source.as.organization.name.text | Multi-field of `source.as.organization.name`. | match_only_text | -| source.geo.city_name | City name. | keyword | -| source.geo.continent_name | Name of the continent. | keyword | -| source.geo.country_iso_code | Country ISO code. | keyword | -| source.geo.country_name | Country name. | keyword | -| source.geo.location | Longitude and latitude. | geo_point | -| source.geo.region_iso_code | Region ISO code. | keyword | -| source.geo.region_name | Region name. | keyword | -| source.ip | IP address of the source (IPv4 or IPv6). | ip | -| tags | List of keywords used to tag each event. | keyword | -| tls.cipher | String indicating the cipher used during the current connection. | keyword | -| tls.client.server_name | Also called an SNI, this tells the server which hostname to which the client is attempting to connect to. When this value is available, it should get copied to `destination.domain`. | keyword | -| tls.version | Numeric part of the version parsed from the original string. | keyword | -| tls.version_protocol | Normalized lowercase protocol name parsed from original string. | keyword | -| user.changes.name | Short name or login of the user. | keyword | -| user.changes.name.text | Multi-field of `user.changes.name`. | match_only_text | -| user.id | Unique identifier of the user. | keyword | -| user.name | Short name or login of the user. | keyword | -| user.name.text | Multi-field of `user.name`. | match_only_text | -| user.target.id | Unique identifier of the user. | keyword | -| user.target.name | Short name or login of the user. | keyword | -| user.target.name.text | Multi-field of `user.target.name`. | match_only_text | -| user_agent.device.name | Name of the device. | keyword | -| user_agent.name | Name of the user agent. | keyword | -| user_agent.original | Unparsed user_agent string. | keyword | -| user_agent.original.text | Multi-field of `user_agent.original`. | match_only_text | -| user_agent.os.full | Operating system name, including the version or code name. | keyword | -| user_agent.os.full.text | Multi-field of `user_agent.os.full`. | match_only_text | -| user_agent.os.name | Operating system name, without the version. | keyword | -| user_agent.os.name.text | Multi-field of `user_agent.os.name`. | match_only_text | -| user_agent.os.version | Operating system version as a raw string. | keyword | -| user_agent.version | Version of the user agent. | keyword | An example event for `cloudtrail` looks as following: @@ -360,7 +282,7 @@ An example event for `cloudtrail` looks as following: "type": "logs" }, "ecs": { - "version": "8.0.0" + "version": "8.11.0" }, "elastic_agent": { "id": "acba78ef-1401-4689-977c-d8c2e5d6a8fa", diff --git a/packages/aws/docs/cloudwatch.md b/packages/aws/docs/cloudwatch.md index 9711ecc7200..9d20452aed1 100644 --- a/packages/aws/docs/cloudwatch.md +++ b/packages/aws/docs/cloudwatch.md @@ -63,52 +63,24 @@ The `number_of_workers` setting defines the number of workers assigned to readin The `cloudwatch` data stream collects CloudWatch logs. Users can use Amazon CloudWatch logs to monitor, store, and access log files from different sources. +**ECS Field Reference** + +Please refer to the following [document](https://www.elastic.co/guide/en/ecs/current/ecs-field-reference.html) for detailed information on ECS fields. + **Exported fields** | Field | Description | Type | |---|---|---| | @timestamp | Event timestamp. | date | | aws.cloudwatch.message | CloudWatch log message. | text | -| cloud.account.id | The cloud account or organization id used to identify different entities in a multi-tenant environment. Examples: AWS account id, Google Cloud ORG Id, or other unique identifier. | keyword | -| cloud.availability_zone | Availability zone in which this host, resource, or service is located. | keyword | | cloud.image.id | Image ID for the cloud instance. | keyword | -| cloud.instance.id | Instance ID of the host machine. | keyword | -| cloud.instance.name | Instance name of the host machine. | keyword | -| cloud.machine.type | Machine type of the host machine. | keyword | -| cloud.project.id | The cloud project identifier. Examples: Google Cloud Project id, Azure Project id. | keyword | -| cloud.provider | Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. | keyword | -| cloud.region | Region in which this host, resource, or service is located. | keyword | -| container.id | Unique container id. | keyword | -| container.image.name | Name of the image the container was built on. | keyword | -| container.labels | Image labels. | object | -| container.name | Container name. | keyword | | data_stream.dataset | Data stream dataset. | constant_keyword | | data_stream.namespace | Data stream namespace. | constant_keyword | | data_stream.type | Data stream type. | constant_keyword | -| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | -| error.message | Error message. | match_only_text | -| event.dataset | Name of the dataset. If an event source publishes more than one type of log or events (e.g. access log, error log), the dataset is used to specify which one the event comes from. It's recommended but not required to start the dataset name with the module name, followed by a dot, then the dataset name. | keyword | -| event.ingested | Timestamp when an event arrived in the central data store. This is different from `@timestamp`, which is when the event originally occurred. It's also different from `event.created`, which is meant to capture the first time an agent saw the event. In normal conditions, assuming no tampering, the timestamps should chronologically look like this: `@timestamp` \< `event.created` \< `event.ingested`. | date | | event.module | Event module | constant_keyword | -| host.architecture | Operating system architecture. | keyword | | host.containerized | If the host is a container. | boolean | -| host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | -| host.hostname | Hostname of the host. It normally contains what the `hostname` command returns on the host machine. | keyword | -| host.id | Unique host id. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`. | keyword | -| host.ip | Host ip addresses. | ip | -| host.mac | Host MAC addresses. The notation format from RFC 7042 is suggested: Each octet (that is, 8-bit byte) is represented by two [uppercase] hexadecimal digits giving the value of the octet as an unsigned integer. Successive octets are separated by a hyphen. | keyword | -| host.name | Name of the host. It can contain what hostname returns on Unix systems, the fully qualified domain name (FQDN), or a name specified by the user. The recommended value is the lowercase FQDN of the host. | keyword | | host.os.build | OS build information. | keyword | | host.os.codename | OS codename, if any. | keyword | -| host.os.family | OS family (such as redhat, debian, freebsd, windows). | keyword | -| host.os.kernel | Operating system kernel version as a raw string. | keyword | -| host.os.name | Operating system name, without the version. | keyword | -| host.os.name.text | Multi-field of `host.os.name`. | match_only_text | -| host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword | -| host.os.version | Operating system version as a raw string. | keyword | -| host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | -| message | For log events the message field contains the log message, optimized for viewing in a log viewer. For structured logs without an original message field, other fields can be concatenated to form a human-readable summary of the event. If multiple messages exist, they can be combined into one message. | match_only_text | -| tags | List of keywords used to tag each event. | keyword | An example event for `cloudwatch` looks as following: @@ -116,23 +88,23 @@ An example event for `cloudwatch` looks as following: ```json { "@timestamp": "2020-02-20T07:02:37.000Z", + "aws": { + "cloudwatch": { + "message": "ip-172-31-81-156 ec2net: [get_meta] Trying to get http://169.254.169.254/latest/meta-data/network/interfaces/macs/12:e2:a9:95:8b:97/local-ipv4s" + } + }, "data_stream": { + "dataset": "aws.cloudwatch_logs", "namespace": "default", - "type": "logs", - "dataset": "aws.cloudwatch_logs" + "type": "logs" }, "ecs": { - "version": "8.0.0" + "version": "8.11.0" }, "event": { "ingested": "2021-07-19T21:47:04.696803300Z", "original": "2020-02-20T07:02:37.000Z Feb 20 07:02:37 ip-172-31-81-156 ec2net: [get_meta] Trying to get http://169.254.169.254/latest/meta-data/network/interfaces/macs/12:e2:a9:95:8b:97/local-ipv4s" }, - "aws": { - "cloudwatch": { - "message": "ip-172-31-81-156 ec2net: [get_meta] Trying to get http://169.254.169.254/latest/meta-data/network/interfaces/macs/12:e2:a9:95:8b:97/local-ipv4s" - } - }, "tags": [ "preserve_original_event" ] @@ -146,14 +118,6 @@ An example event for `cloudwatch` looks as following: ```json { "@timestamp": "2020-05-28T17:17:02.812Z", - "event": { - "duration": 14119105951, - "dataset": "aws.cloudwatch_metrics", - "module": "aws" - }, - "ecs": { - "version": "1.5.0" - }, "agent": { "ephemeral_id": "17803f33-b617-4ce9-a9ac-e218c02aeb4b", "id": "12f376ef-5186-4e8b-a175-70f1140a8f30", @@ -161,44 +125,56 @@ An example event for `cloudwatch` looks as following: "type": "metricbeat", "version": "8.0.0" }, - "service": { - "type": "aws" - }, - "cloud": { - "provider": "aws", - "region": "us-west-2", - "account": { - "name": "elastic-beats", - "id": "428152502467" - } - }, "aws": { + "cloudwatch": { + "namespace": "AWS/EC2" + }, "dimensions": { "InstanceId": "i-0830bfecfa7173cbe" }, "ec2": { "metrics": { - "DiskWriteOps": { - "avg": 0, - "max": 0 - }, "CPUUtilization": { "avg": 0.7661943132361363, "max": 0.833333333333333 + }, + "DiskWriteOps": { + "avg": 0, + "max": 0 } } - }, - "cloudwatch": { - "namespace": "AWS/EC2" } }, + "cloud": { + "account": { + "id": "428152502467", + "name": "elastic-beats" + }, + "provider": "aws", + "region": "us-west-2" + }, + "ecs": { + "version": "8.11.0" + }, + "event": { + "dataset": "aws.cloudwatch_metrics", + "duration": 14119105951, + "module": "aws" + }, "metricset": { - "period": 300000, - "name": "cloudwatch" + "name": "cloudwatch", + "period": 300000 + }, + "service": { + "type": "aws" } } ``` +**ECS Field Reference** + +Please refer to the following [document](https://www.elastic.co/guide/en/ecs/current/ecs-field-reference.html) for detailed information on ECS fields. + **Exported fields** | Field | Description | Type | Metric Type | @@ -210,45 +186,14 @@ An example event for `cloudwatch` looks as following: | aws.dimensions | Metric dimensions. | flattened | | | aws.dimensions_fingerprint | Autogenerated ID representing the fingerprint of the aws.dimensions object | keyword | | | aws.tags | Tag key value pairs from aws resources. | flattened | | -| cloud | Fields related to the cloud or infrastructure the events are coming from. | group | | | cloud.account.id | The cloud account or organization id used to identify different entities in a multi-tenant environment. Examples: AWS account id, Google Cloud ORG Id, or other unique identifier. | keyword | | -| cloud.account.name | The cloud account name or alias used to identify different entities in a multi-tenant environment. Examples: AWS account name, Google Cloud ORG display name. | keyword | | -| cloud.availability_zone | Availability zone in which this host, resource, or service is located. | keyword | | | cloud.image.id | Image ID for the cloud instance. | keyword | | -| cloud.instance.id | Instance ID of the host machine. | keyword | | -| cloud.instance.name | Instance name of the host machine. | keyword | | -| cloud.machine.type | Machine type of the host machine. | keyword | | -| cloud.project.id | The cloud project identifier. Examples: Google Cloud Project id, Azure Project id. | keyword | | -| cloud.provider | Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. | keyword | | | cloud.region | Region in which this host, resource, or service is located. | keyword | | -| container.id | Unique container id. | keyword | | -| container.image.name | Name of the image the container was built on. | keyword | | -| container.labels | Image labels. | object | | -| container.name | Container name. | keyword | | | data_stream.dataset | Data stream dataset. | constant_keyword | | | data_stream.namespace | Data stream namespace. | constant_keyword | | | data_stream.type | Data stream type. | constant_keyword | | -| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | | -| error | These fields can represent errors of any kind. Use them for errors that happen while fetching events or in cases where the event itself contains an error. | group | | -| error.message | Error message. | match_only_text | | -| event.dataset | Name of the dataset. If an event source publishes more than one type of log or events (e.g. access log, error log), the dataset is used to specify which one the event comes from. It's recommended but not required to start the dataset name with the module name, followed by a dot, then the dataset name. | constant_keyword | | | event.module | Event module | constant_keyword | | -| host.architecture | Operating system architecture. | keyword | | | host.containerized | If the host is a container. | boolean | | -| host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | | -| host.hostname | Hostname of the host. It normally contains what the `hostname` command returns on the host machine. | keyword | | -| host.id | Unique host id. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`. | keyword | | -| host.ip | Host ip addresses. | ip | | -| host.mac | Host MAC addresses. The notation format from RFC 7042 is suggested: Each octet (that is, 8-bit byte) is represented by two [uppercase] hexadecimal digits giving the value of the octet as an unsigned integer. Successive octets are separated by a hyphen. | keyword | | -| host.name | Name of the host. It can contain what hostname returns on Unix systems, the fully qualified domain name (FQDN), or a name specified by the user. The recommended value is the lowercase FQDN of the host. | keyword | | | host.os.build | OS build information. | keyword | | | host.os.codename | OS codename, if any. | keyword | | -| host.os.family | OS family (such as redhat, debian, freebsd, windows). | keyword | | -| host.os.kernel | Operating system kernel version as a raw string. | keyword | | -| host.os.name | Operating system name, without the version. | keyword | | -| host.os.name.text | Multi-field of `host.os.name`. | match_only_text | | -| host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword | | -| host.os.version | Operating system version as a raw string. | keyword | | -| host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | | -| service.type | The type of the service data is collected from. The type can be used to group and correlate logs and metrics from one service type. Example: If logs or metrics are collected from Elasticsearch, `service.type` would be `elasticsearch`. | keyword | | diff --git a/packages/aws/docs/dynamodb.md b/packages/aws/docs/dynamodb.md index bcba57aa97a..997e0b95ab0 100644 --- a/packages/aws/docs/dynamodb.md +++ b/packages/aws/docs/dynamodb.md @@ -49,83 +49,87 @@ An example event for `dynamodb` looks as following: { "@timestamp": "2022-07-25T21:53:00.000Z", "agent": { - "name": "docker-fleet-agent", + "ephemeral_id": "64a12b83-a4f1-487c-8d2c-9581fda6ca2a", "id": "2d4b09d0-cdb6-445e-ac3f-6415f87b9864", + "name": "docker-fleet-agent", "type": "metricbeat", - "ephemeral_id": "64a12b83-a4f1-487c-8d2c-9581fda6ca2a", "version": "8.3.2" }, - "elastic_agent": { - "id": "2d4b09d0-cdb6-445e-ac3f-6415f87b9864", - "version": "8.3.2", - "snapshot": false - }, - "cloud": { - "provider": "aws", - "region": "eu-central-1", - "account": { - "name": "elastic-beats", - "id": "428152502467" - } - }, - "ecs": { - "version": "8.0.0" - }, - "service": { - "type": "aws" - }, - "data_stream": { - "namespace": "default", - "type": "metrics", - "dataset": "aws.dynamodb" - }, - "metricset": { - "period": 300000, - "name": "cloudwatch" - }, - "event": { - "duration": 10586366300, - "agent_id_status": "verified", - "ingested": "2022-07-25T21:57:51Z", - "module": "aws", - "dataset": "aws.dynamodb" - }, "aws": { "cloudwatch": { "namespace": "AWS/DynamoDB" }, "dynamodb": { "metrics": { - "AccountProvisionedWriteCapacityUtilization": { - "avg": 0.01 - }, - "MaxProvisionedTableWriteCapacityUtilization": { - "max": 0.01 - }, - "MaxProvisionedTableReadCapacityUtilization": { - "max": 0.01 + "AccountMaxReads": { + "max": 80000 }, "AccountMaxTableLevelReads": { "max": 40000 }, - "AccountMaxReads": { + "AccountMaxTableLevelWrites": { + "max": 40000 + }, + "AccountMaxWrites": { "max": 80000 }, "AccountProvisionedReadCapacityUtilization": { "avg": 0.01 }, - "AccountMaxWrites": { - "max": 80000 + "AccountProvisionedWriteCapacityUtilization": { + "avg": 0.01 }, - "AccountMaxTableLevelWrites": { - "max": 40000 + "MaxProvisionedTableReadCapacityUtilization": { + "max": 0.01 + }, + "MaxProvisionedTableWriteCapacityUtilization": { + "max": 0.01 } } } + }, + "cloud": { + "account": { + "id": "428152502467", + "name": "elastic-beats" + }, + "provider": "aws", + "region": "eu-central-1" + }, + "data_stream": { + "dataset": "aws.dynamodb", + "namespace": "default", + "type": "metrics" + }, + "ecs": { + "version": "8.11.0" + }, + "elastic_agent": { + "id": "2d4b09d0-cdb6-445e-ac3f-6415f87b9864", + "snapshot": false, + "version": "8.3.2" + }, + "event": { + "agent_id_status": "verified", + "dataset": "aws.dynamodb", + "duration": 10586366300, + "ingested": "2022-07-25T21:57:51Z", + "module": "aws" + }, + "metricset": { + "name": "cloudwatch", + "period": 300000 + }, + "service": { + "type": "aws" } } ``` +**ECS Field Reference** + +Please refer to the following [document](https://www.elastic.co/guide/en/ecs/current/ecs-field-reference.html) for detailed information on ECS fields. + **Exported fields** | Field | Description | Type | Metric Type | @@ -169,44 +173,13 @@ An example event for `dynamodb` looks as following: | aws.dynamodb.metrics.TransactionConflict.sum | | long | gauge | | aws.dynamodb.metrics.WriteThrottleEvents.sum | Requests to DynamoDB that exceed the provisioned write capacity units for a table or a global secondary index. | long | gauge | | aws.tags | Tag key value pairs from aws resources. | flattened | | -| cloud | Fields related to the cloud or infrastructure the events are coming from. | group | | | cloud.account.id | The cloud account or organization id used to identify different entities in a multi-tenant environment. Examples: AWS account id, Google Cloud ORG Id, or other unique identifier. | keyword | | -| cloud.account.name | The cloud account name or alias used to identify different entities in a multi-tenant environment. Examples: AWS account name, Google Cloud ORG display name. | keyword | | -| cloud.availability_zone | Availability zone in which this host, resource, or service is located. | keyword | | | cloud.image.id | Image ID for the cloud instance. | keyword | | -| cloud.instance.id | Instance ID of the host machine. | keyword | | -| cloud.instance.name | Instance name of the host machine. | keyword | | -| cloud.machine.type | Machine type of the host machine. | keyword | | -| cloud.project.id | The cloud project identifier. Examples: Google Cloud Project id, Azure Project id. | keyword | | -| cloud.provider | Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. | keyword | | | cloud.region | Region in which this host, resource, or service is located. | keyword | | -| container.id | Unique container id. | keyword | | -| container.image.name | Name of the image the container was built on. | keyword | | -| container.labels | Image labels. | object | | -| container.name | Container name. | keyword | | | data_stream.dataset | Data stream dataset. | constant_keyword | | | data_stream.namespace | Data stream namespace. | constant_keyword | | | data_stream.type | Data stream type. | constant_keyword | | -| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | | -| error | These fields can represent errors of any kind. Use them for errors that happen while fetching events or in cases where the event itself contains an error. | group | | -| error.message | Error message. | match_only_text | | -| event.dataset | Name of the dataset. If an event source publishes more than one type of log or events (e.g. access log, error log), the dataset is used to specify which one the event comes from. It's recommended but not required to start the dataset name with the module name, followed by a dot, then the dataset name. | constant_keyword | | | event.module | Event module | constant_keyword | | -| host.architecture | Operating system architecture. | keyword | | | host.containerized | If the host is a container. | boolean | | -| host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | | -| host.hostname | Hostname of the host. It normally contains what the `hostname` command returns on the host machine. | keyword | | -| host.id | Unique host id. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`. | keyword | | -| host.ip | Host ip addresses. | ip | | -| host.mac | Host MAC addresses. The notation format from RFC 7042 is suggested: Each octet (that is, 8-bit byte) is represented by two [uppercase] hexadecimal digits giving the value of the octet as an unsigned integer. Successive octets are separated by a hyphen. | keyword | | -| host.name | Name of the host. It can contain what hostname returns on Unix systems, the fully qualified domain name (FQDN), or a name specified by the user. The recommended value is the lowercase FQDN of the host. | keyword | | | host.os.build | OS build information. | keyword | | | host.os.codename | OS codename, if any. | keyword | | -| host.os.family | OS family (such as redhat, debian, freebsd, windows). | keyword | | -| host.os.kernel | Operating system kernel version as a raw string. | keyword | | -| host.os.name | Operating system name, without the version. | keyword | | -| host.os.name.text | Multi-field of `host.os.name`. | match_only_text | | -| host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword | | -| host.os.version | Operating system version as a raw string. | keyword | | -| host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | | -| service.type | The type of the service data is collected from. The type can be used to group and correlate logs and metrics from one service type. Example: If logs or metrics are collected from Elasticsearch, `service.type` would be `elasticsearch`. | keyword | | diff --git a/packages/aws/docs/ebs.md b/packages/aws/docs/ebs.md index 3ff173a1151..6c230ca6717 100644 --- a/packages/aws/docs/ebs.md +++ b/packages/aws/docs/ebs.md @@ -47,67 +47,34 @@ An example event for `ebs` looks as following: ```json { + "@timestamp": "2022-08-03T12:21:00.000Z", "agent": { - "name": "docker-fleet-agent", - "id": "618e6f72-9eef-4992-b60e-12515d538189", "ephemeral_id": "2e8fed31-76b5-4efe-9893-947fd2346abd", - "type": "metricbeat", - "version": "8.2.0" - }, - "elastic_agent": { "id": "618e6f72-9eef-4992-b60e-12515d538189", - "version": "8.2.0", - "snapshot": false - }, - "cloud": { - "provider": "aws", - "region": "us-east-2" - }, - "@timestamp": "2022-08-03T12:21:00.000Z", - "ecs": { - "version": "8.0.0" - }, - "data_stream": { - "namespace": "default", - "type": "metrics", - "dataset": "aws.ebs" - }, - "service": { - "type": "aws" - }, - "host": { - "hostname": "docker-fleet-agent", - "os": { - "kernel": "5.18.11-200.fc36.x86_64", - "codename": "focal", - "name": "Ubuntu", - "type": "linux", - "family": "debian", - "version": "20.04.4 LTS (Focal Fossa)", - "platform": "ubuntu" - }, - "containerized": false, - "ip": [ - "172.18.0.7" - ], "name": "docker-fleet-agent", - "mac": [ - "02-42-AC-12-00-07" - ], - "architecture": "x86_64" - }, - "metricset": { - "period": 300000, - "name": "cloudwatch" + "type": "metricbeat", + "version": "8.2.0" }, "aws": { + "cloudwatch": { + "namespace": "AWS/EBS" + }, + "dimensions": { + "VolumeId": "vol-015d88f45122510a5" + }, "ebs": { "metrics": { + "BurstBalance": { + "avg": 100 + }, + "VolumeIdleTime": { + "sum": 239.87 + }, "VolumeQueueLength": { "avg": 0 }, - "BurstBalance": { - "avg": 100 + "VolumeReadOps": { + "avg": 0 }, "VolumeTotalWriteTime": { "sum": 0.062 @@ -117,32 +84,69 @@ An example event for `ebs` looks as following: }, "VolumeWriteOps": { "avg": 23 - }, - "VolumeReadOps": { - "avg": 0 - }, - "VolumeIdleTime": { - "sum": 239.87 } } - }, - "cloudwatch": { - "namespace": "AWS/EBS" - }, - "dimensions": { - "VolumeId": "vol-015d88f45122510a5" } }, + "cloud": { + "provider": "aws", + "region": "us-east-2" + }, + "data_stream": { + "dataset": "aws.ebs", + "namespace": "default", + "type": "metrics" + }, + "ecs": { + "version": "8.11.0" + }, + "elastic_agent": { + "id": "618e6f72-9eef-4992-b60e-12515d538189", + "snapshot": false, + "version": "8.2.0" + }, "event": { - "duration": 1320126957, "agent_id_status": "verified", + "dataset": "aws.ebs", + "duration": 1320126957, "ingested": "2022-08-03T12:25:46Z", - "module": "aws", - "dataset": "aws.ebs" + "module": "aws" + }, + "host": { + "architecture": "x86_64", + "containerized": false, + "hostname": "docker-fleet-agent", + "ip": [ + "172.18.0.7" + ], + "mac": [ + "02-42-AC-12-00-07" + ], + "name": "docker-fleet-agent", + "os": { + "codename": "focal", + "family": "debian", + "kernel": "5.18.11-200.fc36.x86_64", + "name": "Ubuntu", + "platform": "ubuntu", + "type": "linux", + "version": "20.04.4 LTS (Focal Fossa)" + } + }, + "metricset": { + "name": "cloudwatch", + "period": 300000 + }, + "service": { + "type": "aws" } } ``` +**ECS Field Reference** + +Please refer to the following [document](https://www.elastic.co/guide/en/ecs/current/ecs-field-reference.html) for detailed information on ECS fields. + **Exported fields** | Field | Description | Type | Metric Type | @@ -164,44 +168,13 @@ An example event for `ebs` looks as following: | aws.ebs.metrics.VolumeWriteOps.avg | The total number of write operations in a specified period of time. | double | gauge | | aws.s3.bucket.name | Name of a S3 bucket. | keyword | | | aws.tags | Tag key value pairs from aws resources. | flattened | | -| cloud | Fields related to the cloud or infrastructure the events are coming from. | group | | | cloud.account.id | The cloud account or organization id used to identify different entities in a multi-tenant environment. Examples: AWS account id, Google Cloud ORG Id, or other unique identifier. | keyword | | -| cloud.account.name | The cloud account name or alias used to identify different entities in a multi-tenant environment. Examples: AWS account name, Google Cloud ORG display name. | keyword | | -| cloud.availability_zone | Availability zone in which this host, resource, or service is located. | keyword | | | cloud.image.id | Image ID for the cloud instance. | keyword | | -| cloud.instance.id | Instance ID of the host machine. | keyword | | -| cloud.instance.name | Instance name of the host machine. | keyword | | -| cloud.machine.type | Machine type of the host machine. | keyword | | -| cloud.project.id | The cloud project identifier. Examples: Google Cloud Project id, Azure Project id. | keyword | | -| cloud.provider | Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. | keyword | | | cloud.region | Region in which this host, resource, or service is located. | keyword | | -| container.id | Unique container id. | keyword | | -| container.image.name | Name of the image the container was built on. | keyword | | -| container.labels | Image labels. | object | | -| container.name | Container name. | keyword | | | data_stream.dataset | Data stream dataset. | constant_keyword | | | data_stream.namespace | Data stream namespace. | constant_keyword | | | data_stream.type | Data stream type. | constant_keyword | | -| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | | -| error | These fields can represent errors of any kind. Use them for errors that happen while fetching events or in cases where the event itself contains an error. | group | | -| error.message | Error message. | match_only_text | | -| event.dataset | Name of the dataset. If an event source publishes more than one type of log or events (e.g. access log, error log), the dataset is used to specify which one the event comes from. It's recommended but not required to start the dataset name with the module name, followed by a dot, then the dataset name. | constant_keyword | | | event.module | Event module | constant_keyword | | -| host.architecture | Operating system architecture. | keyword | | | host.containerized | If the host is a container. | boolean | | -| host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | | -| host.hostname | Hostname of the host. It normally contains what the `hostname` command returns on the host machine. | keyword | | -| host.id | Unique host id. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`. | keyword | | -| host.ip | Host ip addresses. | ip | | -| host.mac | Host MAC addresses. The notation format from RFC 7042 is suggested: Each octet (that is, 8-bit byte) is represented by two [uppercase] hexadecimal digits giving the value of the octet as an unsigned integer. Successive octets are separated by a hyphen. | keyword | | -| host.name | Name of the host. It can contain what hostname returns on Unix systems, the fully qualified domain name (FQDN), or a name specified by the user. The recommended value is the lowercase FQDN of the host. | keyword | | | host.os.build | OS build information. | keyword | | | host.os.codename | OS codename, if any. | keyword | | -| host.os.family | OS family (such as redhat, debian, freebsd, windows). | keyword | | -| host.os.kernel | Operating system kernel version as a raw string. | keyword | | -| host.os.name | Operating system name, without the version. | keyword | | -| host.os.name.text | Multi-field of `host.os.name`. | match_only_text | | -| host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword | | -| host.os.version | Operating system version as a raw string. | keyword | | -| host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | | -| service.type | The type of the service data is collected from. The type can be used to group and correlate logs and metrics from one service type. Example: If logs or metrics are collected from Elasticsearch, `service.type` would be `elasticsearch`. | keyword | | diff --git a/packages/aws/docs/ec2.md b/packages/aws/docs/ec2.md index 19c654bdfad..766eab4510c 100644 --- a/packages/aws/docs/ec2.md +++ b/packages/aws/docs/ec2.md @@ -66,6 +66,10 @@ For logs stored in S3, you must export logs from log groups to an Amazon S3 buck With this data stream, EC2 logs will be parsed into fields like `ip_address` and `process.name`. For logs from other services, please use the **AWS CloudWatch** integration. +**ECS Field Reference** + +Please refer to the following [document](https://www.elastic.co/guide/en/ecs/current/ecs-field-reference.html) for detailed information on ECS fields. + **Exported fields** | Field | Description | Type | @@ -75,49 +79,16 @@ and `process.name`. For logs from other services, please use the **AWS CloudWatc | aws.s3.bucket.arn | The AWS S3 bucket ARN. | keyword | | aws.s3.bucket.name | The AWS S3 bucket name. | keyword | | aws.s3.object.key | The AWS S3 Object key. | keyword | -| cloud.account.id | The cloud account or organization id used to identify different entities in a multi-tenant environment. Examples: AWS account id, Google Cloud ORG Id, or other unique identifier. | keyword | -| cloud.availability_zone | Availability zone in which this host, resource, or service is located. | keyword | | cloud.image.id | Image ID for the cloud instance. | keyword | -| cloud.instance.id | Instance ID of the host machine. | keyword | -| cloud.instance.name | Instance name of the host machine. | keyword | -| cloud.machine.type | Machine type of the host machine. | keyword | -| cloud.project.id | The cloud project identifier. Examples: Google Cloud Project id, Azure Project id. | keyword | -| cloud.provider | Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. | keyword | -| cloud.region | Region in which this host, resource, or service is located. | keyword | -| container.id | Unique container id. | keyword | -| container.image.name | Name of the image the container was built on. | keyword | -| container.labels | Image labels. | object | -| container.name | Container name. | keyword | | data_stream.dataset | Data stream dataset. | constant_keyword | | data_stream.namespace | Data stream namespace. | constant_keyword | | data_stream.type | Data stream type. | constant_keyword | -| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | -| error.message | Error message. | match_only_text | -| event.dataset | Name of the dataset. If an event source publishes more than one type of log or events (e.g. access log, error log), the dataset is used to specify which one the event comes from. It's recommended but not required to start the dataset name with the module name, followed by a dot, then the dataset name. | constant_keyword | | event.module | Event module | constant_keyword | -| host.architecture | Operating system architecture. | keyword | | host.containerized | If the host is a container. | boolean | -| host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | -| host.hostname | Hostname of the host. It normally contains what the `hostname` command returns on the host machine. | keyword | -| host.id | Unique host id. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`. | keyword | -| host.ip | Host ip addresses. | ip | -| host.mac | Host MAC addresses. The notation format from RFC 7042 is suggested: Each octet (that is, 8-bit byte) is represented by two [uppercase] hexadecimal digits giving the value of the octet as an unsigned integer. Successive octets are separated by a hyphen. | keyword | -| host.name | Name of the host. It can contain what hostname returns on Unix systems, the fully qualified domain name (FQDN), or a name specified by the user. The recommended value is the lowercase FQDN of the host. | keyword | | host.os.build | OS build information. | keyword | | host.os.codename | OS codename, if any. | keyword | -| host.os.family | OS family (such as redhat, debian, freebsd, windows). | keyword | -| host.os.kernel | Operating system kernel version as a raw string. | keyword | -| host.os.name | Operating system name, without the version. | keyword | -| host.os.name.text | Multi-field of `host.os.name`. | match_only_text | -| host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword | -| host.os.version | Operating system version as a raw string. | keyword | -| host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | | input.type | Input type | keyword | -| log.file.path | Full path to the log file this event came from, including the file name. It should include the drive letter, when appropriate. If the event wasn't read from a log file, do not populate this field. | keyword | | log.offset | Log offset | long | -| message | For log events the message field contains the log message, optimized for viewing in a log viewer. For structured logs without an original message field, other fields can be concatenated to form a human-readable summary of the event. If multiple messages exist, they can be combined into one message. | match_only_text | -| process.name | Process name. | keyword | -| tags | List of keywords used to tag each event. | keyword | An example event for `ec2` looks as following: @@ -156,7 +127,7 @@ An example event for `ec2` looks as following: "type": "logs" }, "ecs": { - "version": "8.0.0" + "version": "8.11.0" }, "elastic_agent": { "id": "acba78ef-1401-4689-977c-d8c2e5d6a8fa", @@ -197,152 +168,81 @@ An example event for `ec2` looks as following: ```json { "@timestamp": "2023-08-07T18:35:00.000Z", - "cloud": { - "availability_zone": "eu-north-1c", - "instance": { - "id": "i-0c08512debca266ab" - }, - "provider": "aws", - "machine": { - "type": "t3.medium" - }, - "region": "eu-north-1", - "account": { - "name": "MonitoringAccount", - "id": "627286350134" - } - }, "agent": { - "name": "docker-fleet-agent", + "ephemeral_id": "b8cd4414-f528-43f4-b43f-0edbcc69b46f", "id": "72314f01-98f2-477f-978a-e98d109c640c", + "name": "docker-fleet-agent", "type": "metricbeat", - "ephemeral_id": "b8cd4414-f528-43f4-b43f-0edbcc69b46f", "version": "8.8.1" }, - "ecs": { - "version": "8.0.0" - }, - "service": { - "type": "aws" - }, - "data_stream": { - "namespace": "default", - "type": "metrics", - "dataset": "aws.ec2_metrics" - }, - "host": { - "hostname": "docker-fleet-agent", - "os": { - "kernel": "5.15.49-linuxkit-pr", - "codename": "focal", - "name": "Ubuntu", - "type": "linux", - "family": "debian", - "version": "20.04.6 LTS (Focal Fossa)", - "platform": "ubuntu" + "aws": { + "cloudwatch": { + "namespace": "AWS/EC2" }, - "containerized": false, - "ip": [ - "172.20.0.7" - ], - "name": "docker-fleet-agent", - "cpu": { - "usage": 2.8849988898518673 + "dimensions": { + "InstanceId": "i-0c08512debca266ab" }, - "id": "d08b346fbb8f49f5a2bb1a477f8ceb54", - "mac": [ - "02-42-AC-14-00-07" - ], - "architecture": "aarch64", - "network": { - "ingress": { - "bytes": 1608959, - "packets": 5334 - }, - "egress": { - "bytes": 626755, - "packets": 4977 - } - } - }, - "elastic_agent": { - "id": "72314f01-98f2-477f-978a-e98d109c640c", - "version": "8.8.1", - "snapshot": false - }, - "metricset": { - "period": 300000, - "name": "cloudwatch" - }, - "event": { - "duration": 5858967919, - "agent_id_status": "verified", - "ingested": "2023-08-07T18:41:31Z", - "module": "aws", - "dataset": "aws.ec2_metrics" - }, - "aws": { "ec2": { "instance": { + "core": { + "count": 1 + }, "image": { "id": "ami-00b8290583a865359" }, - "core": { - "count": 1 + "monitoring": { + "state": "disabled" }, "private": { - "ip": "172.31.13.154", - "dns_name": "ip-172-31-13-154.eu-north-1.compute.internal" + "dns_name": "ip-172-31-13-154.eu-north-1.compute.internal", + "ip": "172.31.13.154" }, - "threads_per_core": 2, "public": { - "ip": "16.16.138.5", - "dns_name": "ec2-16-16-138-5.eu-north-1.compute.amazonaws.com" + "dns_name": "ec2-16-16-138-5.eu-north-1.compute.amazonaws.com", + "ip": "16.16.138.5" }, "state": { "code": 16, "name": "running" }, - "monitoring": { - "state": "disabled" - } + "threads_per_core": 2 }, "metrics": { - "NetworkOut": { - "rate": 10445.916666666666, - "sum": 626755 + "CPUCreditBalance": { + "avg": 576 }, - "CPUUtilization": { - "avg": 2.8849988898518673 + "CPUCreditUsage": { + "avg": 0.29100543333333334 }, - "StatusCheckFailed_Instance": { + "CPUSurplusCreditBalance": { "avg": 0 }, - "CPUCreditUsage": { - "avg": 0.29100543333333334 + "CPUSurplusCreditsCharged": { + "avg": 0 }, - "CPUCreditBalance": { - "avg": 576 + "CPUUtilization": { + "avg": 2.8849988898518673 }, - "NetworkPacketsOut": { - "rate": 82.95, - "sum": 4977 + "NetworkIn": { + "rate": 26815.983333333334, + "sum": 1608959 + }, + "NetworkOut": { + "rate": 10445.916666666666, + "sum": 626755 }, "NetworkPacketsIn": { "rate": 88.9, "sum": 5334 }, - "NetworkIn": { - "rate": 26815.983333333334, - "sum": 1608959 + "NetworkPacketsOut": { + "rate": 82.95, + "sum": 4977 }, "StatusCheckFailed": { "avg": 0 }, - "CPUSurplusCreditsCharged": { - "avg": 0 - }, - "CPUSurplusCreditBalance": { + "StatusCheckFailed_Instance": { "avg": 0 }, "StatusCheckFailed_System": { @@ -350,21 +250,96 @@ An example event for `ec2` looks as following: } } }, - "cloudwatch": { - "namespace": "AWS/EC2" - }, - "dimensions": { - "InstanceId": "i-0c08512debca266ab" - }, "tags": { "aws:autoscaling:groupName": "eks-firehose-50c386d7-c8b1-bde8-5d42-d3841ca7ecfe", - "aws:ec2launchtemplate:version": "1", - "aws:ec2launchtemplate:id": "lt-09e1cdf590e35c687" + "aws:ec2launchtemplate:id": "lt-09e1cdf590e35c687", + "aws:ec2launchtemplate:version": "1" + } + }, + "cloud": { + "account": { + "id": "627286350134", + "name": "MonitoringAccount" + }, + "availability_zone": "eu-north-1c", + "instance": { + "id": "i-0c08512debca266ab" + }, + "machine": { + "type": "t3.medium" + }, + "provider": "aws", + "region": "eu-north-1" + }, + "data_stream": { + "dataset": "aws.ec2_metrics", + "namespace": "default", + "type": "metrics" + }, + "ecs": { + "version": "8.11.0" + }, + "elastic_agent": { + "id": "72314f01-98f2-477f-978a-e98d109c640c", + "snapshot": false, + "version": "8.8.1" + }, + "event": { + "agent_id_status": "verified", + "dataset": "aws.ec2_metrics", + "duration": 5858967919, + "ingested": "2023-08-07T18:41:31Z", + "module": "aws" + }, + "host": { + "architecture": "aarch64", + "containerized": false, + "cpu": { + "usage": 2.8849988898518673 + }, + "hostname": "docker-fleet-agent", + "id": "d08b346fbb8f49f5a2bb1a477f8ceb54", + "ip": [ + "172.20.0.7" + ], + "mac": [ + "02-42-AC-14-00-07" + ], + "name": "docker-fleet-agent", + "network": { + "egress": { + "bytes": 626755, + "packets": 4977 + }, + "ingress": { + "bytes": 1608959, + "packets": 5334 + } + }, + "os": { + "codename": "focal", + "family": "debian", + "kernel": "5.15.49-linuxkit-pr", + "name": "Ubuntu", + "platform": "ubuntu", + "type": "linux", + "version": "20.04.6 LTS (Focal Fossa)" } + }, + "metricset": { + "name": "cloudwatch", + "period": 300000 + }, + "service": { + "type": "aws" } } ``` +**ECS Field Reference** + +Please refer to the following [document](https://www.elastic.co/guide/en/ecs/current/ecs-field-reference.html) for detailed information on ECS fields. + **Exported fields** | Field | Description | Type | Metric Type | @@ -411,31 +386,13 @@ An example event for `ec2` looks as following: | aws.ec2.metrics.StatusCheckFailed_Instance.avg | Reports whether the instance has passed the instance status check in the last minute. | long | gauge | | aws.ec2.metrics.StatusCheckFailed_System.avg | Reports whether the instance has passed the system status check in the last minute. | long | gauge | | aws.tags | Tag key value pairs from aws resources. | flattened | | -| cloud | Fields related to the cloud or infrastructure the events are coming from. | group | | | cloud.account.id | The cloud account or organization id used to identify different entities in a multi-tenant environment. Examples: AWS account id, Google Cloud ORG Id, or other unique identifier. | keyword | | -| cloud.account.name | The cloud account name or alias used to identify different entities in a multi-tenant environment. Examples: AWS account name, Google Cloud ORG display name. | keyword | | -| cloud.availability_zone | Availability zone in which this host, resource, or service is located. | keyword | | | cloud.image.id | Image ID for the cloud instance. | keyword | | -| cloud.instance.id | Instance ID of the host machine. | keyword | | -| cloud.machine.type | Machine type of the host machine. | keyword | | -| cloud.provider | Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. | keyword | | | cloud.region | Region in which this host, resource, or service is located. | keyword | | | data_stream.dataset | Data stream dataset. | constant_keyword | | | data_stream.namespace | Data stream namespace. | constant_keyword | | | data_stream.type | Data stream type. | constant_keyword | | -| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | | -| error | These fields can represent errors of any kind. Use them for errors that happen while fetching events or in cases where the event itself contains an error. | group | | -| error.message | Error message. | match_only_text | | -| event.dataset | Name of the dataset. If an event source publishes more than one type of log or events (e.g. access log, error log), the dataset is used to specify which one the event comes from. It's recommended but not required to start the dataset name with the module name, followed by a dot, then the dataset name. | constant_keyword | | | event.module | Event module | constant_keyword | | | host.containerized | If the host is a container. | boolean | | -| host.cpu.usage | Percent CPU used which is normalized by the number of CPU cores and it ranges from 0 to 1. Scaling factor: 1000. For example: For a two core host, this value should be the average of the two cores, between 0 and 1. | scaled_float | | -| host.disk.read.bytes | The total number of bytes (gauge) read successfully (aggregated from all disks) since the last metric collection. | long | | -| host.disk.write.bytes | The total number of bytes (gauge) written successfully (aggregated from all disks) since the last metric collection. | long | | -| host.network.egress.bytes | The number of bytes (gauge) sent out on all network interfaces by the host since the last metric collection. | long | | -| host.network.egress.packets | The number of packets (gauge) sent out on all network interfaces by the host since the last metric collection. | long | | -| host.network.ingress.bytes | The number of bytes received (gauge) on all network interfaces by the host since the last metric collection. | long | | -| host.network.ingress.packets | The number of packets (gauge) received on all network interfaces by the host since the last metric collection. | long | | | host.os.build | OS build information. | keyword | | | host.os.codename | OS codename, if any. | keyword | | -| service.type | The type of the service data is collected from. The type can be used to group and correlate logs and metrics from one service type. Example: If logs or metrics are collected from Elasticsearch, `service.type` would be `elasticsearch`. | keyword | | diff --git a/packages/aws/docs/ecs.md b/packages/aws/docs/ecs.md index 308cc58ddd8..0482fa8e103 100644 --- a/packages/aws/docs/ecs.md +++ b/packages/aws/docs/ecs.md @@ -43,92 +43,96 @@ An example event for `ecs` looks as following: ```json { + "@timestamp": "2022-07-26T08:59:00.000Z", "agent": { - "name": "4b4f1fd6f3ff", + "ephemeral_id": "0c23896b-0bfe-469f-bf76-7203a2d52568", "id": "8c424f1d-e9b1-4aab-8ce5-77dceb4becfb", + "name": "4b4f1fd6f3ff", "type": "metricbeat", - "ephemeral_id": "0c23896b-0bfe-469f-bf76-7203a2d52568", "version": "8.1.0" }, - "elastic_agent": { - "id": "8c424f1d-e9b1-4aab-8ce5-77dceb4becfb", - "version": "8.1.0", - "snapshot": false + "aws": { + "cloudwatch": { + "namespace": "AWS/ECS" + }, + "dimensions": { + "ClusterName": "integration-cluster-1", + "ServiceName": "integration-service-1" + }, + "ecs": { + "metrics": { + "CPUUtilization": { + "avg": 100.040084913373 + }, + "MemoryUtilization": { + "avg": 9.195963541666666 + } + } + } }, "cloud": { - "provider": "aws", - "region": "eu-west-1", "account": { - "name": "elastic-observability", - "id": "627286350134" - } + "id": "627286350134", + "name": "elastic-observability" + }, + "provider": "aws", + "region": "eu-west-1" + }, + "data_stream": { + "dataset": "aws.ecs_metrics", + "namespace": "default", + "type": "metrics" }, - "@timestamp": "2022-07-26T08:59:00.000Z", "ecs": { - "version": "8.0.0" + "version": "8.11.0" }, - "service": { - "type": "aws" + "elastic_agent": { + "id": "8c424f1d-e9b1-4aab-8ce5-77dceb4becfb", + "snapshot": false, + "version": "8.1.0" }, - "data_stream": { - "namespace": "default", - "type": "metrics", - "dataset": "aws.ecs_metrics" + "event": { + "agent_id_status": "verified", + "dataset": "aws.ecs_metrics", + "duration": 1862196584, + "ingested": "2022-07-26T09:04:12Z", + "module": "aws" }, "host": { - "hostname": "4b4f1fd6f3ff", - "os": { - "kernel": "5.10.104-linuxkit", - "codename": "focal", - "name": "Ubuntu", - "family": "debian", - "type": "linux", - "version": "20.04.3 LTS (Focal Fossa)", - "platform": "ubuntu" - }, + "architecture": "aarch64", "containerized": false, + "hostname": "4b4f1fd6f3ff", "ip": [ "172.19.0.4" ], - "name": "4b4f1fd6f3ff", "mac": [ "02-42-AC-13-00-04" ], - "architecture": "aarch64" + "name": "4b4f1fd6f3ff", + "os": { + "codename": "focal", + "family": "debian", + "kernel": "5.10.104-linuxkit", + "name": "Ubuntu", + "platform": "ubuntu", + "type": "linux", + "version": "20.04.3 LTS (Focal Fossa)" + } }, "metricset": { - "period": 300000, - "name": "cloudwatch" - }, - "aws": { - "ecs": { - "metrics": { - "CPUUtilization": { - "avg": 100.040084913373 - }, - "MemoryUtilization": { - "avg": 9.195963541666666 - } - } - }, - "cloudwatch": { - "namespace": "AWS/ECS" - }, - "dimensions": { - "ServiceName": "integration-service-1", - "ClusterName": "integration-cluster-1" - } + "name": "cloudwatch", + "period": 300000 }, - "event": { - "duration": 1862196584, - "agent_id_status": "verified", - "ingested": "2022-07-26T09:04:12Z", - "module": "aws", - "dataset": "aws.ecs_metrics" + "service": { + "type": "aws" } } ``` +**ECS Field Reference** + +Please refer to the following [document](https://www.elastic.co/guide/en/ecs/current/ecs-field-reference.html) for detailed information on ECS fields. + **Exported fields** | Field | Description | Type | Metric Type | @@ -144,44 +148,13 @@ An example event for `ecs` looks as following: | aws.ecs.metrics.MemoryReservation.avg | The percentage of memory that is reserved by running tasks in the cluster. | double | gauge | | aws.ecs.metrics.MemoryUtilization.avg | The percentage of memory that is used in the cluster or service. | double | gauge | | aws.tags | Tag key value pairs from aws resources. | flattened | | -| cloud | Fields related to the cloud or infrastructure the events are coming from. | group | | | cloud.account.id | The cloud account or organization id used to identify different entities in a multi-tenant environment. Examples: AWS account id, Google Cloud ORG Id, or other unique identifier. | keyword | | -| cloud.account.name | The cloud account name or alias used to identify different entities in a multi-tenant environment. Examples: AWS account name, Google Cloud ORG display name. | keyword | | -| cloud.availability_zone | Availability zone in which this host, resource, or service is located. | keyword | | | cloud.image.id | Image ID for the cloud instance. | keyword | | -| cloud.instance.id | Instance ID of the host machine. | keyword | | -| cloud.instance.name | Instance name of the host machine. | keyword | | -| cloud.machine.type | Machine type of the host machine. | keyword | | -| cloud.project.id | The cloud project identifier. Examples: Google Cloud Project id, Azure Project id. | keyword | | -| cloud.provider | Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. | keyword | | | cloud.region | Region in which this host, resource, or service is located. | keyword | | -| container.id | Unique container id. | keyword | | -| container.image.name | Name of the image the container was built on. | keyword | | -| container.labels | Image labels. | object | | -| container.name | Container name. | keyword | | | data_stream.dataset | Data stream dataset. | constant_keyword | | | data_stream.namespace | Data stream namespace. | constant_keyword | | | data_stream.type | Data stream type. | constant_keyword | | -| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | | -| error | These fields can represent errors of any kind. Use them for errors that happen while fetching events or in cases where the event itself contains an error. | group | | -| error.message | Error message. | match_only_text | | -| event.dataset | Name of the dataset. If an event source publishes more than one type of log or events (e.g. access log, error log), the dataset is used to specify which one the event comes from. It's recommended but not required to start the dataset name with the module name, followed by a dot, then the dataset name. | constant_keyword | | | event.module | Event module | constant_keyword | | -| host.architecture | Operating system architecture. | keyword | | | host.containerized | If the host is a container. | boolean | | -| host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | | -| host.hostname | Hostname of the host. It normally contains what the `hostname` command returns on the host machine. | keyword | | -| host.id | Unique host id. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`. | keyword | | -| host.ip | Host ip addresses. | ip | | -| host.mac | Host MAC addresses. The notation format from RFC 7042 is suggested: Each octet (that is, 8-bit byte) is represented by two [uppercase] hexadecimal digits giving the value of the octet as an unsigned integer. Successive octets are separated by a hyphen. | keyword | | -| host.name | Name of the host. It can contain what hostname returns on Unix systems, the fully qualified domain name (FQDN), or a name specified by the user. The recommended value is the lowercase FQDN of the host. | keyword | | | host.os.build | OS build information. | keyword | | | host.os.codename | OS codename, if any. | keyword | | -| host.os.family | OS family (such as redhat, debian, freebsd, windows). | keyword | | -| host.os.kernel | Operating system kernel version as a raw string. | keyword | | -| host.os.name | Operating system name, without the version. | keyword | | -| host.os.name.text | Multi-field of `host.os.name`. | match_only_text | | -| host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword | | -| host.os.version | Operating system version as a raw string. | keyword | | -| host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | | -| service.type | The type of the service data is collected from. The type can be used to group and correlate logs and metrics from one service type. Example: If logs or metrics are collected from Elasticsearch, `service.type` would be `elasticsearch`. | keyword | | diff --git a/packages/aws/docs/elb.md b/packages/aws/docs/elb.md index 5ee0480415d..1b89cecbadb 100644 --- a/packages/aws/docs/elb.md +++ b/packages/aws/docs/elb.md @@ -69,6 +69,10 @@ The `number_of_workers` setting defines the number of workers assigned to readin The `elb` dataset collects logs from AWS ELBs. +**ECS Field Reference** + +Please refer to the following [document](https://www.elastic.co/guide/en/ecs/current/ecs-field-reference.html) for detailed information on ECS fields. + **Exported fields** | Field | Description | Type | @@ -105,85 +109,16 @@ The `elb` dataset collects logs from AWS ELBs. | aws.s3.bucket.arn | The AWS S3 bucket ARN. | keyword | | aws.s3.bucket.name | The AWS S3 bucket name. | keyword | | aws.s3.object.key | The AWS S3 Object key. | keyword | -| cloud.account.id | The cloud account or organization id used to identify different entities in a multi-tenant environment. Examples: AWS account id, Google Cloud ORG Id, or other unique identifier. | keyword | -| cloud.availability_zone | Availability zone in which this host, resource, or service is located. | keyword | | cloud.image.id | Image ID for the cloud instance. | keyword | -| cloud.instance.id | Instance ID of the host machine. | keyword | -| cloud.instance.name | Instance name of the host machine. | keyword | -| cloud.machine.type | Machine type of the host machine. | keyword | -| cloud.project.id | The cloud project identifier. Examples: Google Cloud Project id, Azure Project id. | keyword | -| cloud.provider | Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. | keyword | -| cloud.region | Region in which this host, resource, or service is located. | keyword | -| container.id | Unique container id. | keyword | -| container.image.name | Name of the image the container was built on. | keyword | -| container.labels | Image labels. | object | -| container.name | Container name. | keyword | | data_stream.dataset | Data stream dataset. | constant_keyword | | data_stream.namespace | Data stream namespace. | constant_keyword | | data_stream.type | Data stream type. | constant_keyword | -| destination.bytes | Bytes sent from the destination to the source. | long | -| destination.domain | The domain name of the destination system. This value may be a host name, a fully qualified domain name, or another host naming format. The value may derive from the original event or be added from enrichment. | keyword | -| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | -| error.message | Error message. | match_only_text | -| event.category | This is one of four ECS Categorization Fields, and indicates the second level in the ECS category hierarchy. `event.category` represents the "big buckets" of ECS categories. For example, filtering on `event.category:process` yields all events relating to process activity. This field is closely related to `event.type`, which is used as a subcategory. This field is an array. This will allow proper categorization of some events that fall in multiple categories. | keyword | -| event.dataset | Name of the dataset. If an event source publishes more than one type of log or events (e.g. access log, error log), the dataset is used to specify which one the event comes from. It's recommended but not required to start the dataset name with the module name, followed by a dot, then the dataset name. | constant_keyword | -| event.end | `event.end` contains the date when the event ended or when the activity was last observed. | date | -| event.kind | This is one of four ECS Categorization Fields, and indicates the highest level in the ECS category hierarchy. `event.kind` gives high-level information about what type of information the event contains, without being specific to the contents of the event. For example, values of this field distinguish alert events from metric events. The value of this field can be used to inform how these kinds of events should be handled. They may warrant different retention, different access control, it may also help understand whether the data is coming in at a regular interval or not. | keyword | | event.module | Event module | constant_keyword | -| event.outcome | This is one of four ECS Categorization Fields, and indicates the lowest level in the ECS category hierarchy. `event.outcome` simply denotes whether the event represents a success or a failure from the perspective of the entity that produced the event. Note that when a single transaction is described in multiple events, each event may populate different values of `event.outcome`, according to their perspective. Also note that in the case of a compound event (a single event that contains multiple logical events), this field should be populated with the value that best captures the overall success or failure from the perspective of the event producer. Further note that not all events will have an associated outcome. For example, this field is generally not populated for metric events, events with `event.type:info`, or any events for which an outcome does not make logical sense. | keyword | -| event.start | `event.start` contains the date when the event started or when the activity was first observed. | date | -| host.architecture | Operating system architecture. | keyword | | host.containerized | If the host is a container. | boolean | -| host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | -| host.hostname | Hostname of the host. It normally contains what the `hostname` command returns on the host machine. | keyword | -| host.id | Unique host id. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`. | keyword | -| host.ip | Host ip addresses. | ip | -| host.mac | Host MAC addresses. The notation format from RFC 7042 is suggested: Each octet (that is, 8-bit byte) is represented by two [uppercase] hexadecimal digits giving the value of the octet as an unsigned integer. Successive octets are separated by a hyphen. | keyword | -| host.name | Name of the host. It can contain what hostname returns on Unix systems, the fully qualified domain name (FQDN), or a name specified by the user. The recommended value is the lowercase FQDN of the host. | keyword | | host.os.build | OS build information. | keyword | | host.os.codename | OS codename, if any. | keyword | -| host.os.family | OS family (such as redhat, debian, freebsd, windows). | keyword | -| host.os.kernel | Operating system kernel version as a raw string. | keyword | -| host.os.name | Operating system name, without the version. | keyword | -| host.os.name.text | Multi-field of `host.os.name`. | match_only_text | -| host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword | -| host.os.version | Operating system version as a raw string. | keyword | -| host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | -| http.request.body.bytes | Size in bytes of the request body. | long | -| http.request.method | HTTP request method. The value should retain its casing from the original event. For example, `GET`, `get`, and `GeT` are all considered valid values for this field. | keyword | -| http.request.referrer | Referrer for this HTTP request. | keyword | -| http.response.body.bytes | Size in bytes of the response body. | long | -| http.response.status_code | HTTP response status code. | long | -| http.version | HTTP version. | keyword | | input.type | Input type | keyword | -| log.file.path | Full path to the log file this event came from, including the file name. It should include the drive letter, when appropriate. If the event wasn't read from a log file, do not populate this field. | keyword | | log.offset | Log offset | long | -| source.address | Some event source addresses are defined ambiguously. The event will sometimes list an IP, a domain or a unix socket. You should always store the raw address in the `.address` field. Then it should be duplicated to `.ip` or `.domain`, depending on which one it is. | keyword | -| source.as.number | Unique number allocated to the autonomous system. The autonomous system number (ASN) uniquely identifies each network on the Internet. | long | -| source.as.organization.name | Organization name. | keyword | -| source.as.organization.name.text | Multi-field of `source.as.organization.name`. | match_only_text | -| source.geo.city_name | City name. | keyword | -| source.geo.continent_name | Name of the continent. | keyword | -| source.geo.country_iso_code | Country ISO code. | keyword | -| source.geo.country_name | Country name. | keyword | -| source.geo.location | Longitude and latitude. | geo_point | -| source.geo.region_name | Region name. | keyword | -| source.ip | IP address of the source (IPv4 or IPv6). | ip | -| source.port | Port of the source. | long | -| tags | List of keywords used to tag each event. | keyword | -| trace.id | Unique identifier of the trace. A trace groups multiple events like transactions that belong together. For example, a user request handled by multiple inter-connected services. | keyword | -| url.domain | Domain of the url, such as "www.elastic.co". In some cases a URL may refer to an IP and/or port directly, without a domain name. In this case, the IP address would go to the `domain` field. If the URL contains a literal IPv6 address enclosed by `[` and `]` (IETF RFC 2732), the `[` and `]` characters should also be captured in the `domain` field. | keyword | -| url.original | Unmodified original url as seen in the event source. Note that in network monitoring, the observed URL may be a full URL, whereas in access logs, the URL is often just represented as a path. This field is meant to represent the URL as it was observed, complete or not. | wildcard | -| url.original.text | Multi-field of `url.original`. | match_only_text | -| url.path | Path of the request, such as "/search". | wildcard | -| url.port | Port of the request, such as 443. | long | -| url.query | The query field describes the query string of the request, such as "q=elasticsearch". The `?` is excluded from the query string. If a URL contains no `?`, there is no query field. If there is a `?` but no query, the query field exists with an empty string. The `exists` query can be used to differentiate between the two cases. | keyword | -| url.scheme | Scheme of the request, such as "https". Note: The `:` is not part of the scheme. | keyword | -| user_agent.device.name | Name of the device. | keyword | -| user_agent.name | Name of the user agent. | keyword | -| user_agent.original | Unparsed user_agent string. | keyword | -| user_agent.original.text | Multi-field of `user_agent.original`. | match_only_text | -| user_agent.version | Version of the user agent. | keyword | An example event for `elb` looks as following: @@ -257,7 +192,7 @@ An example event for `elb` looks as following: "type": "logs" }, "ecs": { - "version": "8.2.0" + "version": "8.11.0" }, "elastic_agent": { "id": "acba78ef-1401-4689-977c-d8c2e5d6a8fa", @@ -340,98 +275,102 @@ An example event for `elb` looks as following: { "@timestamp": "2022-06-08T18:19:00.000Z", "agent": { - "name": "docker-fleet-agent", + "ephemeral_id": "8c94e850-82e2-42ae-bd41-44ce7bbbb50c", "id": "90bfb41e-b925-420f-973e-9c1115297278", + "name": "docker-fleet-agent", "type": "metricbeat", - "ephemeral_id": "8c94e850-82e2-42ae-bd41-44ce7bbbb50c", "version": "8.2.0" }, - "elastic_agent": { - "id": "90bfb41e-b925-420f-973e-9c1115297278", - "version": "8.2.0", - "snapshot": false + "aws": { + "cloudwatch": { + "namespace": "AWS/ELB" + }, + "elb": { + "metrics": { + "HTTPCode_Backend_2XX": { + "sum": 31 + }, + "HTTPCode_Backend_4XX": { + "sum": 2 + }, + "HealthyHostCount": { + "max": 2 + }, + "Latency": { + "avg": 0.0010771534659645772 + }, + "RequestCount": { + "sum": 33 + }, + "UnHealthyHostCount": { + "max": 0 + } + } + } }, "cloud": { - "provider": "aws", - "region": "eu-central-1", "account": { - "name": "elastic-beats", - "id": "123456789" - } - }, - "ecs": { - "version": "8.0.0" + "id": "123456789", + "name": "elastic-beats" + }, + "provider": "aws", + "region": "eu-central-1" }, "data_stream": { + "dataset": "aws.elb_metrics", "namespace": "default", - "type": "metrics", - "dataset": "aws.elb_metrics" + "type": "metrics" }, - "service": { - "type": "aws" + "ecs": { + "version": "8.11.0" + }, + "elastic_agent": { + "id": "90bfb41e-b925-420f-973e-9c1115297278", + "snapshot": false, + "version": "8.2.0" + }, + "event": { + "agent_id_status": "verified", + "dataset": "aws.elb_metrics", + "duration": 15866718200, + "ingested": "2022-06-08T18:20:24Z", + "module": "aws" }, "host": { - "hostname": "docker-fleet-agent", - "os": { - "kernel": "5.10.47-linuxkit", - "codename": "focal", - "name": "Ubuntu", - "family": "debian", - "type": "linux", - "version": "20.04.4 LTS (Focal Fossa)", - "platform": "ubuntu" - }, + "architecture": "x86_64", "containerized": true, + "hostname": "docker-fleet-agent", "ip": [ "192.168.96.7" ], - "name": "docker-fleet-agent", "mac": [ "02-42-C0-A8-60-07" ], - "architecture": "x86_64" + "name": "docker-fleet-agent", + "os": { + "codename": "focal", + "family": "debian", + "kernel": "5.10.47-linuxkit", + "name": "Ubuntu", + "platform": "ubuntu", + "type": "linux", + "version": "20.04.4 LTS (Focal Fossa)" + } }, "metricset": { - "period": 60000, - "name": "cloudwatch" + "name": "cloudwatch", + "period": 60000 }, - "aws": { - "elb": { - "metrics": { - "HealthyHostCount": { - "max": 2 - }, - "UnHealthyHostCount": { - "max": 0 - }, - "HTTPCode_Backend_4XX": { - "sum": 2 - }, - "HTTPCode_Backend_2XX": { - "sum": 31 - }, - "RequestCount": { - "sum": 33 - }, - "Latency": { - "avg": 0.0010771534659645772 - } - } - }, - "cloudwatch": { - "namespace": "AWS/ELB" - } - }, - "event": { - "duration": 15866718200, - "agent_id_status": "verified", - "ingested": "2022-06-08T18:20:24Z", - "module": "aws", - "dataset": "aws.elb_metrics" + "service": { + "type": "aws" } } ``` +**ECS Field Reference** + +Please refer to the following [document](https://www.elastic.co/guide/en/ecs/current/ecs-field-reference.html) for detailed information on ECS fields. + **Exported fields** | Field | Description | Type | Unit | Metric Type | @@ -498,45 +437,14 @@ An example event for `elb` looks as following: | aws.networkelb.metrics.TargetTLSNegotiationErrorCount.sum | The total number of TLS handshakes that failed during negotiation between a TLS listener and a target. | long | | gauge | | aws.networkelb.metrics.UnHealthyHostCount.max | The number of targets that are considered unhealthy. | long | | gauge | | aws.tags | Tag key value pairs from aws resources. | flattened | | | -| cloud | Fields related to the cloud or infrastructure the events are coming from. | group | | | | cloud.account.id | The cloud account or organization id used to identify different entities in a multi-tenant environment. Examples: AWS account id, Google Cloud ORG Id, or other unique identifier. | keyword | | | -| cloud.account.name | The cloud account name or alias used to identify different entities in a multi-tenant environment. Examples: AWS account name, Google Cloud ORG display name. | keyword | | | -| cloud.availability_zone | Availability zone in which this host, resource, or service is located. | keyword | | | | cloud.image.id | Image ID for the cloud instance. | keyword | | | -| cloud.instance.id | Instance ID of the host machine. | keyword | | | -| cloud.instance.name | Instance name of the host machine. | keyword | | | -| cloud.machine.type | Machine type of the host machine. | keyword | | | -| cloud.project.id | The cloud project identifier. Examples: Google Cloud Project id, Azure Project id. | keyword | | | -| cloud.provider | Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. | keyword | | | | cloud.region | Region in which this host, resource, or service is located. | keyword | | | -| container.id | Unique container id. | keyword | | | -| container.image.name | Name of the image the container was built on. | keyword | | | -| container.labels | Image labels. | object | | | -| container.name | Container name. | keyword | | | | data_stream.dataset | Data stream dataset. | constant_keyword | | | | data_stream.namespace | Data stream namespace. | constant_keyword | | | | data_stream.type | Data stream type. | constant_keyword | | | -| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | | | -| error | These fields can represent errors of any kind. Use them for errors that happen while fetching events or in cases where the event itself contains an error. | group | | | -| error.message | Error message. | match_only_text | | | -| event.dataset | Name of the dataset. If an event source publishes more than one type of log or events (e.g. access log, error log), the dataset is used to specify which one the event comes from. It's recommended but not required to start the dataset name with the module name, followed by a dot, then the dataset name. | constant_keyword | | | | event.module | Event module | constant_keyword | | | -| host.architecture | Operating system architecture. | keyword | | | | host.containerized | If the host is a container. | boolean | | | -| host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | | | -| host.hostname | Hostname of the host. It normally contains what the `hostname` command returns on the host machine. | keyword | | | -| host.id | Unique host id. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`. | keyword | | | -| host.ip | Host ip addresses. | ip | | | -| host.mac | Host MAC addresses. The notation format from RFC 7042 is suggested: Each octet (that is, 8-bit byte) is represented by two [uppercase] hexadecimal digits giving the value of the octet as an unsigned integer. Successive octets are separated by a hyphen. | keyword | | | -| host.name | Name of the host. It can contain what hostname returns on Unix systems, the fully qualified domain name (FQDN), or a name specified by the user. The recommended value is the lowercase FQDN of the host. | keyword | | | | host.os.build | OS build information. | keyword | | | | host.os.codename | OS codename, if any. | keyword | | | -| host.os.family | OS family (such as redhat, debian, freebsd, windows). | keyword | | | -| host.os.kernel | Operating system kernel version as a raw string. | keyword | | | -| host.os.name | Operating system name, without the version. | keyword | | | -| host.os.name.text | Multi-field of `host.os.name`. | match_only_text | | | -| host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword | | | -| host.os.version | Operating system version as a raw string. | keyword | | | -| host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | | | -| service.type | The type of the service data is collected from. The type can be used to group and correlate logs and metrics from one service type. Example: If logs or metrics are collected from Elasticsearch, `service.type` would be `elasticsearch`. | keyword | | | diff --git a/packages/aws/docs/emr.md b/packages/aws/docs/emr.md index 6568ab08a27..1d02caddef2 100644 --- a/packages/aws/docs/emr.md +++ b/packages/aws/docs/emr.md @@ -48,65 +48,69 @@ An example event for `emr` looks as following: { "@timestamp": "2022-07-26T21:43:00.000Z", "agent": { - "name": "docker-fleet-agent", + "ephemeral_id": "cdaaaabb-be7e-432f-816b-bda019fd7c15", "id": "2d4b09d0-cdb6-445e-ac3f-6415f87b9864", + "name": "docker-fleet-agent", "type": "metricbeat", - "ephemeral_id": "cdaaaabb-be7e-432f-816b-bda019fd7c15", "version": "8.3.2" }, - "elastic_agent": { - "id": "2d4b09d0-cdb6-445e-ac3f-6415f87b9864", - "version": "8.3.2", - "snapshot": false - }, - "cloud": { - "provider": "aws", - "region": "eu-central-1", - "account": { - "name": "elastic-beats", - "id": "428152502467" - } - }, - "ecs": { - "version": "8.0.0" - }, - "service": { - "type": "aws" - }, - "data_stream": { - "namespace": "default", - "type": "metrics", - "dataset": "aws.emr_metrics" - }, - "metricset": { - "period": 300000, - "name": "cloudwatch" - }, "aws": { + "cloudwatch": { + "namespace": "AWS/ElasticMapReduce" + }, + "dimensions": { + "JobFlowId": "j-3LRBO17JBA7H9" + }, "elasticmapreduce": { "metrics": { "IsIdle": { "avg": 1 } } - }, - "cloudwatch": { - "namespace": "AWS/ElasticMapReduce" - }, - "dimensions": { - "JobFlowId": "j-3LRBO17JBA7H9" } }, + "cloud": { + "account": { + "id": "428152502467", + "name": "elastic-beats" + }, + "provider": "aws", + "region": "eu-central-1" + }, + "data_stream": { + "dataset": "aws.emr_metrics", + "namespace": "default", + "type": "metrics" + }, + "ecs": { + "version": "8.11.0" + }, + "elastic_agent": { + "id": "2d4b09d0-cdb6-445e-ac3f-6415f87b9864", + "snapshot": false, + "version": "8.3.2" + }, "event": { - "duration": 11576777300, "agent_id_status": "verified", + "dataset": "aws.emr_metrics", + "duration": 11576777300, "ingested": "2022-07-26T21:47:48Z", - "module": "aws", - "dataset": "aws.emr_metrics" + "module": "aws" + }, + "metricset": { + "name": "cloudwatch", + "period": 300000 + }, + "service": { + "type": "aws" } } ``` +**ECS Field Reference** + +Please refer to the following [document](https://www.elastic.co/guide/en/ecs/current/ecs-field-reference.html) for detailed information on ECS fields. + **Exported fields** | Field | Description | Type | Unit | Metric Type | @@ -175,47 +179,16 @@ An example event for `emr` looks as following: | aws.elasticmapreduce.metrics.UnderReplicatedBlocks.sum | The number of blocks that need to be replicated one or more times. | long | | gauge | | aws.elasticmapreduce.metrics.YARNMemoryAvailablePercentage.avg | The percentage of remaining memory available to YARN | double | percent | gauge | | aws.tags | Tag key value pairs from aws resources. | flattened | | | -| cloud | Fields related to the cloud or infrastructure the events are coming from. | group | | | | cloud.account.id | The cloud account or organization id used to identify different entities in a multi-tenant environment. Examples: AWS account id, Google Cloud ORG Id, or other unique identifier. | keyword | | | -| cloud.account.name | The cloud account name or alias used to identify different entities in a multi-tenant environment. Examples: AWS account name, Google Cloud ORG display name. | keyword | | | -| cloud.availability_zone | Availability zone in which this host, resource, or service is located. | keyword | | | | cloud.image.id | Image ID for the cloud instance. | keyword | | | -| cloud.instance.id | Instance ID of the host machine. | keyword | | | -| cloud.instance.name | Instance name of the host machine. | keyword | | | -| cloud.machine.type | Machine type of the host machine. | keyword | | | -| cloud.project.id | The cloud project identifier. Examples: Google Cloud Project id, Azure Project id. | keyword | | | -| cloud.provider | Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. | keyword | | | | cloud.region | Region in which this host, resource, or service is located. | keyword | | | -| container.id | Unique container id. | keyword | | | -| container.image.name | Name of the image the container was built on. | keyword | | | -| container.labels | Image labels. | object | | | -| container.name | Container name. | keyword | | | | data_stream.dataset | Data stream dataset. | constant_keyword | | | | data_stream.namespace | Data stream namespace. | constant_keyword | | | | data_stream.type | Data stream type. | constant_keyword | | | -| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | | | -| error | These fields can represent errors of any kind. Use them for errors that happen while fetching events or in cases where the event itself contains an error. | group | | | -| error.message | Error message. | match_only_text | | | -| event.dataset | Name of the dataset. If an event source publishes more than one type of log or events (e.g. access log, error log), the dataset is used to specify which one the event comes from. It's recommended but not required to start the dataset name with the module name, followed by a dot, then the dataset name. | constant_keyword | | | | event.module | Event module | constant_keyword | | | -| host.architecture | Operating system architecture. | keyword | | | | host.containerized | If the host is a container. | boolean | | | -| host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | | | -| host.hostname | Hostname of the host. It normally contains what the `hostname` command returns on the host machine. | keyword | | | -| host.id | Unique host id. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`. | keyword | | | -| host.ip | Host ip addresses. | ip | | | -| host.mac | Host MAC addresses. The notation format from RFC 7042 is suggested: Each octet (that is, 8-bit byte) is represented by two [uppercase] hexadecimal digits giving the value of the octet as an unsigned integer. Successive octets are separated by a hyphen. | keyword | | | -| host.name | Name of the host. It can contain what hostname returns on Unix systems, the fully qualified domain name (FQDN), or a name specified by the user. The recommended value is the lowercase FQDN of the host. | keyword | | | | host.os.build | OS build information. | keyword | | | | host.os.codename | OS codename, if any. | keyword | | | -| host.os.family | OS family (such as redhat, debian, freebsd, windows). | keyword | | | -| host.os.kernel | Operating system kernel version as a raw string. | keyword | | | -| host.os.name | Operating system name, without the version. | keyword | | | -| host.os.name.text | Multi-field of `host.os.name`. | match_only_text | | | -| host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword | | | -| host.os.version | Operating system version as a raw string. | keyword | | | -| host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | | | -| service.type | The type of the service data is collected from. The type can be used to group and correlate logs and metrics from one service type. Example: If logs or metrics are collected from Elasticsearch, `service.type` would be `elasticsearch`. | keyword | | | ## Logs reference @@ -252,7 +225,7 @@ An example event for `emr` looks as following: "type": "logs" }, "ecs": { - "version": "8.0.0" + "version": "8.11.0" }, "elastic_agent": { "id": "acba78ef-1401-4689-977c-d8c2e5d6a8fa", @@ -287,6 +260,10 @@ An example event for `emr` looks as following: } ``` +**ECS Field Reference** + +Please refer to the following [document](https://www.elastic.co/guide/en/ecs/current/ecs-field-reference.html) for detailed information on ECS fields. + **Exported fields** | Field | Description | Type | @@ -296,49 +273,15 @@ An example event for `emr` looks as following: | aws.s3.bucket.name | Name of a S3 bucket. | keyword | | aws.s3.metadata | AWS S3 object metadata values. | flattened | | aws.s3.object.key | Name of the S3 object that this log retrieved from. | keyword | -| cloud.account.id | The cloud account or organization id used to identify different entities in a multi-tenant environment. Examples: AWS account id, Google Cloud ORG Id, or other unique identifier. | keyword | -| cloud.availability_zone | Availability zone in which this host, resource, or service is located. | keyword | | cloud.image.id | Image ID for the cloud instance. | keyword | -| cloud.instance.id | Instance ID of the host machine. | keyword | -| cloud.instance.name | Instance name of the host machine. | keyword | -| cloud.machine.type | Machine type of the host machine. | keyword | -| cloud.project.id | The cloud project identifier. Examples: Google Cloud Project id, Azure Project id. | keyword | -| cloud.provider | Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. | keyword | -| cloud.region | Region in which this host, resource, or service is located. | keyword | -| container.id | Unique container id. | keyword | -| container.image.name | Name of the image the container was built on. | keyword | -| container.labels | Image labels. | object | -| container.name | Container name. | keyword | | data_stream.dataset | Data stream dataset. | constant_keyword | | data_stream.namespace | Data stream namespace. | constant_keyword | | data_stream.type | Data stream type. | constant_keyword | -| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | -| error.message | Error message. | match_only_text | -| event.dataset | Name of the dataset. If an event source publishes more than one type of log or events (e.g. access log, error log), the dataset is used to specify which one the event comes from. It's recommended but not required to start the dataset name with the module name, followed by a dot, then the dataset name. | constant_keyword | | event.module | Event module | constant_keyword | -| host.architecture | Operating system architecture. | keyword | | host.containerized | If the host is a container. | boolean | -| host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | -| host.hostname | Hostname of the host. It normally contains what the `hostname` command returns on the host machine. | keyword | -| host.id | Unique host id. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`. | keyword | -| host.ip | Host ip addresses. | ip | -| host.mac | Host MAC addresses. The notation format from RFC 7042 is suggested: Each octet (that is, 8-bit byte) is represented by two [uppercase] hexadecimal digits giving the value of the octet as an unsigned integer. Successive octets are separated by a hyphen. | keyword | -| host.name | Name of the host. It can contain what hostname returns on Unix systems, the fully qualified domain name (FQDN), or a name specified by the user. The recommended value is the lowercase FQDN of the host. | keyword | | host.os.build | OS build information. | keyword | | host.os.codename | OS codename, if any. | keyword | -| host.os.family | OS family (such as redhat, debian, freebsd, windows). | keyword | -| host.os.kernel | Operating system kernel version as a raw string. | keyword | -| host.os.name | Operating system name, without the version. | keyword | -| host.os.name.text | Multi-field of `host.os.name`. | match_only_text | -| host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword | -| host.os.version | Operating system version as a raw string. | keyword | -| host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | | input.type | Input type | keyword | -| log.file.path | Full path to the log file this event came from, including the file name. It should include the drive letter, when appropriate. If the event wasn't read from a log file, do not populate this field. | keyword | -| log.level | Original log level of the log event. If the source of the event provides a log level or textual severity, this is the one that goes in `log.level`. If your source doesn't specify one, you may put your event transport's severity here (e.g. Syslog severity). Some examples are `warn`, `err`, `i`, `informational`. | keyword | | log.offset | Log offset | long | -| message | For log events the message field contains the log message, optimized for viewing in a log viewer. For structured logs without an original message field, other fields can be concatenated to form a human-readable summary of the event. If multiple messages exist, they can be combined into one message. | match_only_text | | process.entrypoint | Process entrypoint. | keyword | | process.message | Process message. | keyword | -| process.name | Process name. | keyword | -| tags | List of keywords used to tag each event. | keyword | diff --git a/packages/aws/docs/firewall.md b/packages/aws/docs/firewall.md index 9d8eaa72470..43b70bdf2e3 100644 --- a/packages/aws/docs/firewall.md +++ b/packages/aws/docs/firewall.md @@ -123,7 +123,7 @@ An example event for `firewall` looks as following: "port": 80 }, "ecs": { - "version": "8.0.0" + "version": "8.11.0" }, "elastic_agent": { "id": "acba78ef-1401-4689-977c-d8c2e5d6a8fa", @@ -225,6 +225,10 @@ An example event for `firewall` looks as following: } ``` +**ECS Field Reference** + +Please refer to the following [document](https://www.elastic.co/guide/en/ecs/current/ecs-field-reference.html) for detailed information on ECS fields. + **Exported fields** | Field | Description | Type | @@ -243,125 +247,16 @@ An example event for `firewall` looks as following: | aws.s3.bucket.arn | The AWS S3 bucket ARN. | keyword | | aws.s3.bucket.name | The AWS S3 bucket name. | keyword | | aws.s3.object.key | The AWS S3 Object key. | keyword | -| cloud.account.id | The cloud account or organization id used to identify different entities in a multi-tenant environment. Examples: AWS account id, Google Cloud ORG Id, or other unique identifier. | keyword | -| cloud.availability_zone | Availability zone in which this host, resource, or service is located. | keyword | | cloud.image.id | Image ID for the cloud instance. | keyword | -| cloud.instance.id | Instance ID of the host machine. | keyword | -| cloud.instance.name | Instance name of the host machine. | keyword | -| cloud.machine.type | Machine type of the host machine. | keyword | -| cloud.project.id | The cloud project identifier. Examples: Google Cloud Project id, Azure Project id. | keyword | -| cloud.provider | Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. | keyword | -| cloud.region | Region in which this host, resource, or service is located. | keyword | -| container.id | Unique container id. | keyword | -| container.image.name | Name of the image the container was built on. | keyword | -| container.labels | Image labels. | object | -| container.name | Container name. | keyword | | data_stream.dataset | Data stream dataset. | constant_keyword | | data_stream.namespace | Data stream namespace. | constant_keyword | | data_stream.type | Data stream type. | constant_keyword | -| destination.address | Some event destination addresses are defined ambiguously. The event will sometimes list an IP, a domain or a unix socket. You should always store the raw address in the `.address` field. Then it should be duplicated to `.ip` or `.domain`, depending on which one it is. | keyword | -| destination.as.number | Unique number allocated to the autonomous system. The autonomous system number (ASN) uniquely identifies each network on the Internet. | long | -| destination.as.organization.name | Organization name. | keyword | -| destination.as.organization.name.text | Multi-field of `destination.as.organization.name`. | match_only_text | -| destination.bytes | Bytes sent from the destination to the source. | long | -| destination.domain | The domain name of the destination system. This value may be a host name, a fully qualified domain name, or another host naming format. The value may derive from the original event or be added from enrichment. | keyword | -| destination.geo.city_name | City name. | keyword | -| destination.geo.continent_name | Name of the continent. | keyword | -| destination.geo.country_iso_code | Country ISO code. | keyword | -| destination.geo.country_name | Country name. | keyword | -| destination.geo.location | Longitude and latitude. | geo_point | -| destination.geo.name | User-defined description of a location, at the level of granularity they care about. Could be the name of their data centers, the floor number, if this describes a local physical entity, city names. Not typically used in automated geolocation. | keyword | -| destination.geo.region_iso_code | Region ISO code. | keyword | -| destination.geo.region_name | Region name. | keyword | -| destination.ip | IP address of the destination (IPv4 or IPv6). | ip | -| destination.port | Port of the destination. | long | -| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | -| error.message | Error message. | match_only_text | -| event.action | The action captured by the event. This describes the information in the event. It is more specific than `event.category`. Examples are `group-add`, `process-started`, `file-created`. The value is normally defined by the implementer. | keyword | -| event.category | This is one of four ECS Categorization Fields, and indicates the second level in the ECS category hierarchy. `event.category` represents the "big buckets" of ECS categories. For example, filtering on `event.category:process` yields all events relating to process activity. This field is closely related to `event.type`, which is used as a subcategory. This field is an array. This will allow proper categorization of some events that fall in multiple categories. | keyword | -| event.created | `event.created` contains the date/time when the event was first read by an agent, or by your pipeline. This field is distinct from `@timestamp` in that `@timestamp` typically contain the time extracted from the original event. In most situations, these two timestamps will be slightly different. The difference can be used to calculate the delay between your source generating an event, and the time when your agent first processed it. This can be used to monitor your agent's or pipeline's ability to keep up with your event source. In case the two timestamps are identical, `@timestamp` should be used. | date | -| event.dataset | Name of the dataset. If an event source publishes more than one type of log or events (e.g. access log, error log), the dataset is used to specify which one the event comes from. It's recommended but not required to start the dataset name with the module name, followed by a dot, then the dataset name. | constant_keyword | -| event.ingested | Timestamp when an event arrived in the central data store. This is different from `@timestamp`, which is when the event originally occurred. It's also different from `event.created`, which is meant to capture the first time an agent saw the event. In normal conditions, assuming no tampering, the timestamps should chronologically look like this: `@timestamp` \< `event.created` \< `event.ingested`. | date | -| event.kind | This is one of four ECS Categorization Fields, and indicates the highest level in the ECS category hierarchy. `event.kind` gives high-level information about what type of information the event contains, without being specific to the contents of the event. For example, values of this field distinguish alert events from metric events. The value of this field can be used to inform how these kinds of events should be handled. They may warrant different retention, different access control, it may also help understand whether the data is coming in at a regular interval or not. | keyword | | event.module | Event module | constant_keyword | -| event.original | Raw text message of entire event. Used to demonstrate log integrity or where the full log message (before splitting it up in multiple parts) may be required, e.g. for reindex. This field is not indexed and doc_values are disabled. It cannot be searched, but it can be retrieved from `_source`. If users wish to override this and index this field, please see `Field data types` in the `Elasticsearch Reference`. | keyword | -| event.provider | Source of the event. Event transports such as Syslog or the Windows Event Log typically mention the source of an event. It can be the name of the software that generated the event (e.g. Sysmon, httpd), or of a subsystem of the operating system (kernel, Microsoft-Windows-Security-Auditing). | keyword | -| event.type | This is one of four ECS Categorization Fields, and indicates the third level in the ECS category hierarchy. `event.type` represents a categorization "sub-bucket" that, when used along with the `event.category` field values, enables filtering events down to a level appropriate for single visualization. This field is an array. This will allow proper categorization of some events that fall in multiple event types. | keyword | -| group.id | Unique identifier for the group on the system/platform. | keyword | -| group.name | Name of the group. | keyword | -| host.architecture | Operating system architecture. | keyword | | host.containerized | If the host is a container. | boolean | -| host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | -| host.hostname | Hostname of the host. It normally contains what the `hostname` command returns on the host machine. | keyword | -| host.id | Unique host id. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`. | keyword | -| host.ip | Host ip addresses. | ip | -| host.mac | Host MAC addresses. The notation format from RFC 7042 is suggested: Each octet (that is, 8-bit byte) is represented by two [uppercase] hexadecimal digits giving the value of the octet as an unsigned integer. Successive octets are separated by a hyphen. | keyword | -| host.name | Name of the host. It can contain what hostname returns on Unix systems, the fully qualified domain name (FQDN), or a name specified by the user. The recommended value is the lowercase FQDN of the host. | keyword | | host.os.build | OS build information. | keyword | | host.os.codename | OS codename, if any. | keyword | -| host.os.family | OS family (such as redhat, debian, freebsd, windows). | keyword | -| host.os.kernel | Operating system kernel version as a raw string. | keyword | -| host.os.name | Operating system name, without the version. | keyword | -| host.os.name.text | Multi-field of `host.os.name`. | match_only_text | -| host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword | -| host.os.version | Operating system version as a raw string. | keyword | -| host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | -| http.request.method | HTTP request method. The value should retain its casing from the original event. For example, `GET`, `get`, and `GeT` are all considered valid values for this field. | keyword | -| http.version | HTTP version. | keyword | | input.type | Input type | keyword | -| log.file.path | Full path to the log file this event came from, including the file name. It should include the drive letter, when appropriate. If the event wasn't read from a log file, do not populate this field. | keyword | | log.offset | Log offset | long | -| message | For log events the message field contains the log message, optimized for viewing in a log viewer. For structured logs without an original message field, other fields can be concatenated to form a human-readable summary of the event. If multiple messages exist, they can be combined into one message. | match_only_text | -| network.community_id | A hash of source and destination IPs and ports, as well as the protocol used in a communication. This is a tool-agnostic standard to identify flows. Learn more at https://github.com/corelight/community-id-spec. | keyword | -| network.protocol | In the OSI Model this would be the Application Layer protocol. For example, `http`, `dns`, or `ssh`. The field value must be normalized to lowercase for querying. | keyword | -| network.transport | Same as network.iana_number, but instead using the Keyword name of the transport layer (udp, tcp, ipv6-icmp, etc.) The field value must be normalized to lowercase for querying. | keyword | -| network.type | In the OSI Model this would be the Network Layer. ipv4, ipv6, ipsec, pim, etc The field value must be normalized to lowercase for querying. | keyword | -| observer.name | Custom name of the observer. This is a name that can be given to an observer. This can be helpful for example if multiple firewalls of the same model are used in an organization. If no custom name is needed, the field can be left empty. | keyword | -| observer.product | The product name of the observer. | keyword | -| observer.type | The type of the observer the data is coming from. There is no predefined list of observer types. Some examples are `forwarder`, `firewall`, `ids`, `ips`, `proxy`, `poller`, `sensor`, `APM server`. | keyword | -| observer.vendor | Vendor name of the observer. | keyword | -| related.hosts | All hostnames or other host identifiers seen on your event. Example identifiers include FQDNs, domain names, workstation names, or aliases. | keyword | -| related.ip | All of the IPs seen on your event. | ip | -| rule.category | A categorization value keyword used by the entity using the rule for detection of this event. | keyword | -| rule.id | A rule ID that is unique within the scope of an agent, observer, or other entity using the rule for detection of this event. | keyword | -| rule.name | The name of the rule or signature generating the event. | keyword | -| source.address | Some event source addresses are defined ambiguously. The event will sometimes list an IP, a domain or a unix socket. You should always store the raw address in the `.address` field. Then it should be duplicated to `.ip` or `.domain`, depending on which one it is. | keyword | -| source.as.number | Unique number allocated to the autonomous system. The autonomous system number (ASN) uniquely identifies each network on the Internet. | long | -| source.as.organization.name | Organization name. | keyword | -| source.as.organization.name.text | Multi-field of `source.as.organization.name`. | match_only_text | -| source.geo.city_name | City name. | keyword | -| source.geo.continent_name | Name of the continent. | keyword | -| source.geo.country_iso_code | Country ISO code. | keyword | -| source.geo.country_name | Country name. | keyword | -| source.geo.location | Longitude and latitude. | geo_point | -| source.geo.region_iso_code | Region ISO code. | keyword | -| source.geo.region_name | Region name. | keyword | -| source.ip | IP address of the source (IPv4 or IPv6). | ip | -| source.port | Port of the source. | long | -| tags | List of keywords used to tag each event. | keyword | -| url.domain | Domain of the url, such as "www.elastic.co". In some cases a URL may refer to an IP and/or port directly, without a domain name. In this case, the IP address would go to the `domain` field. If the URL contains a literal IPv6 address enclosed by `[` and `]` (IETF RFC 2732), the `[` and `]` characters should also be captured in the `domain` field. | keyword | -| url.original | Unmodified original url as seen in the event source. Note that in network monitoring, the observed URL may be a full URL, whereas in access logs, the URL is often just represented as a path. This field is meant to represent the URL as it was observed, complete or not. | wildcard | -| url.original.text | Multi-field of `url.original`. | match_only_text | -| url.path | Path of the request, such as "/search". | wildcard | -| url.scheme | Scheme of the request, such as "https". Note: The `:` is not part of the scheme. | keyword | -| user.changes.name | Short name or login of the user. | keyword | -| user.changes.name.text | Multi-field of `user.changes.name`. | match_only_text | -| user.id | Unique identifier of the user. | keyword | -| user.name | Short name or login of the user. | keyword | -| user.name.text | Multi-field of `user.name`. | match_only_text | -| user.target.id | Unique identifier of the user. | keyword | -| user.target.name | Short name or login of the user. | keyword | -| user.target.name.text | Multi-field of `user.target.name`. | match_only_text | -| user_agent.device.name | Name of the device. | keyword | -| user_agent.name | Name of the user agent. | keyword | -| user_agent.original | Unparsed user_agent string. | keyword | -| user_agent.original.text | Multi-field of `user_agent.original`. | match_only_text | -| user_agent.os.full | Operating system name, including the version or code name. | keyword | -| user_agent.os.full.text | Multi-field of `user_agent.os.full`. | match_only_text | -| user_agent.os.name | Operating system name, without the version. | keyword | -| user_agent.os.name.text | Multi-field of `user_agent.os.name`. | match_only_text | -| user_agent.os.version | Operating system version as a raw string. | keyword | -| user_agent.version | Version of the user agent. | keyword | ## Metrics reference @@ -373,70 +268,74 @@ An example event for `firewall` looks as following: ```json { "@timestamp": "2020-05-28T17:58:27.154Z", - "service": { - "type": "aws" - }, - "ecs": { - "version": "8.0.0" + "agent": { + "ephemeral_id": "d3f31d10-7f16-4834-ae22-0df946c61f92", + "hostname": "docker-fleet-agent", + "id": "88c94c53-cbfe-4657-9a08-527b09d94cee", + "name": "docker-fleet-agent", + "type": "metricbeat", + "version": "7.15.0" }, "aws": { + "cloudwatch": { + "namespace": "AWS/NetworkFirewall" + }, + "dimensions": { + "AvailabilityZone": "us-east-2a", + "Engine": "Stateful", + "FirewallName": "AWSNetworkFirewall" + }, "networkfirewall": { "metrics": { - "PassedPackets": { - "sum": 0 - }, "DroppedPackets": { "sum": 4 }, + "PassedPackets": { + "sum": 0 + }, "ReceivedPackets": { "sum": 4 } } - }, - "cloudwatch": { - "namespace": "AWS/NetworkFirewall" - }, - "dimensions": { - "FirewallName": "AWSNetworkFirewall", - "AvailabilityZone": "us-east-2a", - "Engine": "Stateful" } }, - "event": { - "duration": 8925713800, - "agent_id_status": "verified", - "ingested": "2021-11-18T17:18:46Z", - "module": "aws", - "dataset": "aws.firewall_metrics" - }, - "metricset": { - "period": 60000, - "name": "cloudwatch" - }, "cloud": { - "provider": "aws", - "region": "us-east-2", "account": { - "name": "elastic-beats", - "id": "428152502467" - } + "id": "428152502467", + "name": "elastic-beats" + }, + "provider": "aws", + "region": "us-east-2" }, "data_stream": { + "dataset": "aws.firewall_metrics", "namespace": "default", - "type": "metrics", - "dataset": "aws.firewall_metrics" + "type": "metrics" }, - "agent": { - "hostname": "docker-fleet-agent", - "name": "docker-fleet-agent", - "id": "88c94c53-cbfe-4657-9a08-527b09d94cee", - "type": "metricbeat", - "ephemeral_id": "d3f31d10-7f16-4834-ae22-0df946c61f92", - "version": "7.15.0" + "ecs": { + "version": "8.11.0" + }, + "event": { + "agent_id_status": "verified", + "dataset": "aws.firewall_metrics", + "duration": 8925713800, + "ingested": "2021-11-18T17:18:46Z", + "module": "aws" + }, + "metricset": { + "name": "cloudwatch", + "period": 60000 + }, + "service": { + "type": "aws" } } ``` +**ECS Field Reference** + +Please refer to the following [document](https://www.elastic.co/guide/en/ecs/current/ecs-field-reference.html) for detailed information on ECS fields. + **Exported fields** | Field | Description | Type | Metric Type | @@ -453,44 +352,13 @@ An example event for `firewall` looks as following: | aws.networkfirewall.metrics.PassedPackets.sum | The number of packets passed by the Network Firewall. | long | gauge | | aws.networkfirewall.metrics.ReceivedPackets.sum | The number of packets received by the Network Firewall. | long | gauge | | aws.tags | Tag key value pairs from aws resources. | flattened | | -| cloud | Fields related to the cloud or infrastructure the events are coming from. | group | | | cloud.account.id | The cloud account or organization id used to identify different entities in a multi-tenant environment. Examples: AWS account id, Google Cloud ORG Id, or other unique identifier. | keyword | | -| cloud.account.name | The cloud account name or alias used to identify different entities in a multi-tenant environment. Examples: AWS account name, Google Cloud ORG display name. | keyword | | -| cloud.availability_zone | Availability zone in which this host, resource, or service is located. | keyword | | | cloud.image.id | Image ID for the cloud instance. | keyword | | -| cloud.instance.id | Instance ID of the host machine. | keyword | | -| cloud.instance.name | Instance name of the host machine. | keyword | | -| cloud.machine.type | Machine type of the host machine. | keyword | | -| cloud.project.id | The cloud project identifier. Examples: Google Cloud Project id, Azure Project id. | keyword | | -| cloud.provider | Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. | keyword | | | cloud.region | Region in which this host, resource, or service is located. | keyword | | -| container.id | Unique container id. | keyword | | -| container.image.name | Name of the image the container was built on. | keyword | | -| container.labels | Image labels. | object | | -| container.name | Container name. | keyword | | | data_stream.dataset | Data stream dataset. | constant_keyword | | | data_stream.namespace | Data stream namespace. | constant_keyword | | | data_stream.type | Data stream type. | constant_keyword | | -| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | | -| error | These fields can represent errors of any kind. Use them for errors that happen while fetching events or in cases where the event itself contains an error. | group | | -| error.message | Error message. | match_only_text | | -| event.dataset | Name of the dataset. If an event source publishes more than one type of log or events (e.g. access log, error log), the dataset is used to specify which one the event comes from. It's recommended but not required to start the dataset name with the module name, followed by a dot, then the dataset name. | constant_keyword | | | event.module | Event module | constant_keyword | | -| host.architecture | Operating system architecture. | keyword | | | host.containerized | If the host is a container. | boolean | | -| host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | | -| host.hostname | Hostname of the host. It normally contains what the `hostname` command returns on the host machine. | keyword | | -| host.id | Unique host id. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`. | keyword | | -| host.ip | Host ip addresses. | ip | | -| host.mac | Host MAC addresses. The notation format from RFC 7042 is suggested: Each octet (that is, 8-bit byte) is represented by two [uppercase] hexadecimal digits giving the value of the octet as an unsigned integer. Successive octets are separated by a hyphen. | keyword | | -| host.name | Name of the host. It can contain what hostname returns on Unix systems, the fully qualified domain name (FQDN), or a name specified by the user. The recommended value is the lowercase FQDN of the host. | keyword | | | host.os.build | OS build information. | keyword | | | host.os.codename | OS codename, if any. | keyword | | -| host.os.family | OS family (such as redhat, debian, freebsd, windows). | keyword | | -| host.os.kernel | Operating system kernel version as a raw string. | keyword | | -| host.os.name | Operating system name, without the version. | keyword | | -| host.os.name.text | Multi-field of `host.os.name`. | match_only_text | | -| host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword | | -| host.os.version | Operating system version as a raw string. | keyword | | -| host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | | -| service.type | The type of the service data is collected from. The type can be used to group and correlate logs and metrics from one service type. Example: If logs or metrics are collected from Elasticsearch, `service.type` would be `elasticsearch`. | keyword | | diff --git a/packages/aws/docs/guardduty.md b/packages/aws/docs/guardduty.md index 904ed3f3e25..d9ae103a48a 100644 --- a/packages/aws/docs/guardduty.md +++ b/packages/aws/docs/guardduty.md @@ -229,7 +229,7 @@ An example event for `guardduty` looks as following: "type": "logs" }, "ecs": { - "version": "8.10.0" + "version": "8.11.0" }, "elastic_agent": { "id": "9e5875f3-d206-43b3-b24e-5a5096e50846", @@ -326,6 +326,10 @@ An example event for `guardduty` looks as following: } ``` +**ECS Field Reference** + +Please refer to the following [document](https://www.elastic.co/guide/en/ecs/current/ecs-field-reference.html) for detailed information on ECS fields. + **Exported fields** | Field | Description | Type | @@ -581,90 +585,14 @@ An example event for `guardduty` looks as following: | aws.guardduty.title | The title of the finding. | keyword | | aws.guardduty.type | The type of finding. | keyword | | aws.guardduty.updated_at | The time and date when the finding was last updated. | date | -| cloud.account.id | The cloud account or organization id used to identify different entities in a multi-tenant environment. Examples: AWS account id, Google Cloud ORG Id, or other unique identifier. | keyword | -| cloud.account.name | The cloud account name or alias used to identify different entities in a multi-tenant environment. Examples: AWS account name, Google Cloud ORG display name. | keyword | -| cloud.availability_zone | Availability zone in which this host, resource, or service is located. | keyword | | cloud.image.id | Image ID for the cloud instance. | keyword | -| cloud.instance.id | Instance ID of the host machine. | keyword | -| cloud.instance.name | Instance name of the host machine. | keyword | -| cloud.machine.type | Machine type of the host machine. | keyword | -| cloud.project.id | The cloud project identifier. Examples: Google Cloud Project id, Azure Project id. | keyword | -| cloud.provider | Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. | keyword | -| cloud.region | Region in which this host, resource, or service is located. | keyword | -| cloud.service.name | The cloud service name is intended to distinguish services running on different platforms within a provider, eg AWS EC2 vs Lambda, GCP GCE vs App Engine, Azure VM vs App Server. Examples: app engine, app service, cloud run, fargate, lambda. | keyword | -| container.id | Unique container id. | keyword | -| container.image.name | Name of the image the container was built on. | keyword | -| container.labels | Image labels. | object | -| container.name | Container name. | keyword | -| container.runtime | Runtime managing this container. | keyword | -| container.security_context.privileged | Indicates whether the container is running in privileged mode. | boolean | | data_stream.dataset | Data stream dataset. | constant_keyword | | data_stream.namespace | Data stream namespace. | constant_keyword | | data_stream.type | Data stream type. | constant_keyword | -| destination.address | Some event destination addresses are defined ambiguously. The event will sometimes list an IP, a domain or a unix socket. You should always store the raw address in the `.address` field. Then it should be duplicated to `.ip` or `.domain`, depending on which one it is. | keyword | -| dns.question.name | The name being queried. If the name field contains non-printable characters (below 32 or above 126), those characters should be represented as escaped base 10 integers (\DDD). Back slashes and quotes should be escaped. Tabs, carriage returns, and line feeds should be converted to \t, \r, and \n respectively. | keyword | -| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | -| event.action | The action captured by the event. This describes the information in the event. It is more specific than `event.category`. Examples are `group-add`, `process-started`, `file-created`. The value is normally defined by the implementer. | keyword | -| event.created | `event.created` contains the date/time when the event was first read by an agent, or by your pipeline. This field is distinct from `@timestamp` in that `@timestamp` typically contain the time extracted from the original event. In most situations, these two timestamps will be slightly different. The difference can be used to calculate the delay between your source generating an event, and the time when your agent first processed it. This can be used to monitor your agent's or pipeline's ability to keep up with your event source. In case the two timestamps are identical, `@timestamp` should be used. | date | -| event.dataset | Name of the dataset. If an event source publishes more than one type of log or events (e.g. access log, error log), the dataset is used to specify which one the event comes from. It's recommended but not required to start the dataset name with the module name, followed by a dot, then the dataset name. | constant_keyword | -| event.end | `event.end` contains the date when the event ended or when the activity was last observed. | date | -| event.id | Unique ID to describe the event. | keyword | -| event.kind | This is one of four ECS Categorization Fields, and indicates the highest level in the ECS category hierarchy. `event.kind` gives high-level information about what type of information the event contains, without being specific to the contents of the event. For example, values of this field distinguish alert events from metric events. The value of this field can be used to inform how these kinds of events should be handled. They may warrant different retention, different access control, it may also help understand whether the data is coming in at a regular interval or not. | keyword | | event.module | Event module. | constant_keyword | -| event.original | Raw text message of entire event. Used to demonstrate log integrity or where the full log message (before splitting it up in multiple parts) may be required, e.g. for reindex. This field is not indexed and doc_values are disabled. It cannot be searched, but it can be retrieved from `_source`. If users wish to override this and index this field, please see `Field data types` in the `Elasticsearch Reference`. | keyword | -| event.provider | Source of the event. Event transports such as Syslog or the Windows Event Log typically mention the source of an event. It can be the name of the software that generated the event (e.g. Sysmon, httpd), or of a subsystem of the operating system (kernel, Microsoft-Windows-Security-Auditing). | keyword | -| event.severity | The numeric severity of the event according to your event source. What the different severity values mean can be different between sources and use cases. It's up to the implementer to make sure severities are consistent across events from the same source. The Syslog severity belongs in `log.syslog.severity.code`. `event.severity` is meant to represent the severity according to the event source (e.g. firewall, IDS). If the event source does not publish its own severity, you may optionally copy the `log.syslog.severity.code` to `event.severity`. | long | -| event.start | `event.start` contains the date when the event started or when the activity was first observed. | date | -| event.type | This is one of four ECS Categorization Fields, and indicates the third level in the ECS category hierarchy. `event.type` represents a categorization "sub-bucket" that, when used along with the `event.category` field values, enables filtering events down to a level appropriate for single visualization. This field is an array. This will allow proper categorization of some events that fall in multiple event types. | keyword | -| file.hash.sha256 | SHA256 hash. | keyword | -| file.name | Name of the file including the extension, without the directory. | keyword | -| file.path | Full path to the file, including the file name. It should include the drive letter, when appropriate. | keyword | -| file.path.text | Multi-field of `file.path`. | match_only_text | -| host.architecture | Operating system architecture. | keyword | | host.containerized | If the host is a container. | boolean | -| host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | -| host.hostname | Hostname of the host. It normally contains what the `hostname` command returns on the host machine. | keyword | -| host.id | Unique host id. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`. | keyword | -| host.ip | Host ip addresses. | ip | -| host.mac | Host MAC addresses. The notation format from RFC 7042 is suggested: Each octet (that is, 8-bit byte) is represented by two [uppercase] hexadecimal digits giving the value of the octet as an unsigned integer. Successive octets are separated by a hyphen. | keyword | -| host.name | Name of the host. It can contain what hostname returns on Unix systems, the fully qualified domain name (FQDN), or a name specified by the user. The recommended value is the lowercase FQDN of the host. | keyword | | host.os.build | OS build information. | keyword | | host.os.codename | OS codename, if any. | keyword | -| host.os.family | OS family (such as redhat, debian, freebsd, windows). | keyword | -| host.os.kernel | Operating system kernel version as a raw string. | keyword | -| host.os.name | Operating system name, without the version. | keyword | -| host.os.name.text | Multi-field of `host.os.name`. | match_only_text | -| host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword | -| host.os.version | Operating system version as a raw string. | keyword | -| host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | | input.type | Input type | keyword | | log.offset | Log offset | long | -| message | For log events the message field contains the log message, optimized for viewing in a log viewer. For structured logs without an original message field, other fields can be concatenated to form a human-readable summary of the event. If multiple messages exist, they can be combined into one message. | match_only_text | -| network.direction | Direction of the network traffic. When mapping events from a host-based monitoring context, populate this field from the host's point of view, using the values "ingress" or "egress". When mapping events from a network or perimeter-based monitoring context, populate this field from the point of view of the network perimeter, using the values "inbound", "outbound", "internal" or "external". Note that "internal" is not crossing perimeter boundaries, and is meant to describe communication between two hosts within the perimeter. Note also that "external" is meant to describe traffic between two hosts that are external to the perimeter. This could for example be useful for ISPs or VPN service providers. | keyword | -| network.transport | Same as network.iana_number, but instead using the Keyword name of the transport layer (udp, tcp, ipv6-icmp, etc.) The field value must be normalized to lowercase for querying. | keyword | -| orchestrator.namespace | Namespace in which the action is taking place. | keyword | -| orchestrator.resource.name | Name of the resource being acted upon. | keyword | -| orchestrator.resource.type | Type of resource being acted upon. | keyword | -| related.hash | All the hashes seen on your event. Populating this field, then using it to search for hashes can help in situations where you're unsure what the hash algorithm is (and therefore which key name to search). | keyword | -| related.hosts | All hostnames or other host identifiers seen on your event. Example identifiers include FQDNs, domain names, workstation names, or aliases. | keyword | -| related.ip | All of the IPs seen on your event. | ip | -| related.user | All the user names or other user identifiers seen on the event. | keyword | -| rule.category | A categorization value keyword used by the entity using the rule for detection of this event. | keyword | -| rule.name | The name of the rule or signature generating the event. | keyword | -| rule.ruleset | Name of the ruleset, policy, group, or parent category in which the rule used to generate this event is a member. | keyword | -| source.address | Some event source addresses are defined ambiguously. The event will sometimes list an IP, a domain or a unix socket. You should always store the raw address in the `.address` field. Then it should be duplicated to `.ip` or `.domain`, depending on which one it is. | keyword | -| source.as.number | Unique number allocated to the autonomous system. The autonomous system number (ASN) uniquely identifies each network on the Internet. | long | -| source.as.organization.name | Organization name. | keyword | -| source.as.organization.name.text | Multi-field of `source.as.organization.name`. | match_only_text | -| source.geo.city_name | City name. | keyword | -| source.geo.country_iso_code | Country ISO code. | keyword | -| source.geo.country_name | Country name. | keyword | -| source.geo.location | Longitude and latitude. | geo_point | -| source.ip | IP address of the source (IPv4 or IPv6). | ip | -| source.port | Port of the source. | long | -| tags | List of keywords used to tag each event. | keyword | -| user.id | Unique identifier of the user. | keyword | -| user.name | Short name or login of the user. | keyword | -| user.name.text | Multi-field of `user.name`. | match_only_text | -| user.roles | Array of user roles at the time of the event. | keyword | diff --git a/packages/aws/docs/inspector.md b/packages/aws/docs/inspector.md index 59171ed4823..ec83b96d28c 100644 --- a/packages/aws/docs/inspector.md +++ b/packages/aws/docs/inspector.md @@ -197,7 +197,7 @@ An example event for `inspector` looks as following: "type": "logs" }, "ecs": { - "version": "8.2.0" + "version": "8.11.0" }, "elastic_agent": { "id": "4a3373c9-b63f-4544-a929-761b42f50054", @@ -256,6 +256,10 @@ An example event for `inspector` looks as following: } ``` +**ECS Field Reference** + +Please refer to the following [document](https://www.elastic.co/guide/en/ecs/current/ecs-field-reference.html) for detailed information on ECS fields. + **Exported fields** | Field | Description | Type | @@ -341,58 +345,13 @@ An example event for `inspector` looks as following: | aws.inspector.title | The title of the finding. | keyword | | aws.inspector.type | The type of the finding. | keyword | | aws.inspector.updated_at | The date and time the finding was last updated at. | date | -| cloud.account.id | The cloud account or organization id used to identify different entities in a multi-tenant environment. Examples: AWS account id, Google Cloud ORG Id, or other unique identifier. | keyword | -| cloud.account.name | The cloud account name or alias used to identify different entities in a multi-tenant environment. Examples: AWS account name, Google Cloud ORG display name. | keyword | -| cloud.availability_zone | Availability zone in which this host, resource, or service is located. | keyword | | cloud.image.id | Image ID for the cloud instance. | keyword | -| cloud.instance.id | Instance ID of the host machine. | keyword | -| cloud.instance.name | Instance name of the host machine. | keyword | -| cloud.machine.type | Machine type of the host machine. | keyword | -| cloud.project.id | The cloud project identifier. Examples: Google Cloud Project id, Azure Project id. | keyword | -| cloud.provider | Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. | keyword | -| cloud.region | Region in which this host, resource, or service is located. | keyword | -| container.id | Unique container id. | keyword | -| container.image.name | Name of the image the container was built on. | keyword | -| container.labels | Image labels. | object | -| container.name | Container name. | keyword | | data_stream.dataset | Data stream dataset. | constant_keyword | | data_stream.namespace | Data stream namespace. | constant_keyword | | data_stream.type | Data stream type. | constant_keyword | -| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | -| event.category | This is one of four ECS Categorization Fields, and indicates the second level in the ECS category hierarchy. `event.category` represents the "big buckets" of ECS categories. For example, filtering on `event.category:process` yields all events relating to process activity. This field is closely related to `event.type`, which is used as a subcategory. This field is an array. This will allow proper categorization of some events that fall in multiple categories. | keyword | -| event.created | `event.created` contains the date/time when the event was first read by an agent, or by your pipeline. This field is distinct from `@timestamp` in that `@timestamp` typically contain the time extracted from the original event. In most situations, these two timestamps will be slightly different. The difference can be used to calculate the delay between your source generating an event, and the time when your agent first processed it. This can be used to monitor your agent's or pipeline's ability to keep up with your event source. In case the two timestamps are identical, `@timestamp` should be used. | date | -| event.dataset | Name of the dataset. If an event source publishes more than one type of log or events (e.g. access log, error log), the dataset is used to specify which one the event comes from. It's recommended but not required to start the dataset name with the module name, followed by a dot, then the dataset name. | constant_keyword | -| event.kind | This is one of four ECS Categorization Fields, and indicates the highest level in the ECS category hierarchy. `event.kind` gives high-level information about what type of information the event contains, without being specific to the contents of the event. For example, values of this field distinguish alert events from metric events. The value of this field can be used to inform how these kinds of events should be handled. They may warrant different retention, different access control, it may also help understand whether the data is coming in at a regular interval or not. | keyword | | event.module | Event module. | constant_keyword | -| event.original | Raw text message of entire event. Used to demonstrate log integrity or where the full log message (before splitting it up in multiple parts) may be required, e.g. for reindex. This field is not indexed and doc_values are disabled. It cannot be searched, but it can be retrieved from `_source`. If users wish to override this and index this field, please see `Field data types` in the `Elasticsearch Reference`. | keyword | -| event.severity | The numeric severity of the event according to your event source. What the different severity values mean can be different between sources and use cases. It's up to the implementer to make sure severities are consistent across events from the same source. The Syslog severity belongs in `log.syslog.severity.code`. `event.severity` is meant to represent the severity according to the event source (e.g. firewall, IDS). If the event source does not publish its own severity, you may optionally copy the `log.syslog.severity.code` to `event.severity`. | long | -| event.type | This is one of four ECS Categorization Fields, and indicates the third level in the ECS category hierarchy. `event.type` represents a categorization "sub-bucket" that, when used along with the `event.category` field values, enables filtering events down to a level appropriate for single visualization. This field is an array. This will allow proper categorization of some events that fall in multiple event types. | keyword | -| host.architecture | Operating system architecture. | keyword | | host.containerized | If the host is a container. | boolean | -| host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | -| host.hostname | Hostname of the host. It normally contains what the `hostname` command returns on the host machine. | keyword | -| host.id | Unique host id. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`. | keyword | -| host.ip | Host ip addresses. | ip | -| host.mac | Host MAC addresses. The notation format from RFC 7042 is suggested: Each octet (that is, 8-bit byte) is represented by two [uppercase] hexadecimal digits giving the value of the octet as an unsigned integer. Successive octets are separated by a hyphen. | keyword | -| host.name | Name of the host. It can contain what hostname returns on Unix systems, the fully qualified domain name (FQDN), or a name specified by the user. The recommended value is the lowercase FQDN of the host. | keyword | | host.os.build | OS build information. | keyword | | host.os.codename | OS codename, if any. | keyword | -| host.os.family | OS family (such as redhat, debian, freebsd, windows). | keyword | -| host.os.kernel | Operating system kernel version as a raw string. | keyword | -| host.os.name | Operating system name, without the version. | keyword | -| host.os.name.text | Multi-field of `host.os.name`. | match_only_text | -| host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword | -| host.os.version | Operating system version as a raw string. | keyword | -| host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | | input.type | Input type | keyword | | log.offset | Log offset | long | -| message | For log events the message field contains the log message, optimized for viewing in a log viewer. For structured logs without an original message field, other fields can be concatenated to form a human-readable summary of the event. If multiple messages exist, they can be combined into one message. | match_only_text | -| network.transport | Same as network.iana_number, but instead using the Keyword name of the transport layer (udp, tcp, ipv6-icmp, etc.) The field value must be normalized to lowercase for querying. | keyword | -| related.hash | All the hashes seen on your event. Populating this field, then using it to search for hashes can help in situations where you're unsure what the hash algorithm is (and therefore which key name to search). | keyword | -| related.ip | All of the IPs seen on your event. | ip | -| tags | List of keywords used to tag each event. | keyword | -| vulnerability.id | The identification (ID) is the number portion of a vulnerability entry. It includes a unique identification number for the vulnerability. For example (https://cve.mitre.org/about/faqs.html#what_is_cve_id)[Common Vulnerabilities and Exposure CVE ID] | keyword | -| vulnerability.reference | A resource that provides additional information, context, and mitigations for the identified vulnerability. | keyword | -| vulnerability.score.base | Scores can range from 0.0 to 10.0, with 10.0 being the most severe. Base scores cover an assessment for exploitability metrics (attack vector, complexity, privileges, and user interaction), impact metrics (confidentiality, integrity, and availability), and scope. For example (https://www.first.org/cvss/specification-document) | float | -| vulnerability.score.version | The National Vulnerability Database (NVD) provides qualitative severity rankings of "Low", "Medium", and "High" for CVSS v2.0 base score ranges in addition to the severity ratings for CVSS v3.0 as they are defined in the CVSS v3.0 specification. CVSS is owned and managed by FIRST.Org, Inc. (FIRST), a US-based non-profit organization, whose mission is to help computer security incident response teams across the world. For example (https://nvd.nist.gov/vuln-metrics/cvss) | keyword | -| vulnerability.severity | The severity of the vulnerability can help with metrics and internal prioritization regarding remediation. For example (https://nvd.nist.gov/vuln-metrics/cvss) | keyword | diff --git a/packages/aws/docs/kafka.md b/packages/aws/docs/kafka.md index 842e6c5e99b..6765b9a324a 100644 --- a/packages/aws/docs/kafka.md +++ b/packages/aws/docs/kafka.md @@ -92,7 +92,7 @@ An example event for `kafka` looks as following: "type": "metrics" }, "ecs": { - "version": "8.0.0" + "version": "8.11.0" }, "elastic_agent": { "id": "0395c9d5-9ac1-4ecc-bfd5-fc5376847519", @@ -138,6 +138,10 @@ An example event for `kafka` looks as following: } ``` +**ECS Field Reference** + +Please refer to the following [document](https://www.elastic.co/guide/en/ecs/current/ecs-field-reference.html) for detailed information on ECS fields. + **Exported fields** | Field | Description | Type | Metric Type | @@ -196,45 +200,14 @@ An example event for `kafka` looks as following: | aws.kafka.metrics.UnderReplicatedPartitions.avg | The average number of under-replicated partitions for the broker. | long | gauge | | aws.kafka.metrics.ZooKeeperRequestLatencyMsMean.avg | The mean latency in milliseconds for Apache ZooKeeper requests from broker. | long | gauge | | aws.tags | Tag key value pairs from aws resources. | flattened | | -| cloud | Fields related to the cloud or infrastructure the events are coming from. | group | | | cloud.account.id | The cloud account or organization id used to identify different entities in a multi-tenant environment. Examples: AWS account id, Google Cloud ORG Id, or other unique identifier. | keyword | | -| cloud.account.name | The cloud account name or alias used to identify different entities in a multi-tenant environment. Examples: AWS account name, Google Cloud ORG display name. | keyword | | -| cloud.availability_zone | Availability zone in which this host, resource, or service is located. | keyword | | | cloud.image.id | Image ID for the cloud instance. | keyword | | -| cloud.instance.id | Instance ID of the host machine. | keyword | | -| cloud.instance.name | Instance name of the host machine. | keyword | | -| cloud.machine.type | Machine type of the host machine. | keyword | | -| cloud.project.id | The cloud project identifier. Examples: Google Cloud Project id, Azure Project id. | keyword | | -| cloud.provider | Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. | keyword | | | cloud.region | Region in which this host, resource, or service is located. | keyword | | -| container.id | Unique container id. | keyword | | -| container.image.name | Name of the image the container was built on. | keyword | | -| container.labels | Image labels. | object | | -| container.name | Container name. | keyword | | | data_stream.dataset | Data stream dataset. | constant_keyword | | | data_stream.namespace | Data stream namespace. | constant_keyword | | | data_stream.type | Data stream type. | constant_keyword | | -| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | | -| error | These fields can represent errors of any kind. Use them for errors that happen while fetching events or in cases where the event itself contains an error. | group | | -| error.message | Error message. | match_only_text | | -| event.dataset | Name of the dataset. If an event source publishes more than one type of log or events (e.g. access log, error log), the dataset is used to specify which one the event comes from. It's recommended but not required to start the dataset name with the module name, followed by a dot, then the dataset name. | constant_keyword | | | event.module | Event module | constant_keyword | | -| host.architecture | Operating system architecture. | keyword | | | host.containerized | If the host is a container. | boolean | | -| host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | | -| host.hostname | Hostname of the host. It normally contains what the `hostname` command returns on the host machine. | keyword | | -| host.id | Unique host id. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`. | keyword | | -| host.ip | Host ip addresses. | ip | | -| host.mac | Host MAC addresses. The notation format from RFC 7042 is suggested: Each octet (that is, 8-bit byte) is represented by two [uppercase] hexadecimal digits giving the value of the octet as an unsigned integer. Successive octets are separated by a hyphen. | keyword | | -| host.name | Name of the host. It can contain what hostname returns on Unix systems, the fully qualified domain name (FQDN), or a name specified by the user. The recommended value is the lowercase FQDN of the host. | keyword | | | host.os.build | OS build information. | keyword | | | host.os.codename | OS codename, if any. | keyword | | -| host.os.family | OS family (such as redhat, debian, freebsd, windows). | keyword | | -| host.os.kernel | Operating system kernel version as a raw string. | keyword | | -| host.os.name | Operating system name, without the version. | keyword | | -| host.os.name.text | Multi-field of `host.os.name`. | match_only_text | | -| host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword | | -| host.os.version | Operating system version as a raw string. | keyword | | -| host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | | -| service.type | The type of the service data is collected from. The type can be used to group and correlate logs and metrics from one service type. Example: If logs or metrics are collected from Elasticsearch, `service.type` would be `elasticsearch`. | keyword | | diff --git a/packages/aws/docs/kinesis.md b/packages/aws/docs/kinesis.md index 5a6658acb7a..813ab9d787c 100644 --- a/packages/aws/docs/kinesis.md +++ b/packages/aws/docs/kinesis.md @@ -45,40 +45,12 @@ An example event for `kinesis` looks as following: { "@timestamp": "2022-07-27T20:56:00.000Z", "agent": { - "name": "docker-fleet-agent", - "id": "2d4b09d0-cdb6-445e-ac3f-6415f87b9864", "ephemeral_id": "51866723-6dfa-4a72-a68e-f439d5de7f53", + "id": "2d4b09d0-cdb6-445e-ac3f-6415f87b9864", + "name": "docker-fleet-agent", "type": "metricbeat", "version": "8.3.2" }, - "elastic_agent": { - "id": "2d4b09d0-cdb6-445e-ac3f-6415f87b9864", - "version": "8.3.2", - "snapshot": false - }, - "cloud": { - "provider": "aws", - "region": "us-east-1", - "account": { - "name": "elastic-beats", - "id": "428152502467" - } - }, - "ecs": { - "version": "8.0.0" - }, - "data_stream": { - "namespace": "default", - "type": "metrics", - "dataset": "aws.kinesis" - }, - "service": { - "type": "aws" - }, - "metricset": { - "period": 300000, - "name": "cloudwatch" - }, "aws": { "cloudwatch": { "namespace": "AWS/Kinesis" @@ -109,16 +81,48 @@ An example event for `kinesis` looks as following: } } }, + "cloud": { + "account": { + "id": "428152502467", + "name": "elastic-beats" + }, + "provider": "aws", + "region": "us-east-1" + }, + "data_stream": { + "dataset": "aws.kinesis", + "namespace": "default", + "type": "metrics" + }, + "ecs": { + "version": "8.11.0" + }, + "elastic_agent": { + "id": "2d4b09d0-cdb6-445e-ac3f-6415f87b9864", + "snapshot": false, + "version": "8.3.2" + }, "event": { - "duration": 10483932100, "agent_id_status": "verified", + "dataset": "aws.kinesis", + "duration": 10483932100, "ingested": "2022-07-27T20:56:00.000Z", - "module": "aws", - "dataset": "aws.kinesis" + "module": "aws" + }, + "metricset": { + "name": "cloudwatch", + "period": 300000 + }, + "service": { + "type": "aws" } } ``` +**ECS Field Reference** + +Please refer to the following [document](https://www.elastic.co/guide/en/ecs/current/ecs-field-reference.html) for detailed information on ECS fields. + **Exported fields** | Field | Description | Type | Metric Type | @@ -153,44 +157,13 @@ An example event for `kinesis` looks as following: | aws.kinesis.metrics.SubscribeToShard_Success.avg | This metric records whether the SubscribeToShard subscription was successfully established. | long | gauge | | aws.kinesis.metrics.WriteProvisionedThroughputExceeded.avg | The number of records rejected due to throttling for the stream over the specified time period. This metric includes throttling from PutRecord and PutRecords operations. | long | gauge | | aws.tags | Tag key value pairs from aws resources. | flattened | | -| cloud | Fields related to the cloud or infrastructure the events are coming from. | group | | | cloud.account.id | The cloud account or organization id used to identify different entities in a multi-tenant environment. Examples: AWS account id, Google Cloud ORG Id, or other unique identifier. | keyword | | -| cloud.account.name | The cloud account name or alias used to identify different entities in a multi-tenant environment. Examples: AWS account name, Google Cloud ORG display name. | keyword | | -| cloud.availability_zone | Availability zone in which this host, resource, or service is located. | keyword | | | cloud.image.id | Image ID for the cloud instance. | keyword | | -| cloud.instance.id | Instance ID of the host machine. | keyword | | -| cloud.instance.name | Instance name of the host machine. | keyword | | -| cloud.machine.type | Machine type of the host machine. | keyword | | -| cloud.project.id | The cloud project identifier. Examples: Google Cloud Project id, Azure Project id. | keyword | | -| cloud.provider | Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. | keyword | | | cloud.region | Region in which this host, resource, or service is located. | keyword | | -| container.id | Unique container id. | keyword | | -| container.image.name | Name of the image the container was built on. | keyword | | -| container.labels | Image labels. | object | | -| container.name | Container name. | keyword | | | data_stream.dataset | Data stream dataset. | constant_keyword | | | data_stream.namespace | Data stream namespace. | constant_keyword | | | data_stream.type | Data stream type. | constant_keyword | | -| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | | -| error | These fields can represent errors of any kind. Use them for errors that happen while fetching events or in cases where the event itself contains an error. | group | | -| error.message | Error message. | match_only_text | | -| event.dataset | Name of the dataset. If an event source publishes more than one type of log or events (e.g. access log, error log), the dataset is used to specify which one the event comes from. It's recommended but not required to start the dataset name with the module name, followed by a dot, then the dataset name. | constant_keyword | | | event.module | Event module | constant_keyword | | -| host.architecture | Operating system architecture. | keyword | | | host.containerized | If the host is a container. | boolean | | -| host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | | -| host.hostname | Hostname of the host. It normally contains what the `hostname` command returns on the host machine. | keyword | | -| host.id | Unique host id. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`. | keyword | | -| host.ip | Host ip addresses. | ip | | -| host.mac | Host MAC addresses. The notation format from RFC 7042 is suggested: Each octet (that is, 8-bit byte) is represented by two [uppercase] hexadecimal digits giving the value of the octet as an unsigned integer. Successive octets are separated by a hyphen. | keyword | | -| host.name | Name of the host. It can contain what hostname returns on Unix systems, the fully qualified domain name (FQDN), or a name specified by the user. The recommended value is the lowercase FQDN of the host. | keyword | | | host.os.build | OS build information. | keyword | | | host.os.codename | OS codename, if any. | keyword | | -| host.os.family | OS family (such as redhat, debian, freebsd, windows). | keyword | | -| host.os.kernel | Operating system kernel version as a raw string. | keyword | | -| host.os.name | Operating system name, without the version. | keyword | | -| host.os.name.text | Multi-field of `host.os.name`. | match_only_text | | -| host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword | | -| host.os.version | Operating system version as a raw string. | keyword | | -| host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | | -| service.type | The type of the service data is collected from. The type can be used to group and correlate logs and metrics from one service type. Example: If logs or metrics are collected from Elasticsearch, `service.type` would be `elasticsearch`. | keyword | | diff --git a/packages/aws/docs/lambda.md b/packages/aws/docs/lambda.md index 51ca4147dba..07747dbef8e 100644 --- a/packages/aws/docs/lambda.md +++ b/packages/aws/docs/lambda.md @@ -45,77 +45,81 @@ An example event for `lambda` looks as following: { "@timestamp": "2022-07-19T22:40:00.000Z", "agent": { - "name": "docker-fleet-agent", + "ephemeral_id": "ed2abfa1-df5e-4c3e-9c2b-143edcc0e111", "id": "2d4b09d0-cdb6-445e-ac3f-6415f87b9864", + "name": "docker-fleet-agent", "type": "metricbeat", - "ephemeral_id": "ed2abfa1-df5e-4c3e-9c2b-143edcc0e111", "version": "8.3.2" }, - "elastic_agent": { - "id": "2d4b09d0-cdb6-445e-ac3f-6415f87b9864", - "version": "8.3.2", - "snapshot": false - }, - "cloud": { - "provider": "aws", - "region": "eu-central-1", - "account": { - "name": "elastic-observability", - "id": "627286350134" - } - }, - "ecs": { - "version": "8.0.0" - }, - "service": { - "type": "aws" - }, - "data_stream": { - "namespace": "default", - "type": "metrics", - "dataset": "aws.lambda" - }, - "metricset": { - "period": 300000, - "name": "cloudwatch" - }, "aws": { + "cloudwatch": { + "namespace": "AWS/Lambda" + }, "lambda": { "metrics": { - "Errors": { - "avg": 0 - }, "ConcurrentExecutions": { "avg": 1 }, - "Invocations": { - "avg": 1 - }, - "UnreservedConcurrentExecutions": { - "avg": 1 - }, "Duration": { "avg": 130.97 }, + "Errors": { + "avg": 0 + }, + "Invocations": { + "avg": 1 + }, "Throttles": { "avg": 0 + }, + "UnreservedConcurrentExecutions": { + "avg": 1 } } - }, - "cloudwatch": { - "namespace": "AWS/Lambda" } }, + "cloud": { + "account": { + "id": "627286350134", + "name": "elastic-observability" + }, + "provider": "aws", + "region": "eu-central-1" + }, + "data_stream": { + "dataset": "aws.lambda", + "namespace": "default", + "type": "metrics" + }, + "ecs": { + "version": "8.11.0" + }, + "elastic_agent": { + "id": "2d4b09d0-cdb6-445e-ac3f-6415f87b9864", + "snapshot": false, + "version": "8.3.2" + }, "event": { - "duration": 11364562400, "agent_id_status": "verified", + "dataset": "aws.lambda", + "duration": 11364562400, "ingested": "2022-07-26T22:40:40Z", - "module": "aws", - "dataset": "aws.lambda" + "module": "aws" + }, + "metricset": { + "name": "cloudwatch", + "period": 300000 + }, + "service": { + "type": "aws" } } ``` +**ECS Field Reference** + +Please refer to the following [document](https://www.elastic.co/guide/en/ecs/current/ecs-field-reference.html) for detailed information on ECS fields. + **Exported fields** | Field | Description | Type | Unit | Metric Type | @@ -147,44 +151,13 @@ An example event for `lambda` looks as following: | aws.lambda.metrics.Throttles.sum | The total number of invocation requests that are throttled. | double | | gauge | | aws.lambda.metrics.UnreservedConcurrentExecutions.avg | For an AWS Region, the average number of events that are being processed by functions that don't have reserved concurrency. | double | | gauge | | aws.tags | Tag key value pairs from aws resources. | flattened | | | -| cloud | Fields related to the cloud or infrastructure the events are coming from. | group | | | | cloud.account.id | The cloud account or organization id used to identify different entities in a multi-tenant environment. Examples: AWS account id, Google Cloud ORG Id, or other unique identifier. | keyword | | | -| cloud.account.name | The cloud account name or alias used to identify different entities in a multi-tenant environment. Examples: AWS account name, Google Cloud ORG display name. | keyword | | | -| cloud.availability_zone | Availability zone in which this host, resource, or service is located. | keyword | | | | cloud.image.id | Image ID for the cloud instance. | keyword | | | -| cloud.instance.id | Instance ID of the host machine. | keyword | | | -| cloud.instance.name | Instance name of the host machine. | keyword | | | -| cloud.machine.type | Machine type of the host machine. | keyword | | | -| cloud.project.id | The cloud project identifier. Examples: Google Cloud Project id, Azure Project id. | keyword | | | -| cloud.provider | Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. | keyword | | | | cloud.region | Region in which this host, resource, or service is located. | keyword | | | -| container.id | Unique container id. | keyword | | | -| container.image.name | Name of the image the container was built on. | keyword | | | -| container.labels | Image labels. | object | | | -| container.name | Container name. | keyword | | | | data_stream.dataset | Data stream dataset. | constant_keyword | | | | data_stream.namespace | Data stream namespace. | constant_keyword | | | | data_stream.type | Data stream type. | constant_keyword | | | -| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | | | -| error | These fields can represent errors of any kind. Use them for errors that happen while fetching events or in cases where the event itself contains an error. | group | | | -| error.message | Error message. | match_only_text | | | -| event.dataset | Name of the dataset. If an event source publishes more than one type of log or events (e.g. access log, error log), the dataset is used to specify which one the event comes from. It's recommended but not required to start the dataset name with the module name, followed by a dot, then the dataset name. | constant_keyword | | | | event.module | Event module | constant_keyword | | | -| host.architecture | Operating system architecture. | keyword | | | | host.containerized | If the host is a container. | boolean | | | -| host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | | | -| host.hostname | Hostname of the host. It normally contains what the `hostname` command returns on the host machine. | keyword | | | -| host.id | Unique host id. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`. | keyword | | | -| host.ip | Host ip addresses. | ip | | | -| host.mac | Host MAC addresses. The notation format from RFC 7042 is suggested: Each octet (that is, 8-bit byte) is represented by two [uppercase] hexadecimal digits giving the value of the octet as an unsigned integer. Successive octets are separated by a hyphen. | keyword | | | -| host.name | Name of the host. It can contain what hostname returns on Unix systems, the fully qualified domain name (FQDN), or a name specified by the user. The recommended value is the lowercase FQDN of the host. | keyword | | | | host.os.build | OS build information. | keyword | | | | host.os.codename | OS codename, if any. | keyword | | | -| host.os.family | OS family (such as redhat, debian, freebsd, windows). | keyword | | | -| host.os.kernel | Operating system kernel version as a raw string. | keyword | | | -| host.os.name | Operating system name, without the version. | keyword | | | -| host.os.name.text | Multi-field of `host.os.name`. | match_only_text | | | -| host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword | | | -| host.os.version | Operating system version as a raw string. | keyword | | | -| host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | | | -| service.type | The type of the service data is collected from. The type can be used to group and correlate logs and metrics from one service type. Example: If logs or metrics are collected from Elasticsearch, `service.type` would be `elasticsearch`. | keyword | | | diff --git a/packages/aws/docs/natgateway.md b/packages/aws/docs/natgateway.md index 2eec60fe6d9..7ae1b230fe0 100644 --- a/packages/aws/docs/natgateway.md +++ b/packages/aws/docs/natgateway.md @@ -43,123 +43,127 @@ An example event for `natgateway` looks as following: ```json { + "@timestamp": "2022-07-27T22:02:00.000Z", "agent": { - "name": "a3fc2d7bc1c5", - "id": "8940152e-2f20-4ad1-bc96-4db45cb7fc89", "ephemeral_id": "b7f3d3f4-137a-443f-90a7-ad2a5d81f81b", - "type": "metricbeat", - "version": "8.1.0" - }, - "elastic_agent": { "id": "8940152e-2f20-4ad1-bc96-4db45cb7fc89", - "version": "8.1.0", - "snapshot": false - }, - "cloud": { - "provider": "aws", - "region": "eu-west-1" - }, - "@timestamp": "2022-07-27T22:02:00.000Z", - "ecs": { - "version": "8.0.0" - }, - "service": { - "type": "aws" - }, - "data_stream": { - "namespace": "default", - "type": "metrics", - "dataset": "aws.natgateway" - }, - "host": { - "hostname": "a3fc2d7bc1c5", - "os": { - "kernel": "5.10.104-linuxkit", - "codename": "focal", - "name": "Ubuntu", - "type": "linux", - "family": "debian", - "version": "20.04.3 LTS (Focal Fossa)", - "platform": "ubuntu" - }, - "containerized": false, - "ip": [ - "172.20.0.7" - ], "name": "a3fc2d7bc1c5", - "mac": [ - "02-42-AC-14-00-07" - ], - "architecture": "aarch64" - }, - "metricset": { - "period": 180000, - "name": "cloudwatch" + "type": "metricbeat", + "version": "8.1.0" }, "aws": { "cloudwatch": { "namespace": "AWS/NATGateway" }, + "dimensions": { + "NatGatewayId": "nat-038389b5fc0734aa0" + }, "natgateway": { "metrics": { - "PacketsInFromSource": { - "sum": 421 - }, - "ErrorPortAllocation": { - "sum": 0 + "ActiveConnectionCount": { + "max": 0 }, - "PacketsOutToDestination": { - "sum": 421 + "BytesInFromDestination": { + "sum": 164752 }, - "PacketsOutToSource": { - "sum": 472 + "BytesInFromSource": { + "sum": 42505 }, "BytesOutToDestination": { "sum": 42505 }, - "ConnectionEstablishedCount": { - "sum": 23 + "BytesOutToSource": { + "sum": 164752 }, "ConnectionAttemptCount": { "sum": 23 }, - "PacketsInFromDestination": { - "sum": 472 + "ConnectionEstablishedCount": { + "sum": 23 }, - "BytesInFromDestination": { - "sum": 164752 + "ErrorPortAllocation": { + "sum": 0 + }, + "IdleTimeoutCount": { + "sum": 0 }, "PacketsDropCount": { "sum": 0 }, - "BytesInFromSource": { - "sum": 42505 + "PacketsInFromDestination": { + "sum": 472 }, - "BytesOutToSource": { - "sum": 164752 + "PacketsInFromSource": { + "sum": 421 }, - "IdleTimeoutCount": { - "sum": 0 + "PacketsOutToDestination": { + "sum": 421 }, - "ActiveConnectionCount": { - "max": 0 + "PacketsOutToSource": { + "sum": 472 } } - }, - "dimensions": { - "NatGatewayId": "nat-038389b5fc0734aa0" } }, + "cloud": { + "provider": "aws", + "region": "eu-west-1" + }, + "data_stream": { + "dataset": "aws.natgateway", + "namespace": "default", + "type": "metrics" + }, + "ecs": { + "version": "8.11.0" + }, + "elastic_agent": { + "id": "8940152e-2f20-4ad1-bc96-4db45cb7fc89", + "snapshot": false, + "version": "8.1.0" + }, "event": { - "duration": 612193833, "agent_id_status": "verified", + "dataset": "aws.natgateway", + "duration": 612193833, "ingested": "2022-07-27T22:05:27Z", - "module": "aws", - "dataset": "aws.natgateway" + "module": "aws" + }, + "host": { + "architecture": "aarch64", + "containerized": false, + "hostname": "a3fc2d7bc1c5", + "ip": [ + "172.20.0.7" + ], + "mac": [ + "02-42-AC-14-00-07" + ], + "name": "a3fc2d7bc1c5", + "os": { + "codename": "focal", + "family": "debian", + "kernel": "5.10.104-linuxkit", + "name": "Ubuntu", + "platform": "ubuntu", + "type": "linux", + "version": "20.04.3 LTS (Focal Fossa)" + } + }, + "metricset": { + "name": "cloudwatch", + "period": 180000 + }, + "service": { + "type": "aws" } } ``` +**ECS Field Reference** + +Please refer to the following [document](https://www.elastic.co/guide/en/ecs/current/ecs-field-reference.html) for detailed information on ECS fields. + **Exported fields** | Field | Description | Type | Metric Type | @@ -183,44 +187,13 @@ An example event for `natgateway` looks as following: | aws.natgateway.metrics.PacketsOutToDestination.sum | The number of packets sent out through the NAT gateway to the destination. | long | gauge | | aws.natgateway.metrics.PacketsOutToSource.sum | The number of packets sent through the NAT gateway to the clients in your VPC. | long | gauge | | aws.tags | Tag key value pairs from aws resources. | flattened | | -| cloud | Fields related to the cloud or infrastructure the events are coming from. | group | | | cloud.account.id | The cloud account or organization id used to identify different entities in a multi-tenant environment. Examples: AWS account id, Google Cloud ORG Id, or other unique identifier. | keyword | | -| cloud.account.name | The cloud account name or alias used to identify different entities in a multi-tenant environment. Examples: AWS account name, Google Cloud ORG display name. | keyword | | -| cloud.availability_zone | Availability zone in which this host, resource, or service is located. | keyword | | | cloud.image.id | Image ID for the cloud instance. | keyword | | -| cloud.instance.id | Instance ID of the host machine. | keyword | | -| cloud.instance.name | Instance name of the host machine. | keyword | | -| cloud.machine.type | Machine type of the host machine. | keyword | | -| cloud.project.id | The cloud project identifier. Examples: Google Cloud Project id, Azure Project id. | keyword | | -| cloud.provider | Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. | keyword | | | cloud.region | Region in which this host, resource, or service is located. | keyword | | -| container.id | Unique container id. | keyword | | -| container.image.name | Name of the image the container was built on. | keyword | | -| container.labels | Image labels. | object | | -| container.name | Container name. | keyword | | | data_stream.dataset | Data stream dataset. | constant_keyword | | | data_stream.namespace | Data stream namespace. | constant_keyword | | | data_stream.type | Data stream type. | constant_keyword | | -| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | | -| error | These fields can represent errors of any kind. Use them for errors that happen while fetching events or in cases where the event itself contains an error. | group | | -| error.message | Error message. | match_only_text | | -| event.dataset | Name of the dataset. If an event source publishes more than one type of log or events (e.g. access log, error log), the dataset is used to specify which one the event comes from. It's recommended but not required to start the dataset name with the module name, followed by a dot, then the dataset name. | constant_keyword | | | event.module | Event module | constant_keyword | | -| host.architecture | Operating system architecture. | keyword | | | host.containerized | If the host is a container. | boolean | | -| host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | | -| host.hostname | Hostname of the host. It normally contains what the `hostname` command returns on the host machine. | keyword | | -| host.id | Unique host id. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`. | keyword | | -| host.ip | Host ip addresses. | ip | | -| host.mac | Host MAC addresses. The notation format from RFC 7042 is suggested: Each octet (that is, 8-bit byte) is represented by two [uppercase] hexadecimal digits giving the value of the octet as an unsigned integer. Successive octets are separated by a hyphen. | keyword | | -| host.name | Name of the host. It can contain what hostname returns on Unix systems, the fully qualified domain name (FQDN), or a name specified by the user. The recommended value is the lowercase FQDN of the host. | keyword | | | host.os.build | OS build information. | keyword | | | host.os.codename | OS codename, if any. | keyword | | -| host.os.family | OS family (such as redhat, debian, freebsd, windows). | keyword | | -| host.os.kernel | Operating system kernel version as a raw string. | keyword | | -| host.os.name | Operating system name, without the version. | keyword | | -| host.os.name.text | Multi-field of `host.os.name`. | match_only_text | | -| host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword | | -| host.os.version | Operating system version as a raw string. | keyword | | -| host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | | -| service.type | The type of the service data is collected from. The type can be used to group and correlate logs and metrics from one service type. Example: If logs or metrics are collected from Elasticsearch, `service.type` would be `elasticsearch`. | keyword | | diff --git a/packages/aws/docs/rds.md b/packages/aws/docs/rds.md index 970f7f08f9b..88df051c165 100644 --- a/packages/aws/docs/rds.md +++ b/packages/aws/docs/rds.md @@ -44,50 +44,21 @@ An example event for `rds` looks as following: ```json { "@timestamp": "2022-06-03T15:28:00.000Z", - "ecs": { - "version": "8.0.0" - }, "agent": { - "name": "docker-fleet-agent", + "ephemeral_id": "c4161c81-1e2e-4e8b-a0be-15940cc13226", "id": "90bfb41e-b925-420f-973e-9c1115297278", + "name": "docker-fleet-agent", "type": "metricbeat", - "ephemeral_id": "c4161c81-1e2e-4e8b-a0be-15940cc13226", "version": "8.2.0" }, - "elastic_agent": { - "id": "90bfb41e-b925-420f-973e-9c1115297278", - "version": "8.2.0", - "snapshot": false - }, - "cloud": { - "provider": "aws", - "region": "eu-west-1", - "account": { - "name": "elastic-beats", - "id": "123456789" - } - }, - "data_stream": { - "namespace": "default", - "type": "metrics", - "dataset": "aws.rds" - }, - "service": { - "type": "aws" - }, - "metricset": { - "period": 60000, - "name": "cloudwatch" - }, "aws": { + "cloudwatch": { + "namespace": "AWS/RDS" + }, + "dimensions": { + "DatabaseClass": "db.r5.large" + }, "rds": { - "cache_hit_ratio": { - "buffer": 100, - "result_set": 0 - }, - "aurora_volume_left_total": { - "bytes": 70007366615040 - }, "aurora_bin_log_replica_lag": 0, "aurora_replica": { "lag": { @@ -100,132 +71,165 @@ An example event for `rds` looks as following: "ms": 19.469999313354492 } }, - "latency": { - "dml": 0.09705000000000001, - "read": 0, - "select": 0.2412933510638298, - "commit": 3.536983333333333, - "insert": 0.09705000000000001, - "update": 0, - "ddl": 0, - "write": 0.0006218917818574514, - "delete": 0 - }, - "swap_usage": { - "bytes": 0 + "aurora_volume_left_total": { + "bytes": 70007366615040 }, - "transactions": { - "blocked": 0, - "active": 0 + "cache_hit_ratio": { + "buffer": 100, + "result_set": 0 }, - "queries": 7.737700770575286, "database_connections": 0, + "deadlocks": 0, + "engine_uptime": { + "sec": 53016926.5 + }, "free_local_storage": { "bytes": 28622428160 }, - "login_failures": 0, - "engine_uptime": { - "sec": 53016926.5 + "freeable_memory": { + "bytes": 4705378304 }, + "latency": { + "commit": 3.536983333333333, + "ddl": 0, + "delete": 0, + "dml": 0.09705000000000001, + "insert": 0.09705000000000001, + "read": 0, + "select": 0.2412933510638298, + "update": 0, + "write": 0.0006218917818574514 + }, + "login_failures": 0, "metrics": { - "Aurora_pq_request_not_chosen_below_min_rows": { + "AbortedClients": { "avg": 0 }, - "RowLockTime": { + "Aurora_pq_request_attempted": { "avg": 0 }, - "RollbackSegmentHistoryListLength": { - "avg": 53 + "Aurora_pq_request_executed": { + "avg": 0 }, - "SumBinaryLogSize": { + "Aurora_pq_request_failed": { "avg": 0 }, - "Aurora_pq_request_not_chosen_pq_high_buffer_pool_pct": { + "Aurora_pq_request_in_progress": { "avg": 0 }, - "StorageNetworkThroughput": { - "avg": 22950.537520958267 + "Aurora_pq_request_not_chosen": { + "avg": 0 }, - "Aurora_pq_request_not_chosen_few_pages_outside_buffer_pool": { + "Aurora_pq_request_not_chosen_below_min_rows": { "avg": 0 }, - "Aurora_pq_request_not_chosen_small_table": { + "Aurora_pq_request_not_chosen_few_pages_outside_buffer_pool": { "avg": 0 }, - "StorageNetworkReceiveThroughput": { - "avg": 7104.272100353031 + "Aurora_pq_request_not_chosen_long_trx": { + "avg": 0 }, - "AbortedClients": { + "Aurora_pq_request_not_chosen_pq_high_buffer_pool_pct": { "avg": 0 }, - "Aurora_pq_request_executed": { + "Aurora_pq_request_not_chosen_small_table": { "avg": 0 }, "Aurora_pq_request_not_chosen_unsupported_access": { "avg": 0 }, - "Aurora_pq_request_not_chosen_long_trx": { + "Aurora_pq_request_throttled": { "avg": 0 }, "ConnectionAttempts": { "avg": 0 }, - "Aurora_pq_request_failed": { - "avg": 0 - }, "NumBinaryLogFiles": { "avg": 0 }, - "Aurora_pq_request_not_chosen": { - "avg": 0 + "RollbackSegmentHistoryListLength": { + "avg": 53 }, - "Aurora_pq_request_in_progress": { + "RowLockTime": { "avg": 0 }, - "Aurora_pq_request_throttled": { - "avg": 0 + "StorageNetworkReceiveThroughput": { + "avg": 7104.272100353031 + }, + "StorageNetworkThroughput": { + "avg": 22950.537520958267 }, "StorageNetworkTransmitThroughput": { "avg": 15846.26542060524 }, - "Aurora_pq_request_attempted": { + "SumBinaryLogSize": { "avg": 0 } }, + "queries": 7.737700770575286, + "swap_usage": { + "bytes": 0 + }, "throughput": { - "dml": 0.2500125006250313, - "select": 2.9051419389878808, - "network_transmit": 0.7020888516985455, - "network_receive": 0.7020888516985455, "commit": 0.2500125006250313, - "insert": 0.2500125006250313, - "update": 0, - "delete": 0, "ddl": 0, - "network": 1.404177703397091 + "delete": 0, + "dml": 0.2500125006250313, + "insert": 0.2500125006250313, + "network": 1.404177703397091, + "network_receive": 0.7020888516985455, + "network_transmit": 0.7020888516985455, + "select": 2.9051419389878808, + "update": 0 }, - "deadlocks": 0, - "freeable_memory": { - "bytes": 4705378304 + "transactions": { + "active": 0, + "blocked": 0 } - }, - "cloudwatch": { - "namespace": "AWS/RDS" - }, - "dimensions": { - "DatabaseClass": "db.r5.large" } }, + "cloud": { + "account": { + "id": "123456789", + "name": "elastic-beats" + }, + "provider": "aws", + "region": "eu-west-1" + }, + "data_stream": { + "dataset": "aws.rds", + "namespace": "default", + "type": "metrics" + }, + "ecs": { + "version": "8.11.0" + }, + "elastic_agent": { + "id": "90bfb41e-b925-420f-973e-9c1115297278", + "snapshot": false, + "version": "8.2.0" + }, "event": { - "duration": 12570787900, "agent_id_status": "verified", + "dataset": "aws.rds", + "duration": 12570787900, "ingested": "2022-06-03T15:28:44Z", - "module": "aws", - "dataset": "aws.rds" + "module": "aws" + }, + "metricset": { + "name": "cloudwatch", + "period": 60000 + }, + "service": { + "type": "aws" } } ``` +**ECS Field Reference** + +Please refer to the following [document](https://www.elastic.co/guide/en/ecs/current/ecs-field-reference.html) for detailed information on ECS fields. + **Exported fields** | Field | Description | Type | Metric Type | @@ -315,44 +319,13 @@ An example event for `rds` looks as following: | aws.rds.volume_used.bytes | The amount of storage used by your Aurora DB instance, in bytes. | long | gauge | | aws.rds.write_io.ops_per_sec | The average number of disk write I/O operations per second. | float | gauge | | aws.tags | Tag key value pairs from aws resources. | flattened | | -| cloud | Fields related to the cloud or infrastructure the events are coming from. | group | | | cloud.account.id | The cloud account or organization id used to identify different entities in a multi-tenant environment. Examples: AWS account id, Google Cloud ORG Id, or other unique identifier. | keyword | | -| cloud.account.name | The cloud account name or alias used to identify different entities in a multi-tenant environment. Examples: AWS account name, Google Cloud ORG display name. | keyword | | -| cloud.availability_zone | Availability zone in which this host, resource, or service is located. | keyword | | | cloud.image.id | Image ID for the cloud instance. | keyword | | -| cloud.instance.id | Instance ID of the host machine. | keyword | | -| cloud.instance.name | Instance name of the host machine. | keyword | | -| cloud.machine.type | Machine type of the host machine. | keyword | | -| cloud.project.id | The cloud project identifier. Examples: Google Cloud Project id, Azure Project id. | keyword | | -| cloud.provider | Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. | keyword | | | cloud.region | Region in which this host, resource, or service is located. | keyword | | -| container.id | Unique container id. | keyword | | -| container.image.name | Name of the image the container was built on. | keyword | | -| container.labels | Image labels. | object | | -| container.name | Container name. | keyword | | | data_stream.dataset | Data stream dataset. | constant_keyword | | | data_stream.namespace | Data stream namespace. | constant_keyword | | | data_stream.type | Data stream type. | constant_keyword | | -| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | | -| error | These fields can represent errors of any kind. Use them for errors that happen while fetching events or in cases where the event itself contains an error. | group | | -| error.message | Error message. | match_only_text | | -| event.dataset | Name of the dataset. If an event source publishes more than one type of log or events (e.g. access log, error log), the dataset is used to specify which one the event comes from. It's recommended but not required to start the dataset name with the module name, followed by a dot, then the dataset name. | constant_keyword | | | event.module | Event module | constant_keyword | | -| host.architecture | Operating system architecture. | keyword | | | host.containerized | If the host is a container. | boolean | | -| host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | | -| host.hostname | Hostname of the host. It normally contains what the `hostname` command returns on the host machine. | keyword | | -| host.id | Unique host id. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`. | keyword | | -| host.ip | Host ip addresses. | ip | | -| host.mac | Host MAC addresses. The notation format from RFC 7042 is suggested: Each octet (that is, 8-bit byte) is represented by two [uppercase] hexadecimal digits giving the value of the octet as an unsigned integer. Successive octets are separated by a hyphen. | keyword | | -| host.name | Name of the host. It can contain what hostname returns on Unix systems, the fully qualified domain name (FQDN), or a name specified by the user. The recommended value is the lowercase FQDN of the host. | keyword | | | host.os.build | OS build information. | keyword | | | host.os.codename | OS codename, if any. | keyword | | -| host.os.family | OS family (such as redhat, debian, freebsd, windows). | keyword | | -| host.os.kernel | Operating system kernel version as a raw string. | keyword | | -| host.os.name | Operating system name, without the version. | keyword | | -| host.os.name.text | Multi-field of `host.os.name`. | match_only_text | | -| host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword | | -| host.os.version | Operating system version as a raw string. | keyword | | -| host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | | -| service.type | The type of the service data is collected from. The type can be used to group and correlate logs and metrics from one service type. Example: If logs or metrics are collected from Elasticsearch, `service.type` would be `elasticsearch`. | keyword | | diff --git a/packages/aws/docs/redshift.md b/packages/aws/docs/redshift.md index e36b6a41d55..c322efb90ed 100644 --- a/packages/aws/docs/redshift.md +++ b/packages/aws/docs/redshift.md @@ -131,7 +131,7 @@ An example event for `redshift` looks as following: "type": "metrics" }, "ecs": { - "version": "8.0.0" + "version": "8.11.0" }, "elastic_agent": { "id": "d745bccd-73a3-41b4-9fd0-4d9bac14f77b", @@ -176,6 +176,10 @@ An example event for `redshift` looks as following: } ``` +**ECS Field Reference** + +Please refer to the following [document](https://www.elastic.co/guide/en/ecs/current/ecs-field-reference.html) for detailed information on ECS fields. + **Exported fields** | Field | Description | Type | Metric Type | @@ -224,33 +228,11 @@ An example event for `redshift` looks as following: | aws.redshift.metrics.WriteLatency.avg | The average amount of time taken for disk write I/O operations. | long | gauge | | aws.redshift.metrics.WriteThroughput.avg | The average number of bytes written to disk per second. | long | gauge | | aws.tags | Tag key value pairs from aws resources. | flattened | | -| cloud | Fields related to the cloud or infrastructure the events are coming from. | group | | | cloud.account.id | The cloud account or organization id used to identify different entities in a multi-tenant environment. Examples: AWS account id, Google Cloud ORG Id, or other unique identifier. | keyword | | -| cloud.account.name | The cloud account name or alias used to identify different entities in a multi-tenant environment. Examples: AWS account name, Google Cloud ORG display name. | keyword | | -| cloud.availability_zone | Availability zone in which this host, resource, or service is located. | keyword | | -| cloud.instance.id | Instance ID of the host machine. | keyword | | -| cloud.machine.type | Machine type of the host machine. | keyword | | -| cloud.provider | Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. | keyword | | | cloud.region | Region in which this host, resource, or service is located. | keyword | | | container | Container fields are used for meta information about the specific container that is the source of information. These fields help correlate data based containers from any runtime. | group | | -| container.id | Unique container id. | keyword | | -| container.image.name | Name of the image the container was built on. | keyword | | -| container.labels | Image labels. | object | | -| container.name | Container name. | keyword | | | data_stream.dataset | Data stream dataset. | constant_keyword | | | data_stream.namespace | Data stream namespace. | constant_keyword | | | data_stream.type | Data stream type. | constant_keyword | | -| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | | -| error | These fields can represent errors of any kind. Use them for errors that happen while fetching events or in cases where the event itself contains an error. | group | | -| error.message | Error message. | match_only_text | | -| event.dataset | Name of the dataset. If an event source publishes more than one type of log or events (e.g. access log, error log), the dataset is used to specify which one the event comes from. It's recommended but not required to start the dataset name with the module name, followed by a dot, then the dataset name. | constant_keyword | | | event.module | Event module | constant_keyword | | | host | A host is defined as a general computing instance. ECS host.\* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes. | group | | -| host.architecture | Operating system architecture. | keyword | | -| host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | | -| host.id | Unique host id. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`. | keyword | | -| host.ip | Host ip addresses. | ip | | -| host.mac | Host MAC addresses. The notation format from RFC 7042 is suggested: Each octet (that is, 8-bit byte) is represented by two [uppercase] hexadecimal digits giving the value of the octet as an unsigned integer. Successive octets are separated by a hyphen. | keyword | | -| host.name | Name of the host. It can contain what hostname returns on Unix systems, the fully qualified domain name (FQDN), or a name specified by the user. The recommended value is the lowercase FQDN of the host. | keyword | | -| host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | | -| service.type | The type of the service data is collected from. The type can be used to group and correlate logs and metrics from one service type. Example: If logs or metrics are collected from Elasticsearch, `service.type` would be `elasticsearch`. | keyword | | diff --git a/packages/aws/docs/route53.md b/packages/aws/docs/route53.md index fa16995fe32..f264b41a66c 100644 --- a/packages/aws/docs/route53.md +++ b/packages/aws/docs/route53.md @@ -71,64 +71,76 @@ An example event for `route53_public` looks as following: ```json { - "awscloudwatch": { - "log_group": "test", - "ingestion_time": "2021-12-06T02:18:20.000Z", - "log_stream": "test" - }, + "@timestamp": "2017-12-13T08:16:05.744Z", "agent": { - "name": "docker-fleet-agent", + "ephemeral_id": "1cf87179-f6b3-44b0-a46f-3aa6bc0f995f", "id": "c00f804f-7a02-441b-88f4-aeb9da6410d9", + "name": "docker-fleet-agent", "type": "filebeat", - "ephemeral_id": "1cf87179-f6b3-44b0-a46f-3aa6bc0f995f", "version": "8.0.0" }, - "elastic_agent": { - "id": "c00f804f-7a02-441b-88f4-aeb9da6410d9", - "version": "8.0.0", - "snapshot": true + "aws": { + "route53": { + "edge_location": "JFK5", + "hosted_zone_id": "Z123412341234" + } + }, + "awscloudwatch": { + "ingestion_time": "2021-12-06T02:18:20.000Z", + "log_group": "test", + "log_stream": "test" + }, + "cloud": { + "provider": "aws", + "region": "us-east-1" + }, + "data_stream": { + "dataset": "aws.route53_public_logs", + "namespace": "default", + "type": "logs" }, "dns": { - "response_code": "NOERROR", "question": { - "registered_domain": "example.com", - "top_level_domain": "com", "name": "txt.example.com", + "registered_domain": "example.com", "subdomain": "txt", + "top_level_domain": "com", "type": "TXT" - } - }, - "source": { - "as": { - "number": 721, - "organization": { - "name": "DoD Network Information Center" - } }, - "address": "55.36.5.7", - "ip": "55.36.5.7" + "response_code": "NOERROR" }, - "tags": [ - "preserve_original_event", - "forwarded", - "aws-route53-logs" - ], - "network": { - "protocol": "dns", - "transport": "udp", - "type": "ipv4", - "iana_number": "17" + "ecs": { + "version": "8.11.0" }, - "cloud": { - "provider": "aws", - "region": "us-east-1" + "elastic_agent": { + "id": "c00f804f-7a02-441b-88f4-aeb9da6410d9", + "snapshot": true, + "version": "8.0.0" + }, + "event": { + "agent_id_status": "verified", + "category": [ + "network" + ], + "dataset": "aws.route53_public_logs", + "id": "36545504503447201576705984279898091551471012413796646912", + "ingested": "2021-12-06T02:37:25Z", + "kind": "event", + "original": "1.0 2017-12-13T08:16:05.744Z Z123412341234 txt.example.com TXT NOERROR UDP JFK5 55.36.5.7 -", + "outcome": "success", + "type": [ + "protocol" + ] }, "input": { "type": "aws-cloudwatch" }, - "@timestamp": "2017-12-13T08:16:05.744Z", - "ecs": { - "version": "8.0.0" + "log.file.path": "test/test", + "network": { + "iana_number": "17", + "protocol": "dns", + "transport": "udp", + "type": "ipv4" }, "related": { "hosts": [ @@ -138,36 +150,28 @@ An example event for `route53_public` looks as following: "55.36.5.7" ] }, - "data_stream": { - "namespace": "default", - "type": "logs", - "dataset": "aws.route53_public_logs" - }, - "log.file.path": "test/test", - "event": { - "agent_id_status": "verified", - "ingested": "2021-12-06T02:37:25Z", - "original": "1.0 2017-12-13T08:16:05.744Z Z123412341234 txt.example.com TXT NOERROR UDP JFK5 55.36.5.7 -", - "kind": "event", - "id": "36545504503447201576705984279898091551471012413796646912", - "category": [ - "network" - ], - "type": [ - "protocol" - ], - "dataset": "aws.route53_public_logs", - "outcome": "success" + "source": { + "address": "55.36.5.7", + "as": { + "number": 721, + "organization": { + "name": "DoD Network Information Center" + } + }, + "ip": "55.36.5.7" }, - "aws": { - "route53": { - "hosted_zone_id": "Z123412341234", - "edge_location": "JFK5" - } - } + "tags": [ + "preserve_original_event", + "forwarded", + "aws-route53-logs" + ] } ``` +**ECS Field Reference** + +Please refer to the following [document](https://www.elastic.co/guide/en/ecs/current/ecs-field-reference.html) for detailed information on ECS fields. + **Exported fields** | Field | Description | Type | @@ -180,71 +184,15 @@ An example event for `route53_public` looks as following: | awscloudwatch.ingestion_time | AWS CloudWatch ingest time | date | | awscloudwatch.log_group | AWS CloudWatch Log Group name | keyword | | awscloudwatch.log_stream | AWS CloudWatch Log Stream name | keyword | -| cloud.account.id | The cloud account or organization id used to identify different entities in a multi-tenant environment. Examples: AWS account id, Google Cloud ORG Id, or other unique identifier. | keyword | -| cloud.availability_zone | Availability zone in which this host, resource, or service is located. | keyword | | cloud.image.id | Image ID for the cloud instance. | keyword | -| cloud.instance.id | Instance ID of the host machine. | keyword | -| cloud.instance.name | Instance name of the host machine. | keyword | -| cloud.machine.type | Machine type of the host machine. | keyword | -| cloud.project.id | The cloud project identifier. Examples: Google Cloud Project id, Azure Project id. | keyword | -| cloud.provider | Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. | keyword | -| cloud.region | Region in which this host, resource, or service is located. | keyword | -| container.id | Unique container id. | keyword | -| container.image.name | Name of the image the container was built on. | keyword | -| container.labels | Image labels. | object | -| container.name | Container name. | keyword | | data_stream.dataset | Data stream dataset. | constant_keyword | | data_stream.namespace | Data stream namespace. | constant_keyword | | data_stream.type | Data stream type. | constant_keyword | -| dns.question.name | The name being queried. If the name field contains non-printable characters (below 32 or above 126), those characters should be represented as escaped base 10 integers (\DDD). Back slashes and quotes should be escaped. Tabs, carriage returns, and line feeds should be converted to \t, \r, and \n respectively. | keyword | -| dns.question.registered_domain | The highest registered domain, stripped of the subdomain. For example, the registered domain for "foo.example.com" is "example.com". This value can be determined precisely with a list like the public suffix list (http://publicsuffix.org). Trying to approximate this by simply taking the last two labels will not work well for TLDs such as "co.uk". | keyword | -| dns.question.subdomain | The subdomain is all of the labels under the registered_domain. If the domain has multiple levels of subdomain, such as "sub2.sub1.example.com", the subdomain field should contain "sub2.sub1", with no trailing period. | keyword | -| dns.question.top_level_domain | The effective top level domain (eTLD), also known as the domain suffix, is the last part of the domain name. For example, the top level domain for example.com is "com". This value can be determined precisely with a list like the public suffix list (http://publicsuffix.org). Trying to approximate this by simply taking the last label will not work well for effective TLDs such as "co.uk". | keyword | -| dns.question.type | The type of record being queried. | keyword | -| dns.response_code | The DNS response code. | keyword | -| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | -| error.message | Error message. | match_only_text | -| event.dataset | Name of the dataset. If an event source publishes more than one type of log or events (e.g. access log, error log), the dataset is used to specify which one the event comes from. It's recommended but not required to start the dataset name with the module name, followed by a dot, then the dataset name. | constant_keyword | | event.module | Event module | constant_keyword | -| host.architecture | Operating system architecture. | keyword | | host.containerized | If the host is a container. | boolean | -| host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | -| host.hostname | Hostname of the host. It normally contains what the `hostname` command returns on the host machine. | keyword | -| host.id | Unique host id. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`. | keyword | -| host.ip | Host ip addresses. | ip | -| host.mac | Host MAC addresses. The notation format from RFC 7042 is suggested: Each octet (that is, 8-bit byte) is represented by two [uppercase] hexadecimal digits giving the value of the octet as an unsigned integer. Successive octets are separated by a hyphen. | keyword | -| host.name | Name of the host. It can contain what hostname returns on Unix systems, the fully qualified domain name (FQDN), or a name specified by the user. The recommended value is the lowercase FQDN of the host. | keyword | | host.os.build | OS build information. | keyword | | host.os.codename | OS codename, if any. | keyword | -| host.os.family | OS family (such as redhat, debian, freebsd, windows). | keyword | -| host.os.kernel | Operating system kernel version as a raw string. | keyword | -| host.os.name | Operating system name, without the version. | keyword | -| host.os.name.text | Multi-field of `host.os.name`. | match_only_text | -| host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword | -| host.os.version | Operating system version as a raw string. | keyword | -| host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | | input.type | Type of Filebeat input. | keyword | -| log.file.path | Path to the log file. | keyword | -| message | For log events the message field contains the log message, optimized for viewing in a log viewer. For structured logs without an original message field, other fields can be concatenated to form a human-readable summary of the event. If multiple messages exist, they can be combined into one message. | match_only_text | -| network.iana_number | IANA Protocol Number (https://www.iana.org/assignments/protocol-numbers/protocol-numbers.xhtml). Standardized list of protocols. This aligns well with NetFlow and sFlow related logs which use the IANA Protocol Number. | keyword | -| network.protocol | In the OSI Model this would be the Application Layer protocol. For example, `http`, `dns`, or `ssh`. The field value must be normalized to lowercase for querying. | keyword | -| network.transport | Same as network.iana_number, but instead using the Keyword name of the transport layer (udp, tcp, ipv6-icmp, etc.) The field value must be normalized to lowercase for querying. | keyword | -| network.type | In the OSI Model this would be the Network Layer. ipv4, ipv6, ipsec, pim, etc The field value must be normalized to lowercase for querying. | keyword | -| related.hosts | All hostnames or other host identifiers seen on your event. Example identifiers include FQDNs, domain names, workstation names, or aliases. | keyword | -| related.ip | All of the IPs seen on your event. | ip | -| source.address | Some event source addresses are defined ambiguously. The event will sometimes list an IP, a domain or a unix socket. You should always store the raw address in the `.address` field. Then it should be duplicated to `.ip` or `.domain`, depending on which one it is. | keyword | -| source.as.number | Unique number allocated to the autonomous system. The autonomous system number (ASN) uniquely identifies each network on the Internet. | long | -| source.as.organization.name | Organization name. | keyword | -| source.as.organization.name.text | Multi-field of `source.as.organization.name`. | match_only_text | -| source.geo.city_name | City name. | keyword | -| source.geo.continent_name | Name of the continent. | keyword | -| source.geo.country_iso_code | Country ISO code. | keyword | -| source.geo.country_name | Country name. | keyword | -| source.geo.location | Longitude and latitude. | geo_point | -| source.geo.region_iso_code | Region ISO code. | keyword | -| source.geo.region_name | Region name. | keyword | -| source.ip | IP address of the source (IPv4 or IPv6). | ip | -| tags | List of keywords used to tag each event. | keyword | ### Resolver logs @@ -328,7 +276,7 @@ An example event for `route53_resolver` looks as following: "response_code": "NOERROR" }, "ecs": { - "version": "8.0.0" + "version": "8.11.0" }, "elastic_agent": { "id": "acba78ef-1401-4689-977c-d8c2e5d6a8fa", @@ -398,6 +346,10 @@ An example event for `route53_resolver` looks as following: } ``` +**ECS Field Reference** + +Please refer to the following [document](https://www.elastic.co/guide/en/ecs/current/ecs-field-reference.html) for detailed information on ECS fields. + **Exported fields** | Field | Description | Type | @@ -414,77 +366,13 @@ An example event for `route53_resolver` looks as following: | awscloudwatch.ingestion_time | AWS CloudWatch ingest time | date | | awscloudwatch.log_group | AWS CloudWatch Log Group name | keyword | | awscloudwatch.log_stream | AWS CloudWatch Log Stream name | keyword | -| cloud.account.id | The cloud account or organization id used to identify different entities in a multi-tenant environment. Examples: AWS account id, Google Cloud ORG Id, or other unique identifier. | keyword | -| cloud.availability_zone | Availability zone in which this host, resource, or service is located. | keyword | | cloud.image.id | Image ID for the cloud instance. | keyword | -| cloud.instance.id | Instance ID of the host machine. | keyword | -| cloud.instance.name | Instance name of the host machine. | keyword | -| cloud.machine.type | Machine type of the host machine. | keyword | -| cloud.project.id | The cloud project identifier. Examples: Google Cloud Project id, Azure Project id. | keyword | -| cloud.provider | Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. | keyword | -| cloud.region | Region in which this host, resource, or service is located. | keyword | -| container.id | Unique container id. | keyword | -| container.image.name | Name of the image the container was built on. | keyword | -| container.labels | Image labels. | object | -| container.name | Container name. | keyword | | data_stream.dataset | Data stream dataset. | constant_keyword | | data_stream.namespace | Data stream namespace. | constant_keyword | | data_stream.type | Data stream type. | constant_keyword | -| dns.answers | An array containing an object for each answer section returned by the server. The main keys that should be present in these objects are defined by ECS. Records that have more information may contain more keys than what ECS defines. Not all DNS data sources give all details about DNS answers. At minimum, answer objects must contain the `data` key. If more information is available, map as much of it to ECS as possible, and add any additional fields to the answer objects as custom fields. | group | -| dns.answers.class | The class of DNS data contained in this resource record. | keyword | -| dns.answers.data | The data describing the resource. The meaning of this data depends on the type and class of the resource record. | keyword | -| dns.answers.name | The domain name to which this resource record pertains. If a chain of CNAME is being resolved, each answer's `name` should be the one that corresponds with the answer's `data`. It should not simply be the original `question.name` repeated. | keyword | -| dns.answers.ttl | The time interval in seconds that this resource record may be cached before it should be discarded. Zero values mean that the data should not be cached. | long | -| dns.answers.type | The type of data contained in this resource record. | keyword | -| dns.question.class | The class of records being queried. | keyword | -| dns.question.name | The name being queried. If the name field contains non-printable characters (below 32 or above 126), those characters should be represented as escaped base 10 integers (\DDD). Back slashes and quotes should be escaped. Tabs, carriage returns, and line feeds should be converted to \t, \r, and \n respectively. | keyword | -| dns.question.registered_domain | The highest registered domain, stripped of the subdomain. For example, the registered domain for "foo.example.com" is "example.com". This value can be determined precisely with a list like the public suffix list (http://publicsuffix.org). Trying to approximate this by simply taking the last two labels will not work well for TLDs such as "co.uk". | keyword | -| dns.question.subdomain | The subdomain is all of the labels under the registered_domain. If the domain has multiple levels of subdomain, such as "sub2.sub1.example.com", the subdomain field should contain "sub2.sub1", with no trailing period. | keyword | -| dns.question.top_level_domain | The effective top level domain (eTLD), also known as the domain suffix, is the last part of the domain name. For example, the top level domain for example.com is "com". This value can be determined precisely with a list like the public suffix list (http://publicsuffix.org). Trying to approximate this by simply taking the last label will not work well for effective TLDs such as "co.uk". | keyword | -| dns.question.type | The type of record being queried. | keyword | -| dns.response_code | The DNS response code. | keyword | -| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | -| error.message | Error message. | match_only_text | -| event.dataset | Name of the dataset. If an event source publishes more than one type of log or events (e.g. access log, error log), the dataset is used to specify which one the event comes from. It's recommended but not required to start the dataset name with the module name, followed by a dot, then the dataset name. | constant_keyword | | event.module | Event module | constant_keyword | -| host.architecture | Operating system architecture. | keyword | | host.containerized | If the host is a container. | boolean | -| host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | -| host.hostname | Hostname of the host. It normally contains what the `hostname` command returns on the host machine. | keyword | -| host.id | Unique host id. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`. | keyword | -| host.ip | Host ip addresses. | ip | -| host.mac | Host MAC addresses. The notation format from RFC 7042 is suggested: Each octet (that is, 8-bit byte) is represented by two [uppercase] hexadecimal digits giving the value of the octet as an unsigned integer. Successive octets are separated by a hyphen. | keyword | -| host.name | Name of the host. It can contain what hostname returns on Unix systems, the fully qualified domain name (FQDN), or a name specified by the user. The recommended value is the lowercase FQDN of the host. | keyword | | host.os.build | OS build information. | keyword | | host.os.codename | OS codename, if any. | keyword | -| host.os.family | OS family (such as redhat, debian, freebsd, windows). | keyword | -| host.os.kernel | Operating system kernel version as a raw string. | keyword | -| host.os.name | Operating system name, without the version. | keyword | -| host.os.name.text | Multi-field of `host.os.name`. | match_only_text | -| host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword | -| host.os.version | Operating system version as a raw string. | keyword | -| host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | | input.type | Type of Filebeat input. | keyword | -| log.file.path | Path to the log file. | keyword | | log.offset | Log offset | long | -| message | For log events the message field contains the log message, optimized for viewing in a log viewer. For structured logs without an original message field, other fields can be concatenated to form a human-readable summary of the event. If multiple messages exist, they can be combined into one message. | match_only_text | -| network.iana_number | IANA Protocol Number (https://www.iana.org/assignments/protocol-numbers/protocol-numbers.xhtml). Standardized list of protocols. This aligns well with NetFlow and sFlow related logs which use the IANA Protocol Number. | keyword | -| network.protocol | In the OSI Model this would be the Application Layer protocol. For example, `http`, `dns`, or `ssh`. The field value must be normalized to lowercase for querying. | keyword | -| network.transport | Same as network.iana_number, but instead using the Keyword name of the transport layer (udp, tcp, ipv6-icmp, etc.) The field value must be normalized to lowercase for querying. | keyword | -| network.type | In the OSI Model this would be the Network Layer. ipv4, ipv6, ipsec, pim, etc The field value must be normalized to lowercase for querying. | keyword | -| related.hosts | All hostnames or other host identifiers seen on your event. Example identifiers include FQDNs, domain names, workstation names, or aliases. | keyword | -| related.ip | All of the IPs seen on your event. | ip | -| source.address | Some event source addresses are defined ambiguously. The event will sometimes list an IP, a domain or a unix socket. You should always store the raw address in the `.address` field. Then it should be duplicated to `.ip` or `.domain`, depending on which one it is. | keyword | -| source.as.number | Unique number allocated to the autonomous system. The autonomous system number (ASN) uniquely identifies each network on the Internet. | long | -| source.as.organization.name | Organization name. | keyword | -| source.as.organization.name.text | Multi-field of `source.as.organization.name`. | match_only_text | -| source.geo.city_name | City name. | keyword | -| source.geo.continent_name | Name of the continent. | keyword | -| source.geo.country_iso_code | Country ISO code. | keyword | -| source.geo.country_name | Country name. | keyword | -| source.geo.location | Longitude and latitude. | geo_point | -| source.geo.region_iso_code | Region ISO code. | keyword | -| source.geo.region_name | Region name. | keyword | -| source.ip | IP address of the source (IPv4 or IPv6). | ip | -| source.port | Port of the source. | long | -| tags | List of keywords used to tag each event. | keyword | diff --git a/packages/aws/docs/s3.md b/packages/aws/docs/s3.md index 996c77528d1..f2ec46e3618 100644 --- a/packages/aws/docs/s3.md +++ b/packages/aws/docs/s3.md @@ -51,6 +51,10 @@ Server access logs are useful for many applications. For example, access log information can be useful in security and access audits. It can also help users to learn about customer base and understand Amazon S3 bill. +**ECS Field Reference** + +Please refer to the following [document](https://www.elastic.co/guide/en/ecs/current/ecs-field-reference.html) for detailed information on ECS fields. + **Exported fields** | Field | Description | Type | @@ -82,43 +86,11 @@ to learn about customer base and understand Amazon S3 bill. | aws.s3access.turn_around_time | The total amount of time in milliseconds that Amazon S3 spent processing your request. | long | | aws.s3access.user_agent | The value of the HTTP User-Agent header. | keyword | | aws.s3access.version_id | The version ID in the request, or "-" if the operation does not take a versionId parameter. | keyword | -| client.address | Some event client addresses are defined ambiguously. The event will sometimes list an IP, a domain or a unix socket. You should always store the raw address in the `.address` field. Then it should be duplicated to `.ip` or `.domain`, depending on which one it is. | keyword | -| client.geo.city_name | City name. | keyword | -| client.geo.continent_name | Name of the continent. | keyword | -| client.geo.country_iso_code | Country ISO code. | keyword | -| client.geo.country_name | Country name. | keyword | -| client.geo.location | Longitude and latitude. | geo_point | -| client.geo.region_iso_code | Region ISO code. | keyword | -| client.geo.region_name | Region name. | keyword | -| client.ip | IP address of the client (IPv4 or IPv6). | ip | -| client.user.id | Unique identifier of the user. | keyword | -| cloud.account.id | The cloud account or organization id used to identify different entities in a multi-tenant environment. Examples: AWS account id, Google Cloud ORG Id, or other unique identifier. | keyword | -| cloud.account.name | The cloud account name or alias used to identify different entities in a multi-tenant environment. Examples: AWS account name, Google Cloud ORG display name. | keyword | -| cloud.availability_zone | Availability zone in which this host, resource, or service is located. | keyword | | cloud.image.id | Image ID for the cloud instance. | keyword | -| cloud.instance.id | Instance ID of the host machine. | keyword | -| cloud.instance.name | Instance name of the host machine. | keyword | -| cloud.machine.type | Machine type of the host machine. | keyword | -| cloud.project.id | The cloud project identifier. Examples: Google Cloud Project id, Azure Project id. | keyword | -| cloud.provider | Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. | keyword | -| cloud.region | Region in which this host, resource, or service is located. | keyword | -| container.id | Unique container id. | keyword | -| container.image.name | Name of the image the container was built on. | keyword | -| container.labels | Image labels. | object | -| container.name | Container name. | keyword | | data_stream.dataset | Data stream dataset. | constant_keyword | | data_stream.namespace | Data stream namespace. | constant_keyword | | data_stream.type | Data stream type. | constant_keyword | -| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | -| error.message | Error message. | match_only_text | -| event.action | The action captured by the event. This describes the information in the event. It is more specific than `event.category`. Examples are `group-add`, `process-started`, `file-created`. The value is normally defined by the implementer. | keyword | -| event.code | Identification code for this event, if one exists. Some event sources use event codes to identify messages unambiguously, regardless of message language or wording adjustments over time. An example of this is the Windows Event ID. | keyword | -| event.dataset | Name of the dataset. If an event source publishes more than one type of log or events (e.g. access log, error log), the dataset is used to specify which one the event comes from. It's recommended but not required to start the dataset name with the module name, followed by a dot, then the dataset name. | constant_keyword | -| event.duration | Duration of the event in nanoseconds. If `event.start` and `event.end` are known this value should be the difference between the end and start time. | long | -| event.id | Unique ID to describe the event. | keyword | -| event.kind | This is one of four ECS Categorization Fields, and indicates the highest level in the ECS category hierarchy. `event.kind` gives high-level information about what type of information the event contains, without being specific to the contents of the event. For example, values of this field distinguish alert events from metric events. The value of this field can be used to inform how these kinds of events should be handled. They may warrant different retention, different access control, it may also help understand whether the data is coming in at a regular interval or not. | keyword | | event.module | Event module | constant_keyword | -| event.outcome | This is one of four ECS Categorization Fields, and indicates the lowest level in the ECS category hierarchy. `event.outcome` simply denotes whether the event represents a success or a failure from the perspective of the entity that produced the event. Note that when a single transaction is described in multiple events, each event may populate different values of `event.outcome`, according to their perspective. Also note that in the case of a compound event (a single event that contains multiple logical events), this field should be populated with the value that best captures the overall success or failure from the perspective of the event producer. Further note that not all events will have an associated outcome. For example, this field is generally not populated for metric events, events with `event.type:info`, or any events for which an outcome does not make logical sense. | keyword | | geo.city_name | City name. | keyword | | geo.continent_name | Name of the continent. | keyword | | geo.country_iso_code | Country ISO code. | keyword | @@ -126,54 +98,11 @@ to learn about customer base and understand Amazon S3 bill. | geo.location | Longitude and latitude. | geo_point | | geo.region_iso_code | Region ISO name. | keyword | | geo.region_name | Region name. | keyword | -| host.architecture | Operating system architecture. | keyword | | host.containerized | If the host is a container. | boolean | -| host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | -| host.hostname | Hostname of the host. It normally contains what the `hostname` command returns on the host machine. | keyword | -| host.id | Unique host id. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`. | keyword | -| host.ip | Host ip addresses. | ip | -| host.mac | Host MAC addresses. The notation format from RFC 7042 is suggested: Each octet (that is, 8-bit byte) is represented by two [uppercase] hexadecimal digits giving the value of the octet as an unsigned integer. Successive octets are separated by a hyphen. | keyword | -| host.name | Name of the host. It can contain what hostname returns on Unix systems, the fully qualified domain name (FQDN), or a name specified by the user. The recommended value is the lowercase FQDN of the host. | keyword | | host.os.build | OS build information. | keyword | | host.os.codename | OS codename, if any. | keyword | -| host.os.family | OS family (such as redhat, debian, freebsd, windows). | keyword | -| host.os.kernel | Operating system kernel version as a raw string. | keyword | -| host.os.name | Operating system name, without the version. | keyword | -| host.os.name.text | Multi-field of `host.os.name`. | match_only_text | -| host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword | -| host.os.version | Operating system version as a raw string. | keyword | -| host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | -| http.request.method | HTTP request method. The value should retain its casing from the original event. For example, `GET`, `get`, and `GeT` are all considered valid values for this field. | keyword | -| http.request.referrer | Referrer for this HTTP request. | keyword | -| http.response.body.bytes | Size in bytes of the response body. | long | -| http.response.status_code | HTTP response status code. | long | -| http.version | HTTP version. | keyword | | input.type | Input type | keyword | -| log.file.path | Full path to the log file this event came from, including the file name. It should include the drive letter, when appropriate. If the event wasn't read from a log file, do not populate this field. | keyword | | log.offset | Log offset | long | -| related.ip | All of the IPs seen on your event. | ip | -| related.user | All the user names or other user identifiers seen on the event. | keyword | -| tags | List of keywords used to tag each event. | keyword | -| tls.cipher | String indicating the cipher used during the current connection. | keyword | -| tls.version | Numeric part of the version parsed from the original string. | keyword | -| tls.version_protocol | Normalized lowercase protocol name parsed from original string. | keyword | -| url.domain | Domain of the url, such as "www.elastic.co". In some cases a URL may refer to an IP and/or port directly, without a domain name. In this case, the IP address would go to the `domain` field. If the URL contains a literal IPv6 address enclosed by `[` and `]` (IETF RFC 2732), the `[` and `]` characters should also be captured in the `domain` field. | keyword | -| url.extension | The field contains the file extension from the original request url, excluding the leading dot. The file extension is only set if it exists, as not every url has a file extension. The leading period must not be included. For example, the value must be "png", not ".png". Note that when the file name has multiple extensions (example.tar.gz), only the last one should be captured ("gz", not "tar.gz"). | keyword | -| url.original | Unmodified original url as seen in the event source. Note that in network monitoring, the observed URL may be a full URL, whereas in access logs, the URL is often just represented as a path. This field is meant to represent the URL as it was observed, complete or not. | wildcard | -| url.original.text | Multi-field of `url.original`. | match_only_text | -| url.path | Path of the request, such as "/search". | wildcard | -| url.query | The query field describes the query string of the request, such as "q=elasticsearch". The `?` is excluded from the query string. If a URL contains no `?`, there is no query field. If there is a `?` but no query, the query field exists with an empty string. The `exists` query can be used to differentiate between the two cases. | keyword | -| url.scheme | Scheme of the request, such as "https". Note: The `:` is not part of the scheme. | keyword | -| user_agent.device.name | Name of the device. | keyword | -| user_agent.name | Name of the user agent. | keyword | -| user_agent.original | Unparsed user_agent string. | keyword | -| user_agent.original.text | Multi-field of `user_agent.original`. | match_only_text | -| user_agent.os.full | Operating system name, including the version or code name. | keyword | -| user_agent.os.full.text | Multi-field of `user_agent.os.full`. | match_only_text | -| user_agent.os.name | Operating system name, without the version. | keyword | -| user_agent.os.name.text | Multi-field of `user_agent.os.name`. | match_only_text | -| user_agent.os.version | Operating system version as a raw string. | keyword | -| user_agent.version | Version of the user agent. | keyword | An example event for `s3access` looks as following: @@ -247,7 +176,7 @@ An example event for `s3access` looks as following: "type": "logs" }, "ecs": { - "version": "8.0.0" + "version": "8.11.0" }, "elastic_agent": { "id": "acba78ef-1401-4689-977c-d8c2e5d6a8fa", @@ -257,7 +186,9 @@ An example event for `s3access` looks as following: "event": { "action": "REST.GET.LOCATION", "agent_id_status": "verified", - "category": "web", + "category": [ + "web" + ], "dataset": "aws.s3access", "duration": 17000000, "id": "44EE8651683CB4DA", @@ -351,70 +282,74 @@ An example event for `s3_daily_storage` looks as following: { "@timestamp": "2022-07-25T19:02:00.000Z", "agent": { - "name": "docker-fleet-agent", - "id": "2d4b09d0-cdb6-445e-ac3f-6415f87b9864", "ephemeral_id": "9ef87976-bec2-4a74-9876-4e76d42035bb", + "id": "2d4b09d0-cdb6-445e-ac3f-6415f87b9864", + "name": "docker-fleet-agent", "type": "metricbeat", "version": "8.3.2" }, - "elastic_agent": { - "id": "2d4b09d0-cdb6-445e-ac3f-6415f87b9864", - "version": "8.3.2", - "snapshot": false - }, - "cloud": { - "provider": "aws", - "region": "eu-central-1", - "account": { - "name": "elastic-beats", - "id": "428152502467" - } - }, - "ecs": { - "version": "8.0.0" - }, - "service": { - "type": "aws" - }, - "data_stream": { - "namespace": "default", - "type": "metrics", - "dataset": "aws.s3_daily_storage" - }, - "metricset": { - "period": 86400000, - "name": "cloudwatch" - }, "aws": { + "cloudwatch": { + "namespace": "AWS/S3" + }, + "dimensions": { + "StorageType": "StandardStorage" + }, "s3": { "bucket": { "name": "filebeat-aws-elb-test" } }, - "cloudwatch": { - "namespace": "AWS/S3" - }, "s3_daily_storage": { "bucket": { "size": { "bytes": 469407687 } } - }, - "dimensions": { - "StorageType": "StandardStorage" } }, + "cloud": { + "account": { + "id": "428152502467", + "name": "elastic-beats" + }, + "provider": "aws", + "region": "eu-central-1" + }, + "data_stream": { + "dataset": "aws.s3_daily_storage", + "namespace": "default", + "type": "metrics" + }, + "ecs": { + "version": "8.11.0" + }, + "elastic_agent": { + "id": "2d4b09d0-cdb6-445e-ac3f-6415f87b9864", + "snapshot": false, + "version": "8.3.2" + }, "event": { - "duration": 9553539400, "agent_id_status": "verified", + "dataset": "aws.s3_daily_storage", + "duration": 9553539400, "ingested": "2022-07-26T19:02:17Z", - "module": "aws", - "dataset": "aws.s3_daily_storage" + "module": "aws" + }, + "metricset": { + "name": "cloudwatch", + "period": 86400000 + }, + "service": { + "type": "aws" } } ``` +**ECS Field Reference** + +Please refer to the following [document](https://www.elastic.co/guide/en/ecs/current/ecs-field-reference.html) for detailed information on ECS fields. + **Exported fields** | Field | Description | Type | Metric Type | @@ -429,47 +364,16 @@ An example event for `s3_daily_storage` looks as following: | aws.s3_daily_storage.bucket.size.bytes | The amount of data in bytes stored in a bucket. | long | gauge | | aws.s3_daily_storage.number_of_objects | The total number of objects stored in a bucket for all storage classes. | long | gauge | | aws.tags | Tag key value pairs from aws resources. | flattened | | -| cloud | Fields related to the cloud or infrastructure the events are coming from. | group | | | cloud.account.id | The cloud account or organization id used to identify different entities in a multi-tenant environment. Examples: AWS account id, Google Cloud ORG Id, or other unique identifier. | keyword | | -| cloud.account.name | The cloud account name or alias used to identify different entities in a multi-tenant environment. Examples: AWS account name, Google Cloud ORG display name. | keyword | | -| cloud.availability_zone | Availability zone in which this host, resource, or service is located. | keyword | | | cloud.image.id | Image ID for the cloud instance. | keyword | | -| cloud.instance.id | Instance ID of the host machine. | keyword | | -| cloud.instance.name | Instance name of the host machine. | keyword | | -| cloud.machine.type | Machine type of the host machine. | keyword | | -| cloud.project.id | The cloud project identifier. Examples: Google Cloud Project id, Azure Project id. | keyword | | -| cloud.provider | Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. | keyword | | | cloud.region | Region in which this host, resource, or service is located. | keyword | | -| container.id | Unique container id. | keyword | | -| container.image.name | Name of the image the container was built on. | keyword | | -| container.labels | Image labels. | object | | -| container.name | Container name. | keyword | | | data_stream.dataset | Data stream dataset. | constant_keyword | | | data_stream.namespace | Data stream namespace. | constant_keyword | | | data_stream.type | Data stream type. | constant_keyword | | -| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | | -| error | These fields can represent errors of any kind. Use them for errors that happen while fetching events or in cases where the event itself contains an error. | group | | -| error.message | Error message. | match_only_text | | -| event.dataset | Name of the dataset. If an event source publishes more than one type of log or events (e.g. access log, error log), the dataset is used to specify which one the event comes from. It's recommended but not required to start the dataset name with the module name, followed by a dot, then the dataset name. | constant_keyword | | | event.module | Event module | constant_keyword | | -| host.architecture | Operating system architecture. | keyword | | | host.containerized | If the host is a container. | boolean | | -| host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | | -| host.hostname | Hostname of the host. It normally contains what the `hostname` command returns on the host machine. | keyword | | -| host.id | Unique host id. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`. | keyword | | -| host.ip | Host ip addresses. | ip | | -| host.mac | Host MAC addresses. The notation format from RFC 7042 is suggested: Each octet (that is, 8-bit byte) is represented by two [uppercase] hexadecimal digits giving the value of the octet as an unsigned integer. Successive octets are separated by a hyphen. | keyword | | -| host.name | Name of the host. It can contain what hostname returns on Unix systems, the fully qualified domain name (FQDN), or a name specified by the user. The recommended value is the lowercase FQDN of the host. | keyword | | | host.os.build | OS build information. | keyword | | | host.os.codename | OS codename, if any. | keyword | | -| host.os.family | OS family (such as redhat, debian, freebsd, windows). | keyword | | -| host.os.kernel | Operating system kernel version as a raw string. | keyword | | -| host.os.name | Operating system name, without the version. | keyword | | -| host.os.name.text | Multi-field of `host.os.name`. | match_only_text | | -| host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword | | -| host.os.version | Operating system version as a raw string. | keyword | | -| host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | | -| service.type | The type of the service data is collected from. The type can be used to group and correlate logs and metrics from one service type. Example: If logs or metrics are collected from Elasticsearch, `service.type` would be `elasticsearch`. | keyword | | ### s3_request @@ -480,50 +384,32 @@ An example event for `s3_request` looks as following: { "@timestamp": "2022-07-26T20:10:00.000Z", "agent": { - "name": "docker-fleet-agent", + "ephemeral_id": "287cb701-3031-45be-a8c1-4c4860603d9b", "id": "2d4b09d0-cdb6-445e-ac3f-6415f87b9864", + "name": "docker-fleet-agent", "type": "metricbeat", - "ephemeral_id": "287cb701-3031-45be-a8c1-4c4860603d9b", "version": "8.3.2" }, - "elastic_agent": { - "id": "2d4b09d0-cdb6-445e-ac3f-6415f87b9864", - "version": "8.3.2", - "snapshot": false - }, - "cloud": { - "provider": "aws", - "region": "us-east-1", - "account": { - "name": "elastic-beats", - "id": "428152502467" - } - }, - "ecs": { - "version": "8.0.0" - }, - "service": { - "type": "aws" - }, - "data_stream": { - "namespace": "default", - "type": "metrics", - "dataset": "aws.s3_request" - }, - "metricset": { - "period": 60000, - "name": "cloudwatch" - }, "aws": { + "cloudwatch": { + "namespace": "AWS/S3" + }, + "dimensions": { + "FilterId": "AllItems" + }, "s3": { "bucket": { "name": "vpc-flow-logs-ks" } }, - "cloudwatch": { - "namespace": "AWS/S3" - }, "s3_request": { + "downloaded": { + "bytes": 400 + }, + "errors": { + "4xx": 1, + "5xx": 0 + }, "latency": { "total_request": { "ms": 32 @@ -532,29 +418,51 @@ An example event for `s3_request` looks as following: "requests": { "head": 1, "total": 1 - }, - "downloaded": { - "bytes": 400 - }, - "errors": { - "4xx": 1, - "5xx": 0 } - }, - "dimensions": { - "FilterId": "AllItems" } }, + "cloud": { + "account": { + "id": "428152502467", + "name": "elastic-beats" + }, + "provider": "aws", + "region": "us-east-1" + }, + "data_stream": { + "dataset": "aws.s3_request", + "namespace": "default", + "type": "metrics" + }, + "ecs": { + "version": "8.11.0" + }, + "elastic_agent": { + "id": "2d4b09d0-cdb6-445e-ac3f-6415f87b9864", + "snapshot": false, + "version": "8.3.2" + }, "event": { - "duration": 9552028500, "agent_id_status": "verified", + "dataset": "aws.s3_request", + "duration": 9552028500, "ingested": "2022-07-26T20:16:31Z", - "module": "aws", - "dataset": "aws.s3_request" + "module": "aws" + }, + "metricset": { + "name": "cloudwatch", + "period": 60000 + }, + "service": { + "type": "aws" } } ``` +**ECS Field Reference** + +Please refer to the following [document](https://www.elastic.co/guide/en/ecs/current/ecs-field-reference.html) for detailed information on ECS fields. + **Exported fields** | Field | Description | Type | Metric Type | @@ -585,45 +493,14 @@ An example event for `s3_request` looks as following: | aws.s3_request.uploaded.bytes | The number bytes uploaded that contain a request body, made to an Amazon S3 bucket. | long | gauge | | aws.s3_request.uploaded.bytes_per_period | The number bytes per period uploaded that contain a request body, made to an Amazon S3 bucket. | long | gauge | | aws.tags | Tag key value pairs from aws resources. | flattened | | -| cloud | Fields related to the cloud or infrastructure the events are coming from. | group | | | cloud.account.id | The cloud account or organization id used to identify different entities in a multi-tenant environment. Examples: AWS account id, Google Cloud ORG Id, or other unique identifier. | keyword | | -| cloud.account.name | The cloud account name or alias used to identify different entities in a multi-tenant environment. Examples: AWS account name, Google Cloud ORG display name. | keyword | | -| cloud.availability_zone | Availability zone in which this host, resource, or service is located. | keyword | | | cloud.image.id | Image ID for the cloud instance. | keyword | | -| cloud.instance.id | Instance ID of the host machine. | keyword | | -| cloud.instance.name | Instance name of the host machine. | keyword | | -| cloud.machine.type | Machine type of the host machine. | keyword | | -| cloud.project.id | The cloud project identifier. Examples: Google Cloud Project id, Azure Project id. | keyword | | -| cloud.provider | Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. | keyword | | | cloud.region | Region in which this host, resource, or service is located. | keyword | | -| container.id | Unique container id. | keyword | | -| container.image.name | Name of the image the container was built on. | keyword | | -| container.labels | Image labels. | object | | -| container.name | Container name. | keyword | | | data_stream.dataset | Data stream dataset. | constant_keyword | | | data_stream.namespace | Data stream namespace. | constant_keyword | | | data_stream.type | Data stream type. | constant_keyword | | -| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | | -| error | These fields can represent errors of any kind. Use them for errors that happen while fetching events or in cases where the event itself contains an error. | group | | -| error.message | Error message. | match_only_text | | -| event.dataset | Name of the dataset. If an event source publishes more than one type of log or events (e.g. access log, error log), the dataset is used to specify which one the event comes from. It's recommended but not required to start the dataset name with the module name, followed by a dot, then the dataset name. | constant_keyword | | | event.module | Event module | constant_keyword | | -| host.architecture | Operating system architecture. | keyword | | | host.containerized | If the host is a container. | boolean | | -| host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | | -| host.hostname | Hostname of the host. It normally contains what the `hostname` command returns on the host machine. | keyword | | -| host.id | Unique host id. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`. | keyword | | -| host.ip | Host ip addresses. | ip | | -| host.mac | Host MAC addresses. The notation format from RFC 7042 is suggested: Each octet (that is, 8-bit byte) is represented by two [uppercase] hexadecimal digits giving the value of the octet as an unsigned integer. Successive octets are separated by a hyphen. | keyword | | -| host.name | Name of the host. It can contain what hostname returns on Unix systems, the fully qualified domain name (FQDN), or a name specified by the user. The recommended value is the lowercase FQDN of the host. | keyword | | | host.os.build | OS build information. | keyword | | | host.os.codename | OS codename, if any. | keyword | | -| host.os.family | OS family (such as redhat, debian, freebsd, windows). | keyword | | -| host.os.kernel | Operating system kernel version as a raw string. | keyword | | -| host.os.name | Operating system name, without the version. | keyword | | -| host.os.name.text | Multi-field of `host.os.name`. | match_only_text | | -| host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword | | -| host.os.version | Operating system version as a raw string. | keyword | | -| host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | | -| service.type | The type of the service data is collected from. The type can be used to group and correlate logs and metrics from one service type. Example: If logs or metrics are collected from Elasticsearch, `service.type` would be `elasticsearch`. | keyword | | diff --git a/packages/aws/docs/s3_storage_lens.md b/packages/aws/docs/s3_storage_lens.md index 167fa71bda1..64c7af04034 100644 --- a/packages/aws/docs/s3_storage_lens.md +++ b/packages/aws/docs/s3_storage_lens.md @@ -43,144 +43,148 @@ An example event for `s3_storage_lens` looks as following: ```json { "@timestamp": "2021-11-07T20:38:00.000Z", - "ecs": { - "version": "8.0.0" - }, - "data_stream": { - "namespace": "default", - "type": "metrics", - "dataset": "aws.s3_storage_lens" - }, - "service": { - "type": "aws" - }, - "cloud": { - "provider": "aws", - "region": "us-east-1", - "account": { - "name": "elastic-beats", - "id": "428152502467" - } - }, - "metricset": { - "period": 86400000, - "name": "cloudwatch" - }, - "event": { - "duration": 22973251900, - "agent_id_status": "verified", - "ingested": "2021-11-08T20:38:37Z", - "module": "aws", - "dataset": "aws.s3_storage_lens" - }, "aws": { + "cloudwatch": { + "namespace": "AWS/S3/Storage-Lens" + }, + "dimensions": { + "aws_account_number": "428152502467", + "aws_region": "eu-central-1", + "bucket_name": "filebeat-aws-elb-test", + "configuration_id": "default-account-dashboard", + "metrics_version": "1.0", + "record_type": "BUCKET", + "storage_class": "STANDARD" + }, "s3_storage_lens": { "metrics": { - "NonCurrentVersionStorageBytes": { + "4xxErrors": { "avg": 0 }, - "DeleteMarkerObjectCount": { + "5xxErrors": { "avg": 0 }, - "GetRequests": { - "avg": 0 + "AllRequests": { + "avg": 145 }, - "SelectReturnedBytes": { + "BytesDownloaded": { "avg": 0 }, - "ObjectCount": { + "BytesUploaded": { + "avg": 82537 + }, + "CurrentVersionObjectCount": { "avg": 164195 }, - "HeadRequests": { - "avg": 0 + "CurrentVersionStorageBytes": { + "avg": 154238334 }, - "ListRequests": { + "DeleteMarkerObjectCount": { "avg": 0 }, "DeleteRequests": { "avg": 0 }, - "SelectRequests": { - "avg": 0 + "EncryptedObjectCount": { + "avg": 164191 }, - "5xxErrors": { + "EncryptedStorageBytes": { + "avg": 154237917 + }, + "GetRequests": { "avg": 0 }, - "BytesDownloaded": { + "HeadRequests": { "avg": 0 }, - "BytesUploaded": { - "avg": 82537 + "IncompleteMultipartUploadObjectCount": { + "avg": 0 }, - "CurrentVersionStorageBytes": { - "avg": 154238334 + "IncompleteMultipartUploadStorageBytes": { + "avg": 0 }, - "StorageBytes": { - "avg": 154238334 + "ListRequests": { + "avg": 0 }, - "ObjectLockEnabledStorageBytes": { + "NonCurrentVersionObjectCount": { "avg": 0 }, - "4xxErrors": { + "NonCurrentVersionStorageBytes": { "avg": 0 }, - "PutRequests": { - "avg": 145 + "ObjectCount": { + "avg": 164195 }, "ObjectLockEnabledObjectCount": { "avg": 0 }, - "EncryptedObjectCount": { - "avg": 164191 - }, - "CurrentVersionObjectCount": { - "avg": 164195 - }, - "IncompleteMultipartUploadObjectCount": { + "ObjectLockEnabledStorageBytes": { "avg": 0 }, - "ReplicatedObjectCount": { + "PostRequests": { "avg": 0 }, - "AllRequests": { + "PutRequests": { "avg": 145 }, - "PostRequests": { + "ReplicatedObjectCount": { "avg": 0 }, - "IncompleteMultipartUploadStorageBytes": { + "ReplicatedStorageBytes": { "avg": 0 }, - "NonCurrentVersionObjectCount": { + "SelectRequests": { "avg": 0 }, - "ReplicatedStorageBytes": { + "SelectReturnedBytes": { "avg": 0 }, - "EncryptedStorageBytes": { - "avg": 154237917 - }, "SelectScannedBytes": { "avg": 0 + }, + "StorageBytes": { + "avg": 154238334 } } - }, - "cloudwatch": { - "namespace": "AWS/S3/Storage-Lens" - }, - "dimensions": { - "metrics_version": "1.0", - "storage_class": "STANDARD", - "aws_region": "eu-central-1", - "bucket_name": "filebeat-aws-elb-test", - "aws_account_number": "428152502467", - "configuration_id": "default-account-dashboard", - "record_type": "BUCKET" } + }, + "cloud": { + "account": { + "id": "428152502467", + "name": "elastic-beats" + }, + "provider": "aws", + "region": "us-east-1" + }, + "data_stream": { + "dataset": "aws.s3_storage_lens", + "namespace": "default", + "type": "metrics" + }, + "ecs": { + "version": "8.11.0" + }, + "event": { + "agent_id_status": "verified", + "dataset": "aws.s3_storage_lens", + "duration": 22973251900, + "ingested": "2021-11-08T20:38:37Z", + "module": "aws" + }, + "metricset": { + "name": "cloudwatch", + "period": 86400000 + }, + "service": { + "type": "aws" } } ``` +**ECS Field Reference** + +Please refer to the following [document](https://www.elastic.co/guide/en/ecs/current/ecs-field-reference.html) for detailed information on ECS fields. + **Exported fields** | Field | Description | Type | Metric Type | @@ -227,44 +231,13 @@ An example event for `s3_storage_lens` looks as following: | aws.s3_storage_lens.metrics.SelectScannedBytes.avg | The number of select bytes scanned. | long | gauge | | aws.s3_storage_lens.metrics.StorageBytes.avg | The total storage in bytes | long | gauge | | aws.tags | Tag key value pairs from aws resources. | flattened | | -| cloud | Fields related to the cloud or infrastructure the events are coming from. | group | | | cloud.account.id | The cloud account or organization id used to identify different entities in a multi-tenant environment. Examples: AWS account id, Google Cloud ORG Id, or other unique identifier. | keyword | | -| cloud.account.name | The cloud account name or alias used to identify different entities in a multi-tenant environment. Examples: AWS account name, Google Cloud ORG display name. | keyword | | -| cloud.availability_zone | Availability zone in which this host, resource, or service is located. | keyword | | | cloud.image.id | Image ID for the cloud instance. | keyword | | -| cloud.instance.id | Instance ID of the host machine. | keyword | | -| cloud.instance.name | Instance name of the host machine. | keyword | | -| cloud.machine.type | Machine type of the host machine. | keyword | | -| cloud.project.id | The cloud project identifier. Examples: Google Cloud Project id, Azure Project id. | keyword | | -| cloud.provider | Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. | keyword | | | cloud.region | Region in which this host, resource, or service is located. | keyword | | -| container.id | Unique container id. | keyword | | -| container.image.name | Name of the image the container was built on. | keyword | | -| container.labels | Image labels. | object | | -| container.name | Container name. | keyword | | | data_stream.dataset | Data stream dataset. | constant_keyword | | | data_stream.namespace | Data stream namespace. | constant_keyword | | | data_stream.type | Data stream type. | constant_keyword | | -| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | | -| error | These fields can represent errors of any kind. Use them for errors that happen while fetching events or in cases where the event itself contains an error. | group | | -| error.message | Error message. | match_only_text | | -| event.dataset | Name of the dataset. If an event source publishes more than one type of log or events (e.g. access log, error log), the dataset is used to specify which one the event comes from. It's recommended but not required to start the dataset name with the module name, followed by a dot, then the dataset name. | constant_keyword | | | event.module | Event module | constant_keyword | | -| host.architecture | Operating system architecture. | keyword | | | host.containerized | If the host is a container. | boolean | | -| host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | | -| host.hostname | Hostname of the host. It normally contains what the `hostname` command returns on the host machine. | keyword | | -| host.id | Unique host id. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`. | keyword | | -| host.ip | Host ip addresses. | ip | | -| host.mac | Host MAC addresses. The notation format from RFC 7042 is suggested: Each octet (that is, 8-bit byte) is represented by two [uppercase] hexadecimal digits giving the value of the octet as an unsigned integer. Successive octets are separated by a hyphen. | keyword | | -| host.name | Name of the host. It can contain what hostname returns on Unix systems, the fully qualified domain name (FQDN), or a name specified by the user. The recommended value is the lowercase FQDN of the host. | keyword | | | host.os.build | OS build information. | keyword | | | host.os.codename | OS codename, if any. | keyword | | -| host.os.family | OS family (such as redhat, debian, freebsd, windows). | keyword | | -| host.os.kernel | Operating system kernel version as a raw string. | keyword | | -| host.os.name | Operating system name, without the version. | keyword | | -| host.os.name.text | Multi-field of `host.os.name`. | match_only_text | | -| host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword | | -| host.os.version | Operating system version as a raw string. | keyword | | -| host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | | -| service.type | The type of the service data is collected from. The type can be used to group and correlate logs and metrics from one service type. Example: If logs or metrics are collected from Elasticsearch, `service.type` would be `elasticsearch`. | keyword | | diff --git a/packages/aws/docs/securityhub.md b/packages/aws/docs/securityhub.md index 938cfd2a749..fa46ef19325 100644 --- a/packages/aws/docs/securityhub.md +++ b/packages/aws/docs/securityhub.md @@ -372,7 +372,7 @@ An example event for `securityhub_findings` looks as following: "port": 80 }, "ecs": { - "version": "8.2.0" + "version": "8.11.0" }, "elastic_agent": { "id": "eea1c0db-3657-4195-add3-da25a54834e7", @@ -462,6 +462,10 @@ An example event for `securityhub_findings` looks as following: } ``` +**ECS Field Reference** + +Please refer to the following [document](https://www.elastic.co/guide/en/ecs/current/ecs-field-reference.html) for detailed information on ECS fields. + **Exported fields** | Field | Description | Type | @@ -646,99 +650,17 @@ An example event for `securityhub_findings` looks as following: | aws.securityhub_findings.vulnerabilities.vulnerable_packages.version | The version of the software package. | keyword | | aws.securityhub_findings.workflow.state | The workflow state of a finding. | keyword | | aws.securityhub_findings.workflow.status | The status of the investigation into the finding. | keyword | -| cloud.account.id | The cloud account or organization id used to identify different entities in a multi-tenant environment. Examples: AWS account id, Google Cloud ORG Id, or other unique identifier. | keyword | -| cloud.account.name | The cloud account name or alias used to identify different entities in a multi-tenant environment. Examples: AWS account name, Google Cloud ORG display name. | keyword | -| cloud.availability_zone | Availability zone in which this host, resource, or service is located. | keyword | | cloud.image.id | Image ID for the cloud instance. | keyword | -| cloud.instance.id | Instance ID of the host machine. | keyword | -| cloud.instance.name | Instance name of the host machine. | keyword | -| cloud.machine.type | Machine type of the host machine. | keyword | -| cloud.project.id | The cloud project identifier. Examples: Google Cloud Project id, Azure Project id. | keyword | -| cloud.provider | Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. | keyword | -| cloud.region | Region in which this host, resource, or service is located. | keyword | -| container.id | Unique container id. | keyword | -| container.image.name | Name of the image the container was built on. | keyword | -| container.labels | Image labels. | object | -| container.name | Container name. | keyword | | data_stream.dataset | Data stream dataset. | constant_keyword | | data_stream.namespace | Data stream namespace. | constant_keyword | | data_stream.type | Data stream type. | constant_keyword | -| destination.domain | The domain name of the destination system. This value may be a host name, a fully qualified domain name, or another host naming format. The value may derive from the original event or be added from enrichment. | keyword | -| destination.ip | IP address of the destination (IPv4 or IPv6). | ip | -| destination.port | Port of the destination. | long | -| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | -| event.action | The action captured by the event. This describes the information in the event. It is more specific than `event.category`. Examples are `group-add`, `process-started`, `file-created`. The value is normally defined by the implementer. | keyword | -| event.category | This is one of four ECS Categorization Fields, and indicates the second level in the ECS category hierarchy. `event.category` represents the "big buckets" of ECS categories. For example, filtering on `event.category:process` yields all events relating to process activity. This field is closely related to `event.type`, which is used as a subcategory. This field is an array. This will allow proper categorization of some events that fall in multiple categories. | keyword | -| event.created | `event.created` contains the date/time when the event was first read by an agent, or by your pipeline. This field is distinct from `@timestamp` in that `@timestamp` typically contain the time extracted from the original event. In most situations, these two timestamps will be slightly different. The difference can be used to calculate the delay between your source generating an event, and the time when your agent first processed it. This can be used to monitor your agent's or pipeline's ability to keep up with your event source. In case the two timestamps are identical, `@timestamp` should be used. | date | -| event.dataset | Name of the dataset. If an event source publishes more than one type of log or events (e.g. access log, error log), the dataset is used to specify which one the event comes from. It's recommended but not required to start the dataset name with the module name, followed by a dot, then the dataset name. | constant_keyword | -| event.id | Unique ID to describe the event. | keyword | -| event.kind | This is one of four ECS Categorization Fields, and indicates the highest level in the ECS category hierarchy. `event.kind` gives high-level information about what type of information the event contains, without being specific to the contents of the event. For example, values of this field distinguish alert events from metric events. The value of this field can be used to inform how these kinds of events should be handled. They may warrant different retention, different access control, it may also help understand whether the data is coming in at a regular interval or not. | keyword | | event.module | Event module. | constant_keyword | -| event.original | Raw text message of entire event. Used to demonstrate log integrity or where the full log message (before splitting it up in multiple parts) may be required, e.g. for reindex. This field is not indexed and doc_values are disabled. It cannot be searched, but it can be retrieved from `_source`. If users wish to override this and index this field, please see `Field data types` in the `Elasticsearch Reference`. | keyword | -| event.type | This is one of four ECS Categorization Fields, and indicates the third level in the ECS category hierarchy. `event.type` represents a categorization "sub-bucket" that, when used along with the `event.category` field values, enables filtering events down to a level appropriate for single visualization. This field is an array. This will allow proper categorization of some events that fall in multiple event types. | keyword | -| host.architecture | Operating system architecture. | keyword | | host.containerized | If the host is a container. | boolean | -| host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | -| host.hostname | Hostname of the host. It normally contains what the `hostname` command returns on the host machine. | keyword | -| host.id | Unique host id. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`. | keyword | -| host.ip | Host ip addresses. | ip | -| host.mac | Host MAC addresses. The notation format from RFC 7042 is suggested: Each octet (that is, 8-bit byte) is represented by two [uppercase] hexadecimal digits giving the value of the octet as an unsigned integer. Successive octets are separated by a hyphen. | keyword | -| host.name | Name of the host. It can contain what hostname returns on Unix systems, the fully qualified domain name (FQDN), or a name specified by the user. The recommended value is the lowercase FQDN of the host. | keyword | | host.os.build | OS build information. | keyword | | host.os.codename | OS codename, if any. | keyword | -| host.os.family | OS family (such as redhat, debian, freebsd, windows). | keyword | -| host.os.kernel | Operating system kernel version as a raw string. | keyword | -| host.os.name | Operating system name, without the version. | keyword | -| host.os.name.text | Multi-field of `host.os.name`. | match_only_text | -| host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword | -| host.os.version | Operating system version as a raw string. | keyword | -| host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | | input.type | Input type | keyword | | log.offset | Log offset | long | -| network.direction | Direction of the network traffic. When mapping events from a host-based monitoring context, populate this field from the host's point of view, using the values "ingress" or "egress". When mapping events from a network or perimeter-based monitoring context, populate this field from the point of view of the network perimeter, using the values "inbound", "outbound", "internal" or "external". Note that "internal" is not crossing perimeter boundaries, and is meant to describe communication between two hosts within the perimeter. Note also that "external" is meant to describe traffic between two hosts that are external to the perimeter. This could for example be useful for ISPs or VPN service providers. | keyword | -| network.protocol | In the OSI Model this would be the Application Layer protocol. For example, `http`, `dns`, or `ssh`. The field value must be normalized to lowercase for querying. | keyword | -| organization.name | Organization name. | keyword | -| organization.name.text | Multi-field of `organization.name`. | match_only_text | -| process.end | The time the process ended. | date | -| process.executable | Absolute path to the process executable. | keyword | -| process.executable.text | Multi-field of `process.executable`. | match_only_text | -| process.name | Process name. Sometimes called program name or similar. | keyword | -| process.name.text | Multi-field of `process.name`. | match_only_text | -| process.parent.pid | Process id. | long | -| process.pid | Process id. | long | -| process.start | The time the process started. | date | -| related.ip | All of the IPs seen on your event. | ip | -| source.domain | The domain name of the source system. This value may be a host name, a fully qualified domain name, or another host naming format. The value may derive from the original event or be added from enrichment. | keyword | -| source.ip | IP address of the source (IPv4 or IPv6). | ip | -| source.mac | MAC address of the source. The notation format from RFC 7042 is suggested: Each octet (that is, 8-bit byte) is represented by two [uppercase] hexadecimal digits giving the value of the octet as an unsigned integer. Successive octets are separated by a hyphen. | keyword | -| source.port | Port of the source. | long | -| tags | List of keywords used to tag each event. | keyword | -| threat.enrichments | A list of associated indicators objects enriching the event, and the context of that association/enrichment. | nested | -| threat.enrichments.indicator.file.hash.md5 | MD5 hash. | keyword | -| threat.enrichments.indicator.file.hash.sha1 | SHA1 hash. | keyword | -| threat.enrichments.indicator.file.hash.sha256 | SHA256 hash. | keyword | -| threat.enrichments.indicator.file.hash.sha512 | SHA512 hash. | keyword | -| threat.indicator.last_seen | The date and time when intelligence source last reported sighting this indicator. | date | -| threat.indicator.type | Type of indicator as represented by Cyber Observable in STIX 2.0. | keyword | -| url.domain | Domain of the url, such as "www.elastic.co". In some cases a URL may refer to an IP and/or port directly, without a domain name. In this case, the IP address would go to the `domain` field. If the URL contains a literal IPv6 address enclosed by `[` and `]` (IETF RFC 2732), the `[` and `]` characters should also be captured in the `domain` field. | keyword | -| url.extension | The field contains the file extension from the original request url, excluding the leading dot. The file extension is only set if it exists, as not every url has a file extension. The leading period must not be included. For example, the value must be "png", not ".png". Note that when the file name has multiple extensions (example.tar.gz), only the last one should be captured ("gz", not "tar.gz"). | keyword | -| url.fragment | Portion of the url after the `#`, such as "top". The `#` is not part of the fragment. | keyword | -| url.full | If full URLs are important to your use case, they should be stored in `url.full`, whether this field is reconstructed or present in the event source. | wildcard | -| url.full.text | Multi-field of `url.full`. | match_only_text | -| url.original | Unmodified original url as seen in the event source. Note that in network monitoring, the observed URL may be a full URL, whereas in access logs, the URL is often just represented as a path. This field is meant to represent the URL as it was observed, complete or not. | wildcard | -| url.original.text | Multi-field of `url.original`. | match_only_text | -| url.password | Password of the request. | keyword | -| url.path | Path of the request, such as "/search". | wildcard | -| url.port | Port of the request, such as 443. | long | -| url.query | The query field describes the query string of the request, such as "q=elasticsearch". The `?` is excluded from the query string. If a URL contains no `?`, there is no query field. If there is a `?` but no query, the query field exists with an empty string. The `exists` query can be used to differentiate between the two cases. | keyword | -| url.scheme | Scheme of the request, such as "https". Note: The `:` is not part of the scheme. | keyword | | url.user_info | | keyword | -| url.username | Username of the request. | keyword | -| vulnerability.id | The identification (ID) is the number portion of a vulnerability entry. It includes a unique identification number for the vulnerability. For example (https://cve.mitre.org/about/faqs.html#what_is_cve_id)[Common Vulnerabilities and Exposure CVE ID] | keyword | -| vulnerability.reference | A resource that provides additional information, context, and mitigations for the identified vulnerability. | keyword | -| vulnerability.scanner.vendor | The name of the vulnerability scanner vendor. | keyword | -| vulnerability.score.base | Scores can range from 0.0 to 10.0, with 10.0 being the most severe. Base scores cover an assessment for exploitability metrics (attack vector, complexity, privileges, and user interaction), impact metrics (confidentiality, integrity, and availability), and scope. For example (https://www.first.org/cvss/specification-document) | float | -| vulnerability.score.version | The National Vulnerability Database (NVD) provides qualitative severity rankings of "Low", "Medium", and "High" for CVSS v2.0 base score ranges in addition to the severity ratings for CVSS v3.0 as they are defined in the CVSS v3.0 specification. CVSS is owned and managed by FIRST.Org, Inc. (FIRST), a US-based non-profit organization, whose mission is to help computer security incident response teams across the world. For example (https://nvd.nist.gov/vuln-metrics/cvss) | keyword | ### Insights @@ -1472,7 +1394,7 @@ An example event for `securityhub_insights` looks as following: "type": "logs" }, "ecs": { - "version": "8.2.0" + "version": "8.11.0" }, "elastic_agent": { "id": "eea1c0db-3657-4195-add3-da25a54834e7", @@ -1501,6 +1423,10 @@ An example event for `securityhub_insights` looks as following: } ``` +**ECS Field Reference** + +Please refer to the following [document](https://www.elastic.co/guide/en/ecs/current/ecs-field-reference.html) for detailed information on ECS fields. + **Exported fields** | Field | Description | Type | @@ -1638,48 +1564,13 @@ An example event for `securityhub_insights` looks as following: | aws.securityhub_insights.group_by_attribute | The grouping attribute for the insight's findings. Indicates how to group the matching findings, and identifies the type of item that the insight applies to. For example, if an insight is grouped by resource identifier, then the insight produces a list of resource identifiers. | keyword | | aws.securityhub_insights.insight_arn | The ARN of a Security Hub insight. | keyword | | aws.securityhub_insights.name | The name of a Security Hub insight. | keyword | -| cloud.account.id | The cloud account or organization id used to identify different entities in a multi-tenant environment. Examples: AWS account id, Google Cloud ORG Id, or other unique identifier. | keyword | -| cloud.account.name | The cloud account name or alias used to identify different entities in a multi-tenant environment. Examples: AWS account name, Google Cloud ORG display name. | keyword | -| cloud.availability_zone | Availability zone in which this host, resource, or service is located. | keyword | | cloud.image.id | Image ID for the cloud instance. | keyword | -| cloud.instance.id | Instance ID of the host machine. | keyword | -| cloud.instance.name | Instance name of the host machine. | keyword | -| cloud.machine.type | Machine type of the host machine. | keyword | -| cloud.project.id | The cloud project identifier. Examples: Google Cloud Project id, Azure Project id. | keyword | -| cloud.provider | Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. | keyword | -| cloud.region | Region in which this host, resource, or service is located. | keyword | -| container.id | Unique container id. | keyword | -| container.image.name | Name of the image the container was built on. | keyword | -| container.labels | Image labels. | object | -| container.name | Container name. | keyword | | data_stream.dataset | Data stream dataset. | constant_keyword | | data_stream.namespace | Data stream namespace. | constant_keyword | | data_stream.type | Data stream type. | constant_keyword | -| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | -| event.category | This is one of four ECS Categorization Fields, and indicates the second level in the ECS category hierarchy. `event.category` represents the "big buckets" of ECS categories. For example, filtering on `event.category:process` yields all events relating to process activity. This field is closely related to `event.type`, which is used as a subcategory. This field is an array. This will allow proper categorization of some events that fall in multiple categories. | keyword | -| event.created | `event.created` contains the date/time when the event was first read by an agent, or by your pipeline. This field is distinct from `@timestamp` in that `@timestamp` typically contain the time extracted from the original event. In most situations, these two timestamps will be slightly different. The difference can be used to calculate the delay between your source generating an event, and the time when your agent first processed it. This can be used to monitor your agent's or pipeline's ability to keep up with your event source. In case the two timestamps are identical, `@timestamp` should be used. | date | -| event.dataset | Name of the dataset. If an event source publishes more than one type of log or events (e.g. access log, error log), the dataset is used to specify which one the event comes from. It's recommended but not required to start the dataset name with the module name, followed by a dot, then the dataset name. | constant_keyword | -| event.kind | This is one of four ECS Categorization Fields, and indicates the highest level in the ECS category hierarchy. `event.kind` gives high-level information about what type of information the event contains, without being specific to the contents of the event. For example, values of this field distinguish alert events from metric events. The value of this field can be used to inform how these kinds of events should be handled. They may warrant different retention, different access control, it may also help understand whether the data is coming in at a regular interval or not. | keyword | | event.module | Event module. | constant_keyword | -| event.original | Raw text message of entire event. Used to demonstrate log integrity or where the full log message (before splitting it up in multiple parts) may be required, e.g. for reindex. This field is not indexed and doc_values are disabled. It cannot be searched, but it can be retrieved from `_source`. If users wish to override this and index this field, please see `Field data types` in the `Elasticsearch Reference`. | keyword | -| event.type | This is one of four ECS Categorization Fields, and indicates the third level in the ECS category hierarchy. `event.type` represents a categorization "sub-bucket" that, when used along with the `event.category` field values, enables filtering events down to a level appropriate for single visualization. This field is an array. This will allow proper categorization of some events that fall in multiple event types. | keyword | -| host.architecture | Operating system architecture. | keyword | | host.containerized | If the host is a container. | boolean | -| host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | -| host.hostname | Hostname of the host. It normally contains what the `hostname` command returns on the host machine. | keyword | -| host.id | Unique host id. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`. | keyword | -| host.ip | Host ip addresses. | ip | -| host.mac | Host MAC addresses. The notation format from RFC 7042 is suggested: Each octet (that is, 8-bit byte) is represented by two [uppercase] hexadecimal digits giving the value of the octet as an unsigned integer. Successive octets are separated by a hyphen. | keyword | -| host.name | Name of the host. It can contain what hostname returns on Unix systems, the fully qualified domain name (FQDN), or a name specified by the user. The recommended value is the lowercase FQDN of the host. | keyword | | host.os.build | OS build information. | keyword | | host.os.codename | OS codename, if any. | keyword | -| host.os.family | OS family (such as redhat, debian, freebsd, windows). | keyword | -| host.os.kernel | Operating system kernel version as a raw string. | keyword | -| host.os.name | Operating system name, without the version. | keyword | -| host.os.name.text | Multi-field of `host.os.name`. | match_only_text | -| host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword | -| host.os.version | Operating system version as a raw string. | keyword | -| host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | | input.type | Input type | keyword | | log.offset | Log offset | long | -| tags | List of keywords used to tag each event. | keyword | diff --git a/packages/aws/docs/sns.md b/packages/aws/docs/sns.md index 5ca0dbf5b94..04bd0b48406 100644 --- a/packages/aws/docs/sns.md +++ b/packages/aws/docs/sns.md @@ -45,74 +45,78 @@ An example event for `sns` looks as following: { "@timestamp": "2022-07-26T21:56:00.000Z", "agent": { - "name": "docker-fleet-agent", - "id": "2d4b09d0-cdb6-445e-ac3f-6415f87b9864", "ephemeral_id": "51866723-6dfa-4a72-a68e-f439d5de7f53", + "id": "2d4b09d0-cdb6-445e-ac3f-6415f87b9864", + "name": "docker-fleet-agent", "type": "metricbeat", "version": "8.3.2" }, - "elastic_agent": { - "id": "2d4b09d0-cdb6-445e-ac3f-6415f87b9864", - "version": "8.3.2", - "snapshot": false - }, - "cloud": { - "provider": "aws", - "region": "us-east-1", - "account": { - "name": "elastic-beats", - "id": "428152502467" - } - }, - "ecs": { - "version": "8.0.0" - }, - "data_stream": { - "namespace": "default", - "type": "metrics", - "dataset": "aws.sns" - }, - "service": { - "type": "aws" - }, - "metricset": { - "period": 300000, - "name": "cloudwatch" - }, "aws": { + "cloudwatch": { + "namespace": "AWS/SNS" + }, + "dimensions": { + "TopicName": "vpc-flow-logs-sns-topic" + }, "sns": { "metrics": { - "NumberOfNotificationsDelivered": { - "sum": 5 - }, "NumberOfMessagesPublished": { "sum": 6 }, - "PublishSize": { - "avg": 905 + "NumberOfNotificationsDelivered": { + "sum": 5 }, "NumberOfNotificationsFailed": { "sum": 0 + }, + "PublishSize": { + "avg": 905 } } - }, - "cloudwatch": { - "namespace": "AWS/SNS" - }, - "dimensions": { - "TopicName": "vpc-flow-logs-sns-topic" } }, + "cloud": { + "account": { + "id": "428152502467", + "name": "elastic-beats" + }, + "provider": "aws", + "region": "us-east-1" + }, + "data_stream": { + "dataset": "aws.sns", + "namespace": "default", + "type": "metrics" + }, + "ecs": { + "version": "8.11.0" + }, + "elastic_agent": { + "id": "2d4b09d0-cdb6-445e-ac3f-6415f87b9864", + "snapshot": false, + "version": "8.3.2" + }, "event": { - "duration": 10483932100, "agent_id_status": "verified", + "dataset": "aws.sns", + "duration": 10483932100, "ingested": "2022-07-26T22:01:00Z", - "module": "aws", - "dataset": "aws.sns" + "module": "aws" + }, + "metricset": { + "name": "cloudwatch", + "period": 300000 + }, + "service": { + "type": "aws" } } ``` +**ECS Field Reference** + +Please refer to the following [document](https://www.elastic.co/guide/en/ecs/current/ecs-field-reference.html) for detailed information on ECS fields. + **Exported fields** | Field | Description | Type | Metric Type | @@ -137,44 +141,13 @@ An example event for `sns` looks as following: | aws.sns.metrics.SMSMonthToDateSpentUSD.sum | The charges you have accrued since the start of the current calendar month for sending SMS messages. | long | gauge | | aws.sns.metrics.SMSSuccessRate.avg | The rate of successful SMS message deliveries. | double | gauge | | aws.tags | Tag key value pairs from aws resources. | flattened | | -| cloud | Fields related to the cloud or infrastructure the events are coming from. | group | | | cloud.account.id | The cloud account or organization id used to identify different entities in a multi-tenant environment. Examples: AWS account id, Google Cloud ORG Id, or other unique identifier. | keyword | | -| cloud.account.name | The cloud account name or alias used to identify different entities in a multi-tenant environment. Examples: AWS account name, Google Cloud ORG display name. | keyword | | -| cloud.availability_zone | Availability zone in which this host, resource, or service is located. | keyword | | | cloud.image.id | Image ID for the cloud instance. | keyword | | -| cloud.instance.id | Instance ID of the host machine. | keyword | | -| cloud.instance.name | Instance name of the host machine. | keyword | | -| cloud.machine.type | Machine type of the host machine. | keyword | | -| cloud.project.id | The cloud project identifier. Examples: Google Cloud Project id, Azure Project id. | keyword | | -| cloud.provider | Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. | keyword | | | cloud.region | Region in which this host, resource, or service is located. | keyword | | -| container.id | Unique container id. | keyword | | -| container.image.name | Name of the image the container was built on. | keyword | | -| container.labels | Image labels. | object | | -| container.name | Container name. | keyword | | | data_stream.dataset | Data stream dataset. | constant_keyword | | | data_stream.namespace | Data stream namespace. | constant_keyword | | | data_stream.type | Data stream type. | constant_keyword | | -| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | | -| error | These fields can represent errors of any kind. Use them for errors that happen while fetching events or in cases where the event itself contains an error. | group | | -| error.message | Error message. | match_only_text | | -| event.dataset | Name of the dataset. If an event source publishes more than one type of log or events (e.g. access log, error log), the dataset is used to specify which one the event comes from. It's recommended but not required to start the dataset name with the module name, followed by a dot, then the dataset name. | constant_keyword | | | event.module | Event module | constant_keyword | | -| host.architecture | Operating system architecture. | keyword | | | host.containerized | If the host is a container. | boolean | | -| host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | | -| host.hostname | Hostname of the host. It normally contains what the `hostname` command returns on the host machine. | keyword | | -| host.id | Unique host id. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`. | keyword | | -| host.ip | Host ip addresses. | ip | | -| host.mac | Host MAC addresses. The notation format from RFC 7042 is suggested: Each octet (that is, 8-bit byte) is represented by two [uppercase] hexadecimal digits giving the value of the octet as an unsigned integer. Successive octets are separated by a hyphen. | keyword | | -| host.name | Name of the host. It can contain what hostname returns on Unix systems, the fully qualified domain name (FQDN), or a name specified by the user. The recommended value is the lowercase FQDN of the host. | keyword | | | host.os.build | OS build information. | keyword | | | host.os.codename | OS codename, if any. | keyword | | -| host.os.family | OS family (such as redhat, debian, freebsd, windows). | keyword | | -| host.os.kernel | Operating system kernel version as a raw string. | keyword | | -| host.os.name | Operating system name, without the version. | keyword | | -| host.os.name.text | Multi-field of `host.os.name`. | match_only_text | | -| host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword | | -| host.os.version | Operating system version as a raw string. | keyword | | -| host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | | -| service.type | The type of the service data is collected from. The type can be used to group and correlate logs and metrics from one service type. Example: If logs or metrics are collected from Elasticsearch, `service.type` would be `elasticsearch`. | keyword | | diff --git a/packages/aws/docs/sqs.md b/packages/aws/docs/sqs.md index 1ddb4fbb37f..bdb625914e3 100644 --- a/packages/aws/docs/sqs.md +++ b/packages/aws/docs/sqs.md @@ -45,81 +45,85 @@ An example event for `sqs` looks as following: { "@timestamp": "2022-07-26T21:43:00.000Z", "agent": { - "name": "docker-fleet-agent", + "ephemeral_id": "cdaaaabb-be7e-432f-816b-bda019fd7c15", "id": "2d4b09d0-cdb6-445e-ac3f-6415f87b9864", + "name": "docker-fleet-agent", "type": "metricbeat", - "ephemeral_id": "cdaaaabb-be7e-432f-816b-bda019fd7c15", "version": "8.3.2" }, - "elastic_agent": { - "id": "2d4b09d0-cdb6-445e-ac3f-6415f87b9864", - "version": "8.3.2", - "snapshot": false - }, - "cloud": { - "provider": "aws", - "region": "eu-central-1", - "account": { - "name": "elastic-beats", - "id": "428152502467" - } - }, - "ecs": { - "version": "8.0.0" - }, - "service": { - "type": "aws" - }, - "data_stream": { - "namespace": "default", - "type": "metrics", - "dataset": "aws.sqs" - }, - "metricset": { - "period": 300000, - "name": "cloudwatch" - }, "aws": { + "cloudwatch": { + "namespace": "AWS/SQS" + }, + "dimensions": { + "QueueName": "filebeat-aws-elb-test" + }, "sqs": { + "empty_receives": 0, "messages": { - "visible": 1518.4, + "delayed": 0, "deleted": 0, "not_visible": 0, - "delayed": 0, "received": 0, - "sent": 0.16666666666666666 - }, - "empty_receives": 0, - "sent_message_size": { - "bytes": 1002 + "sent": 0.16666666666666666, + "visible": 1518.4 }, "oldest_message_age": { "sec": 345605.6 }, "queue": { "name": "filebeat-aws-elb-test" + }, + "sent_message_size": { + "bytes": 1002 } }, - "cloudwatch": { - "namespace": "AWS/SQS" - }, - "dimensions": { - "QueueName": "filebeat-aws-elb-test" - }, "tags": { "created-by": "kaiyan" } }, + "cloud": { + "account": { + "id": "428152502467", + "name": "elastic-beats" + }, + "provider": "aws", + "region": "eu-central-1" + }, + "data_stream": { + "dataset": "aws.sqs", + "namespace": "default", + "type": "metrics" + }, + "ecs": { + "version": "8.11.0" + }, + "elastic_agent": { + "id": "2d4b09d0-cdb6-445e-ac3f-6415f87b9864", + "snapshot": false, + "version": "8.3.2" + }, "event": { - "duration": 11576777300, "agent_id_status": "verified", + "dataset": "aws.sqs", + "duration": 11576777300, "ingested": "2022-07-26T21:47:48Z", - "module": "aws", - "dataset": "aws.sqs" + "module": "aws" + }, + "metricset": { + "name": "cloudwatch", + "period": 300000 + }, + "service": { + "type": "aws" } } ``` +**ECS Field Reference** + +Please refer to the following [document](https://www.elastic.co/guide/en/ecs/current/ecs-field-reference.html) for detailed information on ECS fields. + **Exported fields** | Field | Description | Type | Metric Type | @@ -139,44 +143,13 @@ An example event for `sqs` looks as following: | aws.sqs.queue.name | SQS queue name | keyword | | | aws.sqs.sent_message_size.bytes | The average size of messages added to a queue. | long | gauge | | aws.tags | Tag key value pairs from aws resources. | flattened | | -| cloud | Fields related to the cloud or infrastructure the events are coming from. | group | | | cloud.account.id | The cloud account or organization id used to identify different entities in a multi-tenant environment. Examples: AWS account id, Google Cloud ORG Id, or other unique identifier. | keyword | | -| cloud.account.name | The cloud account name or alias used to identify different entities in a multi-tenant environment. Examples: AWS account name, Google Cloud ORG display name. | keyword | | -| cloud.availability_zone | Availability zone in which this host, resource, or service is located. | keyword | | | cloud.image.id | Image ID for the cloud instance. | keyword | | -| cloud.instance.id | Instance ID of the host machine. | keyword | | -| cloud.instance.name | Instance name of the host machine. | keyword | | -| cloud.machine.type | Machine type of the host machine. | keyword | | -| cloud.project.id | The cloud project identifier. Examples: Google Cloud Project id, Azure Project id. | keyword | | -| cloud.provider | Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. | keyword | | | cloud.region | Region in which this host, resource, or service is located. | keyword | | -| container.id | Unique container id. | keyword | | -| container.image.name | Name of the image the container was built on. | keyword | | -| container.labels | Image labels. | object | | -| container.name | Container name. | keyword | | | data_stream.dataset | Data stream dataset. | constant_keyword | | | data_stream.namespace | Data stream namespace. | constant_keyword | | | data_stream.type | Data stream type. | constant_keyword | | -| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | | -| error | These fields can represent errors of any kind. Use them for errors that happen while fetching events or in cases where the event itself contains an error. | group | | -| error.message | Error message. | match_only_text | | -| event.dataset | Name of the dataset. If an event source publishes more than one type of log or events (e.g. access log, error log), the dataset is used to specify which one the event comes from. It's recommended but not required to start the dataset name with the module name, followed by a dot, then the dataset name. | constant_keyword | | | event.module | Event module | constant_keyword | | -| host.architecture | Operating system architecture. | keyword | | | host.containerized | If the host is a container. | boolean | | -| host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | | -| host.hostname | Hostname of the host. It normally contains what the `hostname` command returns on the host machine. | keyword | | -| host.id | Unique host id. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`. | keyword | | -| host.ip | Host ip addresses. | ip | | -| host.mac | Host MAC addresses. The notation format from RFC 7042 is suggested: Each octet (that is, 8-bit byte) is represented by two [uppercase] hexadecimal digits giving the value of the octet as an unsigned integer. Successive octets are separated by a hyphen. | keyword | | -| host.name | Name of the host. It can contain what hostname returns on Unix systems, the fully qualified domain name (FQDN), or a name specified by the user. The recommended value is the lowercase FQDN of the host. | keyword | | | host.os.build | OS build information. | keyword | | | host.os.codename | OS codename, if any. | keyword | | -| host.os.family | OS family (such as redhat, debian, freebsd, windows). | keyword | | -| host.os.kernel | Operating system kernel version as a raw string. | keyword | | -| host.os.name | Operating system name, without the version. | keyword | | -| host.os.name.text | Multi-field of `host.os.name`. | match_only_text | | -| host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword | | -| host.os.version | Operating system version as a raw string. | keyword | | -| host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | | -| service.type | The type of the service data is collected from. The type can be used to group and correlate logs and metrics from one service type. Example: If logs or metrics are collected from Elasticsearch, `service.type` would be `elasticsearch`. | keyword | | diff --git a/packages/aws/docs/transitgateway.md b/packages/aws/docs/transitgateway.md index 48ad557cc7a..6eb4cd0963f 100644 --- a/packages/aws/docs/transitgateway.md +++ b/packages/aws/docs/transitgateway.md @@ -43,109 +43,113 @@ An example event for `transitgateway` looks as following: ```json { + "@timestamp": "2022-07-26T21:58:00.000Z", "agent": { - "name": "a20ad158868c", - "id": "ac8c5411-b1d9-486a-baf7-a719744b13e5", "ephemeral_id": "d43b281f-9a3e-48be-a7b2-e70c0d0b9acd", - "type": "metricbeat", - "version": "8.1.0" - }, - "elastic_agent": { "id": "ac8c5411-b1d9-486a-baf7-a719744b13e5", - "version": "8.1.0", - "snapshot": false - }, - "cloud": { - "provider": "aws", - "region": "eu-west-1", - "account": { - "name": "elastic-observability", - "id": "627286350134" - } - }, - "@timestamp": "2022-07-26T21:58:00.000Z", - "ecs": { - "version": "8.0.0" - }, - "data_stream": { - "namespace": "default", - "type": "metrics", - "dataset": "aws.transitgateway" - }, - "service": { - "type": "aws" - }, - "host": { - "hostname": "a20ad158868c", - "os": { - "kernel": "5.10.104-linuxkit", - "codename": "focal", - "name": "Ubuntu", - "type": "linux", - "family": "debian", - "version": "20.04.3 LTS (Focal Fossa)", - "platform": "ubuntu" - }, - "containerized": false, - "ip": [ - "172.20.0.7" - ], "name": "a20ad158868c", - "mac": [ - "02-42-AC-14-00-07" - ], - "architecture": "aarch64" - }, - "metricset": { - "period": 60000, - "name": "cloudwatch" + "type": "metricbeat", + "version": "8.1.0" }, "aws": { "cloudwatch": { "namespace": "AWS/TransitGateway" }, + "dimensions": { + "TransitGateway": "tgw-04653af6191a63891" + }, "transitgateway": { "metrics": { - "PacketsOut": { + "BytesDropCountBlackhole": { "sum": 0 }, "BytesDropCountNoRoute": { "sum": 0 }, - "PacketDropCountNoRoute": { + "BytesIn": { "sum": 0 }, "BytesOut": { "sum": 0 }, - "BytesIn": { + "PacketDropCountBlackhole": { "sum": 0 }, - "PacketsIn": { + "PacketDropCountNoRoute": { "sum": 0 }, - "BytesDropCountBlackhole": { + "PacketsIn": { "sum": 0 }, - "PacketDropCountBlackhole": { + "PacketsOut": { "sum": 0 } } - }, - "dimensions": { - "TransitGateway": "tgw-04653af6191a63891" } }, + "cloud": { + "account": { + "id": "627286350134", + "name": "elastic-observability" + }, + "provider": "aws", + "region": "eu-west-1" + }, + "data_stream": { + "dataset": "aws.transitgateway", + "namespace": "default", + "type": "metrics" + }, + "ecs": { + "version": "8.11.0" + }, + "elastic_agent": { + "id": "ac8c5411-b1d9-486a-baf7-a719744b13e5", + "snapshot": false, + "version": "8.1.0" + }, "event": { - "duration": 1614567042, "agent_id_status": "verified", + "dataset": "aws.transitgateway", + "duration": 1614567042, "ingested": "2022-07-26T21:59:04Z", - "module": "aws", - "dataset": "aws.transitgateway" + "module": "aws" + }, + "host": { + "architecture": "aarch64", + "containerized": false, + "hostname": "a20ad158868c", + "ip": [ + "172.20.0.7" + ], + "mac": [ + "02-42-AC-14-00-07" + ], + "name": "a20ad158868c", + "os": { + "codename": "focal", + "family": "debian", + "kernel": "5.10.104-linuxkit", + "name": "Ubuntu", + "platform": "ubuntu", + "type": "linux", + "version": "20.04.3 LTS (Focal Fossa)" + } + }, + "metricset": { + "name": "cloudwatch", + "period": 60000 + }, + "service": { + "type": "aws" } } ``` +**ECS Field Reference** + +Please refer to the following [document](https://www.elastic.co/guide/en/ecs/current/ecs-field-reference.html) for detailed information on ECS fields. + **Exported fields** | Field | Description | Type | Metric Type | @@ -164,44 +168,13 @@ An example event for `transitgateway` looks as following: | aws.transitgateway.metrics.PacketDropCountNoRoute.sum | The number of packets dropped because they did not match a route. | long | gauge | | aws.transitgateway.metrics.PacketsIn.sum | The number of packets received by the transit gateway. | long | gauge | | aws.transitgateway.metrics.PacketsOut.sum | The number of packets sent by the transit gateway. | long | gauge | -| cloud | Fields related to the cloud or infrastructure the events are coming from. | group | | | cloud.account.id | The cloud account or organization id used to identify different entities in a multi-tenant environment. Examples: AWS account id, Google Cloud ORG Id, or other unique identifier. | keyword | | -| cloud.account.name | The cloud account name or alias used to identify different entities in a multi-tenant environment. Examples: AWS account name, Google Cloud ORG display name. | keyword | | -| cloud.availability_zone | Availability zone in which this host, resource, or service is located. | keyword | | | cloud.image.id | Image ID for the cloud instance. | keyword | | -| cloud.instance.id | Instance ID of the host machine. | keyword | | -| cloud.instance.name | Instance name of the host machine. | keyword | | -| cloud.machine.type | Machine type of the host machine. | keyword | | -| cloud.project.id | The cloud project identifier. Examples: Google Cloud Project id, Azure Project id. | keyword | | -| cloud.provider | Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. | keyword | | | cloud.region | Region in which this host, resource, or service is located. | keyword | | -| container.id | Unique container id. | keyword | | -| container.image.name | Name of the image the container was built on. | keyword | | -| container.labels | Image labels. | object | | -| container.name | Container name. | keyword | | | data_stream.dataset | Data stream dataset. | constant_keyword | | | data_stream.namespace | Data stream namespace. | constant_keyword | | | data_stream.type | Data stream type. | constant_keyword | | -| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | | -| error | These fields can represent errors of any kind. Use them for errors that happen while fetching events or in cases where the event itself contains an error. | group | | -| error.message | Error message. | match_only_text | | -| event.dataset | Name of the dataset. If an event source publishes more than one type of log or events (e.g. access log, error log), the dataset is used to specify which one the event comes from. It's recommended but not required to start the dataset name with the module name, followed by a dot, then the dataset name. | constant_keyword | | | event.module | Event module | constant_keyword | | -| host.architecture | Operating system architecture. | keyword | | | host.containerized | If the host is a container. | boolean | | -| host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | | -| host.hostname | Hostname of the host. It normally contains what the `hostname` command returns on the host machine. | keyword | | -| host.id | Unique host id. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`. | keyword | | -| host.ip | Host ip addresses. | ip | | -| host.mac | Host MAC addresses. The notation format from RFC 7042 is suggested: Each octet (that is, 8-bit byte) is represented by two [uppercase] hexadecimal digits giving the value of the octet as an unsigned integer. Successive octets are separated by a hyphen. | keyword | | -| host.name | Name of the host. It can contain what hostname returns on Unix systems, the fully qualified domain name (FQDN), or a name specified by the user. The recommended value is the lowercase FQDN of the host. | keyword | | | host.os.build | OS build information. | keyword | | | host.os.codename | OS codename, if any. | keyword | | -| host.os.family | OS family (such as redhat, debian, freebsd, windows). | keyword | | -| host.os.kernel | Operating system kernel version as a raw string. | keyword | | -| host.os.name | Operating system name, without the version. | keyword | | -| host.os.name.text | Multi-field of `host.os.name`. | match_only_text | | -| host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword | | -| host.os.version | Operating system version as a raw string. | keyword | | -| host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | | -| service.type | The type of the service data is collected from. The type can be used to group and correlate logs and metrics from one service type. Example: If logs or metrics are collected from Elasticsearch, `service.type` would be `elasticsearch`. | keyword | | diff --git a/packages/aws/docs/usage.md b/packages/aws/docs/usage.md index 8f2fef7be72..775c491eb54 100644 --- a/packages/aws/docs/usage.md +++ b/packages/aws/docs/usage.md @@ -47,68 +47,72 @@ An example event for `usage` looks as following: { "@timestamp": "2022-07-25T20:50:00.000Z", "agent": { - "name": "docker-fleet-agent", + "ephemeral_id": "6bab70d4-84d9-411d-887c-f144d4244e78", "id": "2d4b09d0-cdb6-445e-ac3f-6415f87b9864", + "name": "docker-fleet-agent", "type": "metricbeat", - "ephemeral_id": "6bab70d4-84d9-411d-887c-f144d4244e78", "version": "8.3.2" }, - "elastic_agent": { - "id": "2d4b09d0-cdb6-445e-ac3f-6415f87b9864", - "version": "8.3.2", - "snapshot": false - }, - "cloud": { - "provider": "aws", - "region": "eu-north-1", - "account": { - "name": "elastic-beats", - "id": "428152502467" - } - }, - "ecs": { - "version": "8.0.0" - }, - "service": { - "type": "aws" - }, - "data_stream": { - "namespace": "default", - "type": "metrics", - "dataset": "aws.usage" - }, - "metricset": { - "period": 60000, - "name": "cloudwatch" - }, "aws": { - "usage": { - "metrics": { - "CallCount": { - "sum": 1 - } - } - }, "cloudwatch": { "namespace": "AWS/Usage" }, "dimensions": { - "Type": "API", + "Class": "None", "Resource": "ListMetrics", "Service": "CloudWatch", - "Class": "None" + "Type": "API" + }, + "usage": { + "metrics": { + "CallCount": { + "sum": 1 + } + } } }, + "cloud": { + "account": { + "id": "428152502467", + "name": "elastic-beats" + }, + "provider": "aws", + "region": "eu-north-1" + }, + "data_stream": { + "dataset": "aws.usage", + "namespace": "default", + "type": "metrics" + }, + "ecs": { + "version": "8.11.0" + }, + "elastic_agent": { + "id": "2d4b09d0-cdb6-445e-ac3f-6415f87b9864", + "snapshot": false, + "version": "8.3.2" + }, "event": { - "duration": 1432082500, "agent_id_status": "verified", + "dataset": "aws.usage", + "duration": 1432082500, "ingested": "2022-07-25T20:51:19Z", - "module": "aws", - "dataset": "aws.usage" + "module": "aws" + }, + "metricset": { + "name": "cloudwatch", + "period": 60000 + }, + "service": { + "type": "aws" } } ``` +**ECS Field Reference** + +Please refer to the following [document](https://www.elastic.co/guide/en/ecs/current/ecs-field-reference.html) for detailed information on ECS fields. + **Exported fields** | Field | Description | Type | Metric Type | @@ -123,44 +127,13 @@ An example event for `usage` looks as following: | aws.tags | Tag key value pairs from aws resources. | flattened | | | aws.usage.metrics.CallCount.sum | The number of specified API operations performed in your account. | long | gauge | | aws.usage.metrics.ResourceCount.sum | The number of the specified resources running in your account. The resources are defined by the dimensions associated with the metric. | long | gauge | -| cloud | Fields related to the cloud or infrastructure the events are coming from. | group | | | cloud.account.id | The cloud account or organization id used to identify different entities in a multi-tenant environment. Examples: AWS account id, Google Cloud ORG Id, or other unique identifier. | keyword | | -| cloud.account.name | The cloud account name or alias used to identify different entities in a multi-tenant environment. Examples: AWS account name, Google Cloud ORG display name. | keyword | | -| cloud.availability_zone | Availability zone in which this host, resource, or service is located. | keyword | | | cloud.image.id | Image ID for the cloud instance. | keyword | | -| cloud.instance.id | Instance ID of the host machine. | keyword | | -| cloud.instance.name | Instance name of the host machine. | keyword | | -| cloud.machine.type | Machine type of the host machine. | keyword | | -| cloud.project.id | The cloud project identifier. Examples: Google Cloud Project id, Azure Project id. | keyword | | -| cloud.provider | Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. | keyword | | | cloud.region | Region in which this host, resource, or service is located. | keyword | | -| container.id | Unique container id. | keyword | | -| container.image.name | Name of the image the container was built on. | keyword | | -| container.labels | Image labels. | object | | -| container.name | Container name. | keyword | | | data_stream.dataset | Data stream dataset. | constant_keyword | | | data_stream.namespace | Data stream namespace. | constant_keyword | | | data_stream.type | Data stream type. | constant_keyword | | -| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | | -| error | These fields can represent errors of any kind. Use them for errors that happen while fetching events or in cases where the event itself contains an error. | group | | -| error.message | Error message. | match_only_text | | -| event.dataset | Name of the dataset. If an event source publishes more than one type of log or events (e.g. access log, error log), the dataset is used to specify which one the event comes from. It's recommended but not required to start the dataset name with the module name, followed by a dot, then the dataset name. | constant_keyword | | | event.module | Event module | constant_keyword | | -| host.architecture | Operating system architecture. | keyword | | | host.containerized | If the host is a container. | boolean | | -| host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | | -| host.hostname | Hostname of the host. It normally contains what the `hostname` command returns on the host machine. | keyword | | -| host.id | Unique host id. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`. | keyword | | -| host.ip | Host ip addresses. | ip | | -| host.mac | Host MAC addresses. The notation format from RFC 7042 is suggested: Each octet (that is, 8-bit byte) is represented by two [uppercase] hexadecimal digits giving the value of the octet as an unsigned integer. Successive octets are separated by a hyphen. | keyword | | -| host.name | Name of the host. It can contain what hostname returns on Unix systems, the fully qualified domain name (FQDN), or a name specified by the user. The recommended value is the lowercase FQDN of the host. | keyword | | | host.os.build | OS build information. | keyword | | | host.os.codename | OS codename, if any. | keyword | | -| host.os.family | OS family (such as redhat, debian, freebsd, windows). | keyword | | -| host.os.kernel | Operating system kernel version as a raw string. | keyword | | -| host.os.name | Operating system name, without the version. | keyword | | -| host.os.name.text | Multi-field of `host.os.name`. | match_only_text | | -| host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword | | -| host.os.version | Operating system version as a raw string. | keyword | | -| host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | | -| service.type | The type of the service data is collected from. The type can be used to group and correlate logs and metrics from one service type. Example: If logs or metrics are collected from Elasticsearch, `service.type` would be `elasticsearch`. | keyword | | diff --git a/packages/aws/docs/vpcflow.md b/packages/aws/docs/vpcflow.md index 97cb387de83..95a85f4d91f 100644 --- a/packages/aws/docs/vpcflow.md +++ b/packages/aws/docs/vpcflow.md @@ -80,6 +80,10 @@ The `number_of_workers` setting defines the number of workers assigned to readin > Note: The Parquet format is not supported. +**ECS Field Reference** + +Please refer to the following [document](https://www.elastic.co/guide/en/ecs/current/ecs-field-reference.html) for detailed information on ECS fields. + **Exported fields** | Field | Description | Type | @@ -107,90 +111,16 @@ The `number_of_workers` setting defines the number of workers assigned to readin | aws.vpcflow.type | The type of traffic: IPv4, IPv6, or EFA. | keyword | | aws.vpcflow.version | The VPC Flow Logs version. If you use the default format, the version is 2. If you specify a custom format, the version is 3. | keyword | | aws.vpcflow.vpc_id | The ID of the VPC that contains the network interface for which the traffic is recorded. | keyword | -| cloud.account.id | The cloud account or organization id used to identify different entities in a multi-tenant environment. Examples: AWS account id, Google Cloud ORG Id, or other unique identifier. | keyword | -| cloud.availability_zone | Availability zone in which this host, resource, or service is located. | keyword | | cloud.image.id | Image ID for the cloud instance. | keyword | -| cloud.instance.id | Instance ID of the host machine. | keyword | -| cloud.instance.name | Instance name of the host machine. | keyword | -| cloud.machine.type | Machine type of the host machine. | keyword | -| cloud.project.id | The cloud project identifier. Examples: Google Cloud Project id, Azure Project id. | keyword | -| cloud.provider | Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. | keyword | -| cloud.region | Region in which this host, resource, or service is located. | keyword | -| container.id | Unique container id. | keyword | -| container.image.name | Name of the image the container was built on. | keyword | -| container.labels | Image labels. | object | -| container.name | Container name. | keyword | | data_stream.dataset | Data stream dataset. | constant_keyword | | data_stream.namespace | Data stream namespace. | constant_keyword | | data_stream.type | Data stream type. | constant_keyword | -| destination.address | Some event destination addresses are defined ambiguously. The event will sometimes list an IP, a domain or a unix socket. You should always store the raw address in the `.address` field. Then it should be duplicated to `.ip` or `.domain`, depending on which one it is. | keyword | -| destination.as.number | Unique number allocated to the autonomous system. The autonomous system number (ASN) uniquely identifies each network on the Internet. | long | -| destination.as.organization.name | Organization name. | keyword | -| destination.as.organization.name.text | Multi-field of `destination.as.organization.name`. | match_only_text | -| destination.geo.city_name | City name. | keyword | -| destination.geo.continent_name | Name of the continent. | keyword | -| destination.geo.country_iso_code | Country ISO code. | keyword | -| destination.geo.country_name | Country name. | keyword | -| destination.geo.location | Longitude and latitude. | geo_point | -| destination.geo.region_iso_code | Region ISO code. | keyword | -| destination.geo.region_name | Region name. | keyword | -| destination.ip | IP address of the destination (IPv4 or IPv6). | ip | -| destination.port | Port of the destination. | long | -| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | -| error.message | Error message. | match_only_text | -| event.category | This is one of four ECS Categorization Fields, and indicates the second level in the ECS category hierarchy. `event.category` represents the "big buckets" of ECS categories. For example, filtering on `event.category:process` yields all events relating to process activity. This field is closely related to `event.type`, which is used as a subcategory. This field is an array. This will allow proper categorization of some events that fall in multiple categories. | keyword | -| event.dataset | Name of the dataset. If an event source publishes more than one type of log or events (e.g. access log, error log), the dataset is used to specify which one the event comes from. It's recommended but not required to start the dataset name with the module name, followed by a dot, then the dataset name. | constant_keyword | -| event.end | `event.end` contains the date when the event ended or when the activity was last observed. | date | -| event.kind | This is one of four ECS Categorization Fields, and indicates the highest level in the ECS category hierarchy. `event.kind` gives high-level information about what type of information the event contains, without being specific to the contents of the event. For example, values of this field distinguish alert events from metric events. The value of this field can be used to inform how these kinds of events should be handled. They may warrant different retention, different access control, it may also help understand whether the data is coming in at a regular interval or not. | keyword | | event.module | Event module | constant_keyword | -| event.original | Raw text message of entire event. Used to demonstrate log integrity or where the full log message (before splitting it up in multiple parts) may be required, e.g. for reindex. This field is not indexed and doc_values are disabled. It cannot be searched, but it can be retrieved from `_source`. If users wish to override this and index this field, please see `Field data types` in the `Elasticsearch Reference`. | keyword | -| event.outcome | This is one of four ECS Categorization Fields, and indicates the lowest level in the ECS category hierarchy. `event.outcome` simply denotes whether the event represents a success or a failure from the perspective of the entity that produced the event. Note that when a single transaction is described in multiple events, each event may populate different values of `event.outcome`, according to their perspective. Also note that in the case of a compound event (a single event that contains multiple logical events), this field should be populated with the value that best captures the overall success or failure from the perspective of the event producer. Further note that not all events will have an associated outcome. For example, this field is generally not populated for metric events, events with `event.type:info`, or any events for which an outcome does not make logical sense. | keyword | -| event.start | `event.start` contains the date when the event started or when the activity was first observed. | date | -| event.type | This is one of four ECS Categorization Fields, and indicates the third level in the ECS category hierarchy. `event.type` represents a categorization "sub-bucket" that, when used along with the `event.category` field values, enables filtering events down to a level appropriate for single visualization. This field is an array. This will allow proper categorization of some events that fall in multiple event types. | keyword | -| host.architecture | Operating system architecture. | keyword | | host.containerized | If the host is a container. | boolean | -| host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | -| host.hostname | Hostname of the host. It normally contains what the `hostname` command returns on the host machine. | keyword | -| host.id | Unique host id. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`. | keyword | -| host.ip | Host ip addresses. | ip | -| host.mac | Host MAC addresses. The notation format from RFC 7042 is suggested: Each octet (that is, 8-bit byte) is represented by two [uppercase] hexadecimal digits giving the value of the octet as an unsigned integer. Successive octets are separated by a hyphen. | keyword | -| host.name | Name of the host. It can contain what hostname returns on Unix systems, the fully qualified domain name (FQDN), or a name specified by the user. The recommended value is the lowercase FQDN of the host. | keyword | | host.os.build | OS build information. | keyword | | host.os.codename | OS codename, if any. | keyword | -| host.os.family | OS family (such as redhat, debian, freebsd, windows). | keyword | -| host.os.kernel | Operating system kernel version as a raw string. | keyword | -| host.os.name | Operating system name, without the version. | keyword | -| host.os.name.text | Multi-field of `host.os.name`. | match_only_text | -| host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword | -| host.os.version | Operating system version as a raw string. | keyword | -| host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | | input.type | Input type | keyword | -| log.file.path | Full path to the log file this event came from, including the file name. It should include the drive letter, when appropriate. If the event wasn't read from a log file, do not populate this field. | keyword | | log.offset | Log offset | long | -| network.bytes | Total bytes transferred in both directions. If `source.bytes` and `destination.bytes` are known, `network.bytes` is their sum. | long | -| network.community_id | A hash of source and destination IPs and ports, as well as the protocol used in a communication. This is a tool-agnostic standard to identify flows. Learn more at https://github.com/corelight/community-id-spec. | keyword | -| network.direction | Direction of the network traffic. When mapping events from a host-based monitoring context, populate this field from the host's point of view, using the values "ingress" or "egress". When mapping events from a network or perimeter-based monitoring context, populate this field from the point of view of the network perimeter, using the values "inbound", "outbound", "internal" or "external". Note that "internal" is not crossing perimeter boundaries, and is meant to describe communication between two hosts within the perimeter. Note also that "external" is meant to describe traffic between two hosts that are external to the perimeter. This could for example be useful for ISPs or VPN service providers. | keyword | -| network.iana_number | IANA Protocol Number (https://www.iana.org/assignments/protocol-numbers/protocol-numbers.xhtml). Standardized list of protocols. This aligns well with NetFlow and sFlow related logs which use the IANA Protocol Number. | keyword | -| network.packets | Total packets transferred in both directions. If `source.packets` and `destination.packets` are known, `network.packets` is their sum. | long | -| network.transport | Same as network.iana_number, but instead using the Keyword name of the transport layer (udp, tcp, ipv6-icmp, etc.) The field value must be normalized to lowercase for querying. | keyword | -| network.type | In the OSI Model this would be the Network Layer. ipv4, ipv6, ipsec, pim, etc The field value must be normalized to lowercase for querying. | keyword | -| related.ip | All of the IPs seen on your event. | ip | -| source.address | Some event source addresses are defined ambiguously. The event will sometimes list an IP, a domain or a unix socket. You should always store the raw address in the `.address` field. Then it should be duplicated to `.ip` or `.domain`, depending on which one it is. | keyword | -| source.as.number | Unique number allocated to the autonomous system. The autonomous system number (ASN) uniquely identifies each network on the Internet. | long | -| source.as.organization.name | Organization name. | keyword | -| source.as.organization.name.text | Multi-field of `source.as.organization.name`. | match_only_text | -| source.bytes | Bytes sent from the source to the destination. | long | -| source.geo.city_name | City name. | keyword | -| source.geo.continent_name | Name of the continent. | keyword | -| source.geo.country_iso_code | Country ISO code. | keyword | -| source.geo.country_name | Country name. | keyword | -| source.geo.location | Longitude and latitude. | geo_point | -| source.geo.region_iso_code | Region ISO code. | keyword | -| source.geo.region_name | Region name. | keyword | -| source.ip | IP address of the source (IPv4 or IPv6). | ip | -| source.packets | Packets sent from the source to the destination. | long | -| source.port | Port of the source. | long | -| tags | List of keywords used to tag each event. | keyword | An example event for `vpcflow` looks as following: @@ -259,7 +189,7 @@ An example event for `vpcflow` looks as following: "port": 22 }, "ecs": { - "version": "8.0.0" + "version": "8.11.0" }, "elastic_agent": { "id": "acba78ef-1401-4689-977c-d8c2e5d6a8fa", diff --git a/packages/aws/docs/vpn.md b/packages/aws/docs/vpn.md index 917701e2241..ca8364c4fd9 100644 --- a/packages/aws/docs/vpn.md +++ b/packages/aws/docs/vpn.md @@ -44,57 +44,61 @@ An example event for `vpn` looks as following: ```json { "@timestamp": "2020-05-28T17:58:27.154Z", - "service": { - "type": "aws" - }, - "ecs": { - "version": "1.5.0" + "agent": { + "ephemeral_id": "17803f33-b617-4ce9-a9ac-e218c02aeb4b", + "id": "12f376ef-5186-4e8b-a175-70f1140a8f30", + "name": "MacBook-Elastic.local", + "type": "metricbeat", + "version": "8.0.0" }, "aws": { + "cloudwatch": { + "namespace": "AWS/VPN" + }, "vpn": { "metrics": { - "TunnelState": { - "avg": 0 - }, "TunnelDataIn": { "sum": 0 }, "TunnelDataOut": { "sum": 0 + }, + "TunnelState": { + "avg": 0 } } - }, - "cloudwatch": { - "namespace": "AWS/VPN" } }, + "cloud": { + "account": { + "id": "428152502467", + "name": "elastic-beats" + }, + "provider": "aws", + "region": "us-west-2" + }, + "ecs": { + "version": "8.11.0" + }, "event": { "dataset": "aws.vpn", - "module": "aws", - "duration": 10418157072 + "duration": 10418157072, + "module": "aws" }, "metricset": { - "period": 60000, - "name": "vpn" + "name": "vpn", + "period": 60000 }, - "cloud": { - "region": "us-west-2", - "account": { - "name": "elastic-beats", - "id": "428152502467" - }, - "provider": "aws" - }, - "agent": { - "version": "8.0.0", - "ephemeral_id": "17803f33-b617-4ce9-a9ac-e218c02aeb4b", - "id": "12f376ef-5186-4e8b-a175-70f1140a8f30", - "name": "MacBook-Elastic.local", - "type": "metricbeat" + "service": { + "type": "aws" } } ``` +**ECS Field Reference** + +Please refer to the following [document](https://www.elastic.co/guide/en/ecs/current/ecs-field-reference.html) for detailed information on ECS fields. + **Exported fields** | Field | Description | Type | Metric Type | @@ -108,44 +112,13 @@ An example event for `vpn` looks as following: | aws.vpn.metrics.TunnelDataIn.sum | The bytes received through the VPN tunnel. | double | gauge | | aws.vpn.metrics.TunnelDataOut.sum | The bytes sent through the VPN tunnel. | double | gauge | | aws.vpn.metrics.TunnelState.avg | The state of the tunnel. For static VPNs, 0 indicates DOWN and 1 indicates UP. For BGP VPNs, 1 indicates ESTABLISHED and 0 is used for all other states. | double | gauge | -| cloud | Fields related to the cloud or infrastructure the events are coming from. | group | | | cloud.account.id | The cloud account or organization id used to identify different entities in a multi-tenant environment. Examples: AWS account id, Google Cloud ORG Id, or other unique identifier. | keyword | | -| cloud.account.name | The cloud account name or alias used to identify different entities in a multi-tenant environment. Examples: AWS account name, Google Cloud ORG display name. | keyword | | -| cloud.availability_zone | Availability zone in which this host, resource, or service is located. | keyword | | | cloud.image.id | Image ID for the cloud instance. | keyword | | -| cloud.instance.id | Instance ID of the host machine. | keyword | | -| cloud.instance.name | Instance name of the host machine. | keyword | | -| cloud.machine.type | Machine type of the host machine. | keyword | | -| cloud.project.id | The cloud project identifier. Examples: Google Cloud Project id, Azure Project id. | keyword | | -| cloud.provider | Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. | keyword | | | cloud.region | Region in which this host, resource, or service is located. | keyword | | -| container.id | Unique container id. | keyword | | -| container.image.name | Name of the image the container was built on. | keyword | | -| container.labels | Image labels. | object | | -| container.name | Container name. | keyword | | | data_stream.dataset | Data stream dataset. | constant_keyword | | | data_stream.namespace | Data stream namespace. | constant_keyword | | | data_stream.type | Data stream type. | constant_keyword | | -| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | | -| error | These fields can represent errors of any kind. Use them for errors that happen while fetching events or in cases where the event itself contains an error. | group | | -| error.message | Error message. | match_only_text | | -| event.dataset | Name of the dataset. If an event source publishes more than one type of log or events (e.g. access log, error log), the dataset is used to specify which one the event comes from. It's recommended but not required to start the dataset name with the module name, followed by a dot, then the dataset name. | constant_keyword | | | event.module | Event module | constant_keyword | | -| host.architecture | Operating system architecture. | keyword | | | host.containerized | If the host is a container. | boolean | | -| host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | | -| host.hostname | Hostname of the host. It normally contains what the `hostname` command returns on the host machine. | keyword | | -| host.id | Unique host id. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`. | keyword | | -| host.ip | Host ip addresses. | ip | | -| host.mac | Host MAC addresses. The notation format from RFC 7042 is suggested: Each octet (that is, 8-bit byte) is represented by two [uppercase] hexadecimal digits giving the value of the octet as an unsigned integer. Successive octets are separated by a hyphen. | keyword | | -| host.name | Name of the host. It can contain what hostname returns on Unix systems, the fully qualified domain name (FQDN), or a name specified by the user. The recommended value is the lowercase FQDN of the host. | keyword | | | host.os.build | OS build information. | keyword | | | host.os.codename | OS codename, if any. | keyword | | -| host.os.family | OS family (such as redhat, debian, freebsd, windows). | keyword | | -| host.os.kernel | Operating system kernel version as a raw string. | keyword | | -| host.os.name | Operating system name, without the version. | keyword | | -| host.os.name.text | Multi-field of `host.os.name`. | match_only_text | | -| host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword | | -| host.os.version | Operating system version as a raw string. | keyword | | -| host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | | -| service.type | The type of the service data is collected from. The type can be used to group and correlate logs and metrics from one service type. Example: If logs or metrics are collected from Elasticsearch, `service.type` would be `elasticsearch`. | keyword | | diff --git a/packages/aws/docs/waf.md b/packages/aws/docs/waf.md index 58d1acf7920..a14dbbc2ed6 100644 --- a/packages/aws/docs/waf.md +++ b/packages/aws/docs/waf.md @@ -63,6 +63,10 @@ The `number_of_workers` setting defines the number of workers assigned to readin The `waf` dataset is specifically for WAF logs. Export logs from Kinesis Data Firehose to Amazon S3 bucket which has SQS notification setup already. +**ECS Field Reference** + +Please refer to the following [document](https://www.elastic.co/guide/en/ecs/current/ecs-field-reference.html) for detailed information on ECS fields. + **Exported fields** | Field | Description | Type | @@ -80,74 +84,16 @@ The `waf` dataset is specifically for WAF logs. Export logs from Kinesis Data Fi | aws.waf.source.id | The source ID. This field shows the ID of the associated resource. | keyword | | aws.waf.source.name | The source of the request. Possible values: CF for Amazon CloudFront, APIGW for Amazon API Gateway, ALB for Application Load Balancer, and APPSYNC for AWS AppSync. | keyword | | aws.waf.terminating_rule_match_details | Detailed information about the terminating rule that matched the request. A terminating rule has an action that ends the inspection process against a web request. Possible actions for a terminating rule are ALLOW and BLOCK. This is only populated for SQL injection and cross-site scripting (XSS) match rule statements. As with all rule statements that inspect for more than one thing, AWS WAF applies the action on the first match and stops inspecting the web request. A web request with a terminating action could contain other threats, in addition to the one reported in the log. | nested | -| cloud.account.id | The cloud account or organization id used to identify different entities in a multi-tenant environment. Examples: AWS account id, Google Cloud ORG Id, or other unique identifier. | keyword | -| cloud.account.name | The cloud account name or alias used to identify different entities in a multi-tenant environment. Examples: AWS account name, Google Cloud ORG display name. | keyword | -| cloud.availability_zone | Availability zone in which this host, resource, or service is located. | keyword | | cloud.image.id | Image ID for the cloud instance. | keyword | -| cloud.instance.id | Instance ID of the host machine. | keyword | -| cloud.instance.name | Instance name of the host machine. | keyword | -| cloud.machine.type | Machine type of the host machine. | keyword | -| cloud.project.id | The cloud project identifier. Examples: Google Cloud Project id, Azure Project id. | keyword | -| cloud.provider | Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. | keyword | -| cloud.region | Region in which this host, resource, or service is located. | keyword | -| container.id | Unique container id. | keyword | -| container.image.name | Name of the image the container was built on. | keyword | -| container.labels | Image labels. | object | -| container.name | Container name. | keyword | | data_stream.dataset | Data stream dataset. | constant_keyword | | data_stream.namespace | Data stream namespace. | constant_keyword | | data_stream.type | Data stream type. | constant_keyword | -| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | -| error.message | Error message. | match_only_text | -| event.action | The action captured by the event. This describes the information in the event. It is more specific than `event.category`. Examples are `group-add`, `process-started`, `file-created`. The value is normally defined by the implementer. | keyword | -| event.dataset | Name of the dataset. If an event source publishes more than one type of log or events (e.g. access log, error log), the dataset is used to specify which one the event comes from. It's recommended but not required to start the dataset name with the module name, followed by a dot, then the dataset name. | constant_keyword | -| event.id | Unique ID to describe the event. | keyword | -| event.kind | This is one of four ECS Categorization Fields, and indicates the highest level in the ECS category hierarchy. `event.kind` gives high-level information about what type of information the event contains, without being specific to the contents of the event. For example, values of this field distinguish alert events from metric events. The value of this field can be used to inform how these kinds of events should be handled. They may warrant different retention, different access control, it may also help understand whether the data is coming in at a regular interval or not. | keyword | | event.module | Event module | constant_keyword | -| event.outcome | This is one of four ECS Categorization Fields, and indicates the lowest level in the ECS category hierarchy. `event.outcome` simply denotes whether the event represents a success or a failure from the perspective of the entity that produced the event. Note that when a single transaction is described in multiple events, each event may populate different values of `event.outcome`, according to their perspective. Also note that in the case of a compound event (a single event that contains multiple logical events), this field should be populated with the value that best captures the overall success or failure from the perspective of the event producer. Further note that not all events will have an associated outcome. For example, this field is generally not populated for metric events, events with `event.type:info`, or any events for which an outcome does not make logical sense. | keyword | -| host.architecture | Operating system architecture. | keyword | | host.containerized | If the host is a container. | boolean | -| host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | -| host.hostname | Hostname of the host. It normally contains what the `hostname` command returns on the host machine. | keyword | -| host.id | Unique host id. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`. | keyword | -| host.ip | Host ip addresses. | ip | -| host.mac | Host MAC addresses. The notation format from RFC 7042 is suggested: Each octet (that is, 8-bit byte) is represented by two [uppercase] hexadecimal digits giving the value of the octet as an unsigned integer. Successive octets are separated by a hyphen. | keyword | -| host.name | Name of the host. It can contain what hostname returns on Unix systems, the fully qualified domain name (FQDN), or a name specified by the user. The recommended value is the lowercase FQDN of the host. | keyword | | host.os.build | OS build information. | keyword | | host.os.codename | OS codename, if any. | keyword | -| host.os.family | OS family (such as redhat, debian, freebsd, windows). | keyword | -| host.os.kernel | Operating system kernel version as a raw string. | keyword | -| host.os.name | Operating system name, without the version. | keyword | -| host.os.name.text | Multi-field of `host.os.name`. | match_only_text | -| host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword | -| host.os.version | Operating system version as a raw string. | keyword | -| host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | -| http.request.id | A unique identifier for each HTTP request to correlate logs between clients and servers in transactions. The id may be contained in a non-standard HTTP header, such as `X-Request-ID` or `X-Correlation-ID`. | keyword | -| http.request.method | HTTP request method. The value should retain its casing from the original event. For example, `GET`, `get`, and `GeT` are all considered valid values for this field. | keyword | -| http.version | HTTP version. | keyword | | input.type | Input type | keyword | -| log.file.path | Full path to the log file this event came from, including the file name. It should include the drive letter, when appropriate. If the event wasn't read from a log file, do not populate this field. | keyword | | log.offset | Log offset | long | -| network.protocol | In the OSI Model this would be the Application Layer protocol. For example, `http`, `dns`, or `ssh`. The field value must be normalized to lowercase for querying. | keyword | -| network.transport | Same as network.iana_number, but instead using the Keyword name of the transport layer (udp, tcp, ipv6-icmp, etc.) The field value must be normalized to lowercase for querying. | keyword | -| related.ip | All of the IPs seen on your event. | ip | -| rule.id | A rule ID that is unique within the scope of an agent, observer, or other entity using the rule for detection of this event. | keyword | -| rule.ruleset | Name of the ruleset, policy, group, or parent category in which the rule used to generate this event is a member. | keyword | -| source.address | Some event source addresses are defined ambiguously. The event will sometimes list an IP, a domain or a unix socket. You should always store the raw address in the `.address` field. Then it should be duplicated to `.ip` or `.domain`, depending on which one it is. | keyword | -| source.as.number | Unique number allocated to the autonomous system. The autonomous system number (ASN) uniquely identifies each network on the Internet. | long | -| source.as.organization.name | Organization name. | keyword | -| source.as.organization.name.text | Multi-field of `source.as.organization.name`. | match_only_text | -| source.geo.city_name | City name. | keyword | -| source.geo.continent_name | Name of the continent. | keyword | -| source.geo.country_iso_code | Country ISO code. | keyword | -| source.geo.country_name | Country name. | keyword | -| source.geo.location | Longitude and latitude. | geo_point | -| source.geo.region_iso_code | Region ISO code. | keyword | -| source.geo.region_name | Region name. | keyword | -| source.ip | IP address of the source (IPv4 or IPv6). | ip | -| tags | List of keywords used to tag each event. | keyword | -| url.path | Path of the request, such as "/search". | wildcard | -| url.query | The query field describes the query string of the request, such as "q=elasticsearch". The `?` is excluded from the query string. If a URL contains no `?`, there is no query field. If there is a `?` but no query, the query field exists with an empty string. The `exists` query can be used to differentiate between the two cases. | keyword | An example event for `waf` looks as following: @@ -212,7 +158,7 @@ An example event for `waf` looks as following: "type": "logs" }, "ecs": { - "version": "8.0.0" + "version": "8.11.0" }, "elastic_agent": { "id": "acba78ef-1401-4689-977c-d8c2e5d6a8fa", @@ -222,7 +168,10 @@ An example event for `waf` looks as following: "event": { "action": "BLOCK", "agent_id_status": "verified", - "category": "web", + "category": [ + "web", + "network" + ], "dataset": "aws.waf", "ingested": "2023-11-08T08:24:54Z", "kind": "event", diff --git a/packages/aws/manifest.yml b/packages/aws/manifest.yml index 2a342d83de2..4ad987500af 100644 --- a/packages/aws/manifest.yml +++ b/packages/aws/manifest.yml @@ -1,7 +1,7 @@ format_version: 3.0.0 name: aws title: AWS -version: 2.16.0 +version: 2.17.0 description: Collect logs and metrics from Amazon Web Services (AWS) with Elastic Agent. type: integration categories: @@ -10,7 +10,7 @@ conditions: elastic: subscription: basic kibana: - version: "^8.12.0" + version: "^8.13.0" screenshots: - src: /img/metricbeat-aws-overview.png title: metricbeat aws overview