From a4406694f5511c345ce78e8177c4201142454cda Mon Sep 17 00:00:00 2001 From: Jan David Date: Tue, 16 Jan 2024 16:41:12 +0100 Subject: [PATCH 1/3] Move resources into the right region The SQS queue must be in the same region as the S3 bucket, otherwise it cannot be configured as the target for notifications. With our current Terragrunt setup, it is not possible to create resources in a different region, so we had to globally change the region that is associated with this account. --- terragrunt/accounts/crates-io-staging/account.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/terragrunt/accounts/crates-io-staging/account.json b/terragrunt/accounts/crates-io-staging/account.json index 4e93e20fc..a3300d35a 100644 --- a/terragrunt/accounts/crates-io-staging/account.json +++ b/terragrunt/accounts/crates-io-staging/account.json @@ -1,6 +1,6 @@ { "aws": { "profile": "crates-io-staging", - "region": "us-east-2" + "region": "us-west-1" } } From 30afce5291b22e561c8e3a87d080728691d0222a Mon Sep 17 00:00:00 2001 From: Jan David Date: Tue, 16 Jan 2024 17:29:13 +0100 Subject: [PATCH 2/3] Send S3 notifications in staging environment The S3 bucket with the CDN logs for crates.io now sends S3 notifications to an SQS queue. This enables crates.io to start counting downloads in the background and not as part of the request-response cycle. --- .../crates-io-staging/crates-io/terragrunt.hcl | 2 ++ terragrunt/modules/crates-io/_terraform.tf | 6 ++++++ terragrunt/modules/crates-io/s3-logs.tf | 18 ++++++++++++++++++ 3 files changed, 26 insertions(+) diff --git a/terragrunt/accounts/legacy/crates-io-staging/crates-io/terragrunt.hcl b/terragrunt/accounts/legacy/crates-io-staging/crates-io/terragrunt.hcl index c29f76d31..28d9ac604 100644 --- a/terragrunt/accounts/legacy/crates-io-staging/crates-io/terragrunt.hcl +++ b/terragrunt/accounts/legacy/crates-io-staging/crates-io/terragrunt.hcl @@ -27,4 +27,6 @@ inputs = { static_fastly_weight = 100 fastly_customer_id_ssm_parameter = "/staging/crates-io/fastly/customer-id" + + cdn_log_event_queue_arn = "arn:aws:sqs:us-west-1:359172468976:cdn-log-event-queue" } diff --git a/terragrunt/modules/crates-io/_terraform.tf b/terragrunt/modules/crates-io/_terraform.tf index 5039979fb..5c8345f40 100644 --- a/terragrunt/modules/crates-io/_terraform.tf +++ b/terragrunt/modules/crates-io/_terraform.tf @@ -102,3 +102,9 @@ variable "fastly_aws_account_id" { description = "The AWS account ID that Fastly uses to write logs" default = "717331877981" } + +variable "cdn_log_event_queue_arn" { + # See the `crates-io-logs` module + description = "ARN of the SQS queue that receives S3 notifications for CDN logs" + type = string +} diff --git a/terragrunt/modules/crates-io/s3-logs.tf b/terragrunt/modules/crates-io/s3-logs.tf index 0a7931d71..dc5260a31 100644 --- a/terragrunt/modules/crates-io/s3-logs.tf +++ b/terragrunt/modules/crates-io/s3-logs.tf @@ -25,3 +25,21 @@ resource "aws_s3_bucket_public_access_block" "logs" { ignore_public_acls = true restrict_public_buckets = true } + +resource "aws_s3_bucket_notification" "cdn_log_event_queue" { + bucket = aws_s3_bucket.logs.id + + queue { + id = "cloudfront" + events = ["s3:ObjectCreated:*"] + queue_arn = var.cdn_log_event_queue_arn + filter_prefix = "cloudfront/" + } + + queue { + id = "fastly" + events = ["s3:ObjectCreated:*"] + queue_arn = var.cdn_log_event_queue_arn + filter_prefix = "fastly-requests/" + } +} From 09d6ae9c047243dc64ec9a094984b6d4fc3f64c3 Mon Sep 17 00:00:00 2001 From: Jan David Date: Tue, 16 Jan 2024 17:30:09 +0100 Subject: [PATCH 3/3] Output the ARN of the SQS queue --- terragrunt/modules/crates-io-logs/_terraform.tf | 10 ---------- terragrunt/modules/crates-io-logs/outputs.tf | 4 ++++ terragrunt/modules/crates-io-logs/variables.tf | 9 +++++++++ 3 files changed, 13 insertions(+), 10 deletions(-) create mode 100644 terragrunt/modules/crates-io-logs/outputs.tf create mode 100644 terragrunt/modules/crates-io-logs/variables.tf diff --git a/terragrunt/modules/crates-io-logs/_terraform.tf b/terragrunt/modules/crates-io-logs/_terraform.tf index 3278468fe..a4f0a52be 100644 --- a/terragrunt/modules/crates-io-logs/_terraform.tf +++ b/terragrunt/modules/crates-io-logs/_terraform.tf @@ -8,13 +8,3 @@ terraform { } } } - -variable "bucket_account" { - type = number - description = "Account ID of the S3 bucket which will send events to the SQS queue" -} - -variable "bucket_arn" { - type = string - description = "ARN of the S3 bucket which will send events to the SQS queue" -} diff --git a/terragrunt/modules/crates-io-logs/outputs.tf b/terragrunt/modules/crates-io-logs/outputs.tf new file mode 100644 index 000000000..67e40ca94 --- /dev/null +++ b/terragrunt/modules/crates-io-logs/outputs.tf @@ -0,0 +1,4 @@ +# ARN of the SQS queue that receives S3 bucket notifications +output "sqs_queue_arn" { + value = aws_sqs_queue.cdn_log_event_queue.arn +} diff --git a/terragrunt/modules/crates-io-logs/variables.tf b/terragrunt/modules/crates-io-logs/variables.tf new file mode 100644 index 000000000..94d931b1f --- /dev/null +++ b/terragrunt/modules/crates-io-logs/variables.tf @@ -0,0 +1,9 @@ +variable "bucket_account" { + type = number + description = "Account ID of the S3 bucket which will send events to the SQS queue" +} + +variable "bucket_arn" { + type = string + description = "ARN of the S3 bucket which will send events to the SQS queue" +}