From 6e232e85d01ca1514442fc877d4f97ca0f8a7255 Mon Sep 17 00:00:00 2001 From: Cris Daniluk Date: Mon, 6 Jul 2020 13:13:36 -0400 Subject: [PATCH] Cross account support (#2) * Update linting/checks * not needed * added crossaccount support * fix badge Co-authored-by: Steven B Co-authored-by: Steven B --- .github/CODEOWNERS | 1 + .github/workflows/check.yml | 20 ------ .github/workflows/misspell.yaml | 23 +++++++ .github/workflows/pre-commit.yaml | 39 +++++++++++ .github/workflows/pullRequest.yaml | 93 +++++++++++++++++++++++++++ .github/workflows/tflint.yaml | 29 +++++++++ .github/workflows/tfsec.yaml | 28 ++++++++ .github/workflows/yamllint.yaml | 22 +++++++ .gitignore | 9 +++ .pre-commit-config.yaml | 82 +++++++++++++++++++---- .terraform-version | 1 - .tflint.hcl | 45 +++++++++++++ .yamllint.yml | 2 + README.md | 39 +++++++++-- bin/install-macos.sh | 18 ++++++ bin/install-ubuntu.sh | 23 +++++++ examples/basic/README.md | 12 ++++ examples/basic/main.tf | 10 +++ main.tf | 100 ++++++++++++++++++++++++++++- outputs.tf | 5 +- variables.tf | 31 ++++++++- versions.tf | 3 + 22 files changed, 591 insertions(+), 44 deletions(-) create mode 100644 .github/CODEOWNERS delete mode 100644 .github/workflows/check.yml create mode 100644 .github/workflows/misspell.yaml create mode 100644 .github/workflows/pre-commit.yaml create mode 100644 .github/workflows/pullRequest.yaml create mode 100644 .github/workflows/tflint.yaml create mode 100644 .github/workflows/tfsec.yaml create mode 100644 .github/workflows/yamllint.yaml create mode 100644 .gitignore delete mode 100644 .terraform-version create mode 100644 .tflint.hcl create mode 100644 .yamllint.yml create mode 100755 bin/install-macos.sh create mode 100755 bin/install-ubuntu.sh create mode 100644 examples/basic/README.md create mode 100644 examples/basic/main.tf create mode 100644 versions.tf diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS new file mode 100644 index 0000000..f67eda9 --- /dev/null +++ b/.github/CODEOWNERS @@ -0,0 +1 @@ +* @rhythmictech/engineering diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml deleted file mode 100644 index b50cee9..0000000 --- a/.github/workflows/check.yml +++ /dev/null @@ -1,20 +0,0 @@ -name: check -on: [push, pull_request] - -jobs: - build: - runs-on: macOS-latest - steps: - - uses: actions/checkout@v1 - - - name: Install prereq - run: | - brew install docker tfenv tflint - tfenv install - - - name: tf fmt - run: | - terraform fmt - - name: tflint - run: | - tflint diff --git a/.github/workflows/misspell.yaml b/.github/workflows/misspell.yaml new file mode 100644 index 0000000..6574e3d --- /dev/null +++ b/.github/workflows/misspell.yaml @@ -0,0 +1,23 @@ +--- +name: misspell +on: + push: + branches: + - main + - master + - prod + - develop + +jobs: + misspell: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - name: misspell + uses: reviewdog/action-misspell@v1 + with: + github_token: ${{ secrets.GITHUB_TOKEN }} + locale: "US" + reporter: github-check + filter_mode: nofilter + level: error diff --git a/.github/workflows/pre-commit.yaml b/.github/workflows/pre-commit.yaml new file mode 100644 index 0000000..03de2a3 --- /dev/null +++ b/.github/workflows/pre-commit.yaml @@ -0,0 +1,39 @@ +--- +name: pre-commit-check +on: + push: + branches: -- main + - master + - prod + - develop + +jobs: + pre-commit-check: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - name: Set up Python + uses: actions/setup-python@v2 + - name: Install prerequisites + run: ./bin/install-ubuntu.sh + - name: initialize Terraform + run: terraform init --backend=false + - name: pre-commit + uses: pre-commit/action@v2.0.0 + env: + AWS_DEFAULT_REGION: us-east-1 + # many of these are covered by better reviewdog linters below + SKIP: >- + terraform_tflint_deep, + no-commit-to-branch, + terraform_tflint_nocreds, + terraform_tfsec + - uses: stefanzweifel/git-auto-commit-action@v4 + if: ${{ failure() }} + with: + commit_message: Apply automatic changes + commit_options: "--no-verify" + # Optional commit user and author settings + commit_user_name: Linter Bot + commit_user_email: noreply@rhythmictech.com + commit_author: Linter Bot diff --git a/.github/workflows/pullRequest.yaml b/.github/workflows/pullRequest.yaml new file mode 100644 index 0000000..5f5186c --- /dev/null +++ b/.github/workflows/pullRequest.yaml @@ -0,0 +1,93 @@ +--- +name: pull request +on: + pull_request: + +jobs: + pre-commit: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - name: Set up Python + uses: actions/setup-python@v2 + - name: Install prerequisites + run: ./bin/install-ubuntu.sh + - name: initialize Terraform + run: terraform init --backend=false + - name: pre-commit + uses: pre-commit/action@v2.0.0 + env: + AWS_DEFAULT_REGION: us-east-1 + # many of these are covered by better reviewdog linters below + SKIP: >- + terraform_tflint_deep, + no-commit-to-branch, + terraform_tflint_nocreds, + terraform_tfsec + - uses: stefanzweifel/git-auto-commit-action@v4 + if: ${{ failure() }} + with: + commit_message: Apply automatic changes + commit_options: "--no-verify" + # Optional commit user and author settings + commit_user_name: Linter Bot + commit_user_email: noreply@rhythmictech.com + commit_author: Linter Bot + tflint: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - name: setup Terraform + uses: hashicorp/setup-terraform@v1 + with: + terraform_version: 0.12.26 + - name: Terraform init + run: terraform init --backend=false + - name: tflint + uses: reviewdog/action-tflint@master + with: + github_token: ${{ secrets.GITHUB_TOKEN }} + reporter: github-pr-check + filter_mode: added + flags: --module + level: error + tfsec: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - name: setup Terraform + uses: hashicorp/setup-terraform@v1 + with: + terraform_version: 0.12.26 + - name: Terraform init + run: terraform init --backend=false + - name: tfsec + uses: reviewdog/action-tfsec@master + with: + github_token: ${{ secrets.GITHUB_TOKEN }} + reporter: github-pr-check + filter_mode: added + level: warning + misspell: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - name: misspell + uses: reviewdog/action-misspell@v1 + with: + github_token: ${{ secrets.GITHUB_TOKEN }} + locale: "US" + reporter: github-pr-check + filter_mode: added + level: error + yamllint: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - name: yamllint + uses: reviewdog/action-yamllint@v1 + with: + github_token: ${{ secrets.GITHUB_TOKEN }} + reporter: github-pr-check + filter_mode: added + level: error diff --git a/.github/workflows/tflint.yaml b/.github/workflows/tflint.yaml new file mode 100644 index 0000000..ce54833 --- /dev/null +++ b/.github/workflows/tflint.yaml @@ -0,0 +1,29 @@ +--- +name: tflint +on: + push: + branches: + - main + - master + - prod + - develop + +jobs: + tflint: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - name: setup Terraform + uses: hashicorp/setup-terraform@v1 + with: + terraform_version: 0.12.26 + - name: Terraform init + run: terraform init --backend=false + - name: tflint + uses: reviewdog/action-tflint@master + with: + github_token: ${{ secrets.GITHUB_TOKEN }} + reporter: github-check + filter_mode: nofilter + flags: --module + level: error diff --git a/.github/workflows/tfsec.yaml b/.github/workflows/tfsec.yaml new file mode 100644 index 0000000..9067568 --- /dev/null +++ b/.github/workflows/tfsec.yaml @@ -0,0 +1,28 @@ +--- +name: tfsec +on: + push: + branches: + - main + - master + - prod + - develop + +jobs: + tfsec: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - name: setup Terraform + uses: hashicorp/setup-terraform@v1 + with: + terraform_version: 0.12.26 + - name: Terraform init + run: terraform init --backend=false + - name: tfsec + uses: reviewdog/action-tfsec@master + with: + github_token: ${{ secrets.GITHUB_TOKEN }} + reporter: github-check + filter_mode: nofilter + level: error diff --git a/.github/workflows/yamllint.yaml b/.github/workflows/yamllint.yaml new file mode 100644 index 0000000..0b6f93a --- /dev/null +++ b/.github/workflows/yamllint.yaml @@ -0,0 +1,22 @@ +--- +name: yamllint +on: + push: + branches: + - main + - master + - prod + - develop + +jobs: + yamllint: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - name: yamllint + uses: reviewdog/action-yamllint@v1 + with: + github_token: ${{ secrets.GITHUB_TOKEN }} + reporter: github-check + filter_mode: nofilter + level: error diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..1fef4ab --- /dev/null +++ b/.gitignore @@ -0,0 +1,9 @@ +# Local .terraform directories +**/.terraform/* + +# .tfstate files +*.tfstate +*.tfstate.* + +# .tfvars files +*.tfvars diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 79e0dbb..f13d6ce 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,13 +1,71 @@ ---- repos: -- repo: git://github.com/antonbabenko/pre-commit-terraform - rev: v1.24.0 - hooks: - - id: terraform_fmt - - id: terraform_docs -- repo: https://github.com/pre-commit/pre-commit-hooks - rev: v2.4.0 - hooks: - - id: end-of-file-fixer - - id: trailing-whitespace - - id: no-commit-to-branch + - repo: https://github.com/antonbabenko/pre-commit-terraform + rev: v1.31.0 + hooks: + - id: terraform_docs + always_run: true + args: + - --args=--sort-by-required + - id: terraform_fmt + - id: terraform_tflint + alias: terraform_tflint_deep + name: terraform_tflint_deep + args: + - --args=--deep + - id: terraform_tflint + alias: terraform_tflint_nocreds + name: terraform_tflint_nocreds + - id: terraform_tfsec + - repo: local + hooks: + - id: terraform_validate + name: terraform_validate + entry: | + bash -c ' + AWS_DEFAULT_REGION=us-east-1 + declare -a DIRS + for FILE in "$@" + do + DIRS+=($(dirname "$FILE")) + done + for DIR in $(printf "%s\n" "${DIRS[@]}" | sort -u) + do + cd $(dirname "$FILE") + terraform init --backend=false + terraform validate . + done + ' + language: system + verbose: true + files: \.tf(vars)?$ + exclude: examples + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v3.0.0 + hooks: + - id: check-case-conflict + - id: check-json + - id: check-merge-conflict + - id: check-symlinks + - id: check-yaml + args: + - --unsafe + - id: end-of-file-fixer + - id: mixed-line-ending + args: + - --fix=lf + - id: no-commit-to-branch + args: + - --branch + - main + - --branch + - master + - --branch + - prod + - id: pretty-format-json + args: + - --autofix + - --top-keys=name,Name + - id: trailing-whitespace + args: + - --markdown-linebreak-ext=md + exclude: README.md diff --git a/.terraform-version b/.terraform-version deleted file mode 100644 index 43b2a52..0000000 --- a/.terraform-version +++ /dev/null @@ -1 +0,0 @@ -0.12.21 \ No newline at end of file diff --git a/.tflint.hcl b/.tflint.hcl new file mode 100644 index 0000000..5cba22a --- /dev/null +++ b/.tflint.hcl @@ -0,0 +1,45 @@ +config { + module = true + deep_check = false +} + +rule "terraform_deprecated_interpolation" { + enabled = true +} + +rule "terraform_unused_declarations" { + enabled = true +} + +rule "terraform_comment_syntax" { + enabled = true +} + +rule "terraform_documented_outputs" { + enabled = true +} + +rule "terraform_documented_variables" { + enabled = true +} + +rule "terraform_typed_variables" { + enabled = true +} + +rule "terraform_module_pinned_source" { + enabled = true +} + +rule "terraform_naming_convention" { + enabled = true + format = "snake_case" +} + +rule "terraform_required_version" { + enabled = true +} + +rule "terraform_required_providers" { + enabled = true +} diff --git a/.yamllint.yml b/.yamllint.yml new file mode 100644 index 0000000..e1a518a --- /dev/null +++ b/.yamllint.yml @@ -0,0 +1,2 @@ +truthy: + check-keys: false diff --git a/README.md b/README.md index 0c61afe..f384d1b 100644 --- a/README.md +++ b/README.md @@ -1,19 +1,48 @@ # terraform-aws-helmrepo -Create an S3 bucket intended to serve as a Helm repo. Configures basic encryption. +[![tflint](https://github.com/rhythmictech/terraform-aws-helmrepo/workflows/tflint/badge.svg?branch=main&event=push)](https://github.com/rhythmictech/terraform-aws-helmrepo/actions?query=workflow%3Atflint+event%3Apush+branch%3Amain) +[![tfsec](https://github.com/rhythmictech/terraform-aws-helmrepo/workflows/tfsec/badge.svg?branch=main&event=push)](https://github.com/rhythmictech/terraform-aws-helmrepo/actions?query=workflow%3Atfsec+event%3Apush+branch%3Amain) +[![yamllint](https://github.com/rhythmictech/terraform-aws-helmrepo/workflows/yamllint/badge.svg?branch=main&event=push)](https://github.com/rhythmictech/terraform-aws-helmrepo/actions?query=workflow%3Ayamllint+event%3Apush+branch%3Amain) +[![misspell](https://github.com/rhythmictech/terraform-aws-helmrepo/workflows/misspell/badge.svg?branch=main&event=push)](https://github.com/rhythmictech/terraform-aws-helmrepo/actions?query=workflow%3Amisspell+event%3Apush+branch%3Amain) +[![pre-commit-check](https://github.com/rhythmictech/terraform-aws-helmrepo/workflows/pre-commit-check/badge.svg?branch=main&event=push)](https://github.com/rhythmictech/terraform-aws-helmrepo/actions?query=workflow%3Apre-commit-check+event%3Apush+branch%3Amain) +follow on Twitter +Create an S3 bucket intended to serve as a Helm repo. Configures basic encryption and supports sharing the bucket across many accounts. + +## Usage +``` +module { + source = "rhythmictech/helmrepo/aws" +} +``` +## Requirements + +| Name | Version | +|------|---------| +| terraform | >= 0.12.19 | + +## Providers + +| Name | Version | +|------|---------| +| aws | n/a | + ## Inputs | Name | Description | Type | Default | Required | -|------|-------------|:----:|:-----:|:-----:| -| name | Prefix name for the helm repo \(e.g., EKS cluster name\) | string | `"default"` | no | -| tags | Tags to add to supported resources | map(string) | `{}` | no | +|------|-------------|------|---------|:--------:| +| allowed\_account\_ids | List of AWS account IDs to grant read-only access to the repo. Due to how policies are constructed, there's effectively a limit of about 9 accounts. | `list(string)` | `[]` | no | +| logging\_bucket | S3 bucket name to log bucket access requests to (optional) | `string` | `null` | no | +| logging\_bucket\_prefix | S3 bucket prefix to log bucket access requests to (optional). If blank but a `logging_bucket` is specified, this will be set to the name of the bucket | `string` | `null` | no | +| name | Bucket name for the helm repo. Specify to control the exact name of the bucket, otherwise use `name_suffix` | `string` | `null` | no | +| name\_suffix | Bucket suffix for the repo (bucket will be named `[ACCOUNT_ID]-[REGION]-[name_suffix]`, not used if `name` is specified) | `string` | `"helmrepo"` | no | +| tags | Tags to add to supported resources | `map(string)` | `{}` | no | ## Outputs | Name | Description | |------|-------------| -| s3\_bucket\_repo | | +| s3\_bucket | Bucket name of the repo | diff --git a/bin/install-macos.sh b/bin/install-macos.sh new file mode 100755 index 0000000..4bc710b --- /dev/null +++ b/bin/install-macos.sh @@ -0,0 +1,18 @@ +#!/bin/bash + +echo 'installing brew packages' +brew update +brew tap liamg/tfsec +brew install tfenv tflint terraform-docs pre-commit liamg/tfsec/tfsec coreutils +brew upgrade tfenv tflint terraform-docs pre-commit liamg/tfsec/tfsec coreutils + +echo 'installing pre-commit hooks' +pre-commit install + +echo 'setting pre-commit hooks to auto-install on clone in the future' +git config --global init.templateDir ~/.git-template +pre-commit init-templatedir ~/.git-template + +echo 'installing terraform with tfenv' +tfenv install min-required +tfenv use min-required diff --git a/bin/install-ubuntu.sh b/bin/install-ubuntu.sh new file mode 100755 index 0000000..355de0b --- /dev/null +++ b/bin/install-ubuntu.sh @@ -0,0 +1,23 @@ +#!/bin/bash + +echo 'installing dependencies' +sudo apt install python3-pip gawk &&\ +pip3 install pre-commit +curl -L "$(curl -s https://api.github.com/repos/segmentio/terraform-docs/releases/latest | grep -o -E "https://.+?-linux-amd64")" > terraform-docs && chmod +x terraform-docs && sudo mv terraform-docs /usr/bin/ +curl -L "$(curl -s https://api.github.com/repos/terraform-linters/tflint/releases/latest | grep -o -E "https://.+?_linux_amd64.zip")" > tflint.zip && unzip tflint.zip && rm tflint.zip && sudo mv tflint /usr/bin/ +env GO111MODULE=on go get -u github.com/liamg/tfsec/cmd/tfsec +git clone https://github.com/tfutils/tfenv.git ~/.tfenv || true +mkdir -p ~/.local/bin/ +. ~/.profile +ln -s ~/.tfenv/bin/* ~/.local/bin + +echo 'installing pre-commit hooks' +pre-commit install + +echo 'setting pre-commit hooks to auto-install on clone in the future' +git config --global init.templateDir ~/.git-template +pre-commit init-templatedir ~/.git-template + +echo 'installing terraform with tfenv' +tfenv install min-required +tfenv use min-required diff --git a/examples/basic/README.md b/examples/basic/README.md new file mode 100644 index 0000000..c1d4ab9 --- /dev/null +++ b/examples/basic/README.md @@ -0,0 +1,12 @@ +# basic example + +## Code + +## Applying +``` +> terraform apply + +Apply complete! Resources: 4 added, 0 changed, 0 destroyed. + +Outputs: +``` diff --git a/examples/basic/main.tf b/examples/basic/main.tf new file mode 100644 index 0000000..13ef2c1 --- /dev/null +++ b/examples/basic/main.tf @@ -0,0 +1,10 @@ + +module "example" { + source = "../.." + + name = "test" +} + +output "example" { + value = module.example +} diff --git a/main.tf b/main.tf index a81742a..c5babfc 100644 --- a/main.tf +++ b/main.tf @@ -4,11 +4,32 @@ data "aws_caller_identity" "current" { data "aws_region" "region" { } +locals { + bucket_name = var.name != null ? var.name : "${data.aws_caller_identity.current.account_id}-${data.aws_region.region.name}-${var.name_suffix}" + + logging_map = var.logging_bucket == null ? [] : [{ + bucket = var.logging_bucket + prefix = var.logging_bucket_prefix != null ? var.logging_bucket_prefix : local.bucket_name + }] +} + +# This bucket uses a dynamic block to generate logging. If users do not wish to log, +# that's on them, but the module supports it. +#tfsec:ignore:AWS002 resource "aws_s3_bucket" "this" { - bucket = "${data.aws_caller_identity.current.account_id}-${data.aws_region.region.name}-${var.name}-helm-repo" + bucket = local.bucket_name acl = "private" tags = var.tags + dynamic "logging" { + for_each = local.logging_map + + content { + target_bucket = logging.value.bucket + target_prefix = logging.value.prefix + } + } + server_side_encryption_configuration { rule { apply_server_side_encryption_by_default { @@ -20,7 +41,6 @@ resource "aws_s3_bucket" "this" { versioning { enabled = true } - } resource "aws_s3_bucket_public_access_block" "this" { @@ -36,3 +56,79 @@ resource "aws_s3_bucket_public_access_block" "this" { depends_on = [aws_s3_bucket.this] } + +data "aws_iam_policy_document" "this" { + statement { + sid = "DenyIncorrectEncryptionHeader" + actions = ["s3:PutObject*"] + effect = "Deny" + resources = ["${aws_s3_bucket.this.arn}/*"] + + condition { + test = "StringNotEquals" + variable = "s3:x-amz-server-side-encryption" + values = ["AES256"] + } + + principals { + identifiers = ["*"] + type = "AWS" + } + } + + statement { + sid = "DenyUnEncryptedObjectUploads" + actions = ["s3:PutObject*"] + effect = "Deny" + resources = ["${aws_s3_bucket.this.arn}/*"] + + condition { + test = "Null" + variable = "s3:x-amz-server-side-encryption" + values = [true] + } + + principals { + identifiers = ["*"] + type = "AWS" + } + } + + dynamic "statement" { + for_each = var.allowed_account_ids + + content { + + sid = "Allow cross-account access to list objects (${statement.value})" + actions = ["s3:ListBucket"] + effect = "Allow" + resources = [aws_s3_bucket.this.arn] + + principals { + identifiers = ["arn:aws:iam::${statement.value}:root"] + type = "AWS" + } + } + } + + dynamic "statement" { + for_each = var.allowed_account_ids + + content { + sid = "Allow Cross-account read-only access (${statement.value})" + actions = ["s3:GetObject*"] + effect = "Allow" + resources = ["${aws_s3_bucket.this.arn}/*"] + + principals { + identifiers = ["arn:aws:iam::${statement.value}:root"] + type = "AWS" + } + } + } +} + +resource "aws_s3_bucket_policy" "this" { + bucket = aws_s3_bucket.this.id + policy = data.aws_iam_policy_document.this.json +} diff --git a/outputs.tf b/outputs.tf index 12d2997..144db29 100644 --- a/outputs.tf +++ b/outputs.tf @@ -1,3 +1,4 @@ -output "s3_bucket_repo" { - value = aws_s3_bucket.this.bucket +output "s3_bucket" { + description = "Bucket name of the repo" + value = aws_s3_bucket.this.bucket } diff --git a/variables.tf b/variables.tf index 2530c35..29d3dcd 100644 --- a/variables.tf +++ b/variables.tf @@ -1,6 +1,33 @@ +######################################## +# General Vars +######################################## +variable "allowed_account_ids" { + default = [] + description = "List of AWS account IDs to grant read-only access to the repo. Due to how policies are constructed, there's effectively a limit of about 9 accounts." + type = list(string) +} + +variable "logging_bucket" { + default = null + description = "S3 bucket name to log bucket access requests to (optional)" + type = string +} + +variable "logging_bucket_prefix" { + default = null + description = "S3 bucket prefix to log bucket access requests to (optional). If blank but a `logging_bucket` is specified, this will be set to the name of the bucket" + type = string +} + variable "name" { - default = "default" - description = "Prefix name for the helm repo (e.g., EKS cluster name)" + default = null + description = "Bucket name for the helm repo. Specify to control the exact name of the bucket, otherwise use `name_suffix`" + type = string +} + +variable "name_suffix" { + default = "helmrepo" + description = "Bucket suffix for the repo (bucket will be named `[ACCOUNT_ID]-[REGION]-[name_suffix]`, not used if `name` is specified)" type = string } diff --git a/versions.tf b/versions.tf new file mode 100644 index 0000000..749d1a2 --- /dev/null +++ b/versions.tf @@ -0,0 +1,3 @@ +terraform { + required_version = ">= 0.12.19" +}