diff --git a/.github/workflows/clean_ami.yml b/.github/workflows/clean_ami.yml index b7ce912ebc..d86b39c0f1 100644 --- a/.github/workflows/clean_ami.yml +++ b/.github/workflows/clean_ami.yml @@ -11,6 +11,9 @@ on: jobs: clean-ami: runs-on: ubuntu-latest + permissions: + id-token: write + contents: read steps: - uses: actions/checkout@v2 - uses: actions/setup-go@v3 @@ -18,8 +21,7 @@ jobs: - name: Configure AWS Credentials uses: aws-actions/configure-aws-credentials@v1 with: - aws-access-key-id: ${{ secrets.TERRAFORM_AWS_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.TERRAFORM_AWS_SECRET_ACCESS_KEY }} + role-to-assume: ${{ secrets.TERRAFORM_AWS_ASSUME_ROLE }} aws-region: us-west-2 - name: Clean old ami diff --git a/.github/workflows/clean_dedicated_host.yml b/.github/workflows/clean_dedicated_host.yml index f333aac2de..5de607cd26 100644 --- a/.github/workflows/clean_dedicated_host.yml +++ b/.github/workflows/clean_dedicated_host.yml @@ -11,6 +11,9 @@ on: jobs: clean-dedicated-hosts: runs-on: ubuntu-latest + permissions: + id-token: write + contents: read steps: - uses: actions/checkout@v2 - uses: actions/setup-go@v3 @@ -18,8 +21,7 @@ jobs: - name: Configure AWS Credentials uses: aws-actions/configure-aws-credentials@v1 with: - aws-access-key-id: ${{ secrets.TERRAFORM_AWS_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.TERRAFORM_AWS_SECRET_ACCESS_KEY }} + role-to-assume: ${{ secrets.TERRAFORM_AWS_ASSUME_ROLE }} aws-region: us-west-2 - name: Clean old dedicated host diff --git a/.github/workflows/integrationTest.yml b/.github/workflows/integrationTest.yml index 0bf55752fc..4971dc0ace 100644 --- a/.github/workflows/integrationTest.yml +++ b/.github/workflows/integrationTest.yml @@ -4,8 +4,7 @@ name: Run Integration Tests env: PRIVATE_KEY: ${{ secrets.AWS_PRIVATE_KEY }} - TERRAFORM_AWS_ACCESS_KEY_ID: ${{ secrets.TERRAFORM_AWS_ACCESS_KEY_ID }} - TERRAFORM_AWS_SECRET_ACCESS_KEY: ${{ secrets.TERRAFORM_AWS_SECRET_ACCESS_KEY }} + TERRAFORM_AWS_ASSUME_ROLE: ${{ secrets.TERRAFORM_AWS_ASSUME_ROLE }} S3_INTEGRATION_BUCKET: ${{ secrets.S3_INTEGRATION_BUCKET }} KEY_NAME: ${{ secrets.KEY_NAME }} VPC_SECURITY_GROUPS_IDS: ${{ secrets.VPC_SECURITY_GROUPS_IDS }} @@ -21,7 +20,6 @@ on: branches: - master - workflow_dispatch: concurrency: @@ -32,6 +30,9 @@ jobs: MakeDockerImage: name: 'MakeDockerImage' runs-on: ubuntu-latest + permissions: + id-token: write + contents: read steps: - uses: actions/checkout@v3 @@ -47,8 +48,7 @@ jobs: if: steps.build-docker-image.outputs.cache-hit != 'true' uses: aws-actions/configure-aws-credentials@v1 with: - aws-access-key-id: ${{ secrets.TERRAFORM_AWS_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.TERRAFORM_AWS_SECRET_ACCESS_KEY }} + role-to-assume: ${{ env.TERRAFORM_AWS_ASSUME_ROLE }} aws-region: us-west-2 - name: Login ECR @@ -82,6 +82,9 @@ jobs: MakeBinary: name: 'MakeBinary' runs-on: ubuntu-latest + permissions: + id-token: write + contents: read steps: - uses: actions/checkout@v3 with: @@ -100,8 +103,7 @@ jobs: - name: Configure AWS Credentials uses: aws-actions/configure-aws-credentials@v1 with: - aws-access-key-id: ${{ secrets.TERRAFORM_AWS_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.TERRAFORM_AWS_SECRET_ACCESS_KEY }} + role-to-assume: ${{ env.TERRAFORM_AWS_ASSUME_ROLE }} aws-region: us-west-2 - name: Cache go @@ -172,6 +174,9 @@ jobs: name: 'MakeMSIZip' runs-on: ubuntu-latest needs: [MakeBinary] + permissions: + id-token: write + contents: read steps: - uses: actions/checkout@v2 @@ -183,8 +188,7 @@ jobs: - name: Configure AWS Credentials uses: aws-actions/configure-aws-credentials@v1 with: - aws-access-key-id: ${{ secrets.TERRAFORM_AWS_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.TERRAFORM_AWS_SECRET_ACCESS_KEY }} + role-to-assume: ${{ env.TERRAFORM_AWS_ASSUME_ROLE }} aws-region: us-west-2 - name: Cache win zip @@ -199,10 +203,11 @@ jobs: run: | aws s3 cp s3://${S3_INTEGRATION_BUCKET}/integration-test/binary/${{ github.sha }} . --recursive - - uses: montudor/action-zip@v1 + - name: Unzip if: steps.cached_win_zip.outputs.cache-hit != 'true' - with: - args: unzip -qq windows/amd64/amazon-cloudwatch-agent.zip -d windows-agent + run: | + sudo apt install unzip + unzip windows/amd64/amazon-cloudwatch-agent.zip -d windows-agent - name: Create msi dep folder and copy deps if: steps.cached_win_zip.outputs.cache-hit != 'true' @@ -215,11 +220,11 @@ jobs: go run integration/msi/tools/msiversion/msiversionconverter.go $version msi_dep/amazon-cloudwatch-agent.wxs '' --tags=integration go run integration/msi/tools/msiversion/msiversionconverter.go $version msi_dep/manifest.json __VERSION__ --tags=integration - - uses: papeloto/action-zip@v1 + - name: Zip if: steps.cached_win_zip.outputs.cache-hit != 'true' - with: - files: msi_dep/ - dest: buildMSI.zip + run: | + sudo apt install zip + zip buildMSI.zip msi_dep/* - name: Upload zip if: steps.cached_win_zip.outputs.cache-hit != 'true' @@ -229,6 +234,9 @@ jobs: name: 'MakeMacPkg' runs-on: macos-latest needs: [MakeBinary] + permissions: + id-token: write + contents: read steps: - uses: actions/checkout@v2 @@ -240,8 +248,7 @@ jobs: - name: Configure AWS Credentials uses: aws-actions/configure-aws-credentials@v1 with: - aws-access-key-id: ${{ secrets.TERRAFORM_AWS_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.TERRAFORM_AWS_SECRET_ACCESS_KEY }} + role-to-assume: ${{ env.TERRAFORM_AWS_ASSUME_ROLE }} aws-region: us-west-2 - name: Cache pkg @@ -275,14 +282,16 @@ jobs: name: 'BuildMSI' runs-on: windows-latest needs: [MakeMSIZip] + permissions: + id-token: write + contents: read steps: - uses: actions/checkout@v2 - name: Configure AWS Credentials uses: aws-actions/configure-aws-credentials@v1 with: - aws-access-key-id: ${{ secrets.TERRAFORM_AWS_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.TERRAFORM_AWS_SECRET_ACCESS_KEY }} + role-to-assume: ${{ env.TERRAFORM_AWS_ASSUME_ROLE }} aws-region: us-west-2 - name: Cache msi @@ -305,7 +314,7 @@ jobs: $wixToolsetBinPath = ";C:\Program Files (x86)\WiX Toolset v3.11\bin;" $env:PATH = $env:PATH + $wixToolsetBinPath Expand-Archive buildMSI.zip -Force - cd buildMSI + cd buildMSI/msi_dep .\create_msi.ps1 ${{ github.sha }} ${{ secrets.S3_INTEGRATION_BUCKET }} #GH actions set up gpg only works on ubuntu as of this commit date @@ -313,6 +322,9 @@ jobs: name: 'SignMacAndWindowsPackage' runs-on: ubuntu-latest needs: [BuildMSI, MakeMacPkg] + permissions: + id-token: write + contents: read steps: - uses: actions/checkout@v2 - uses: olafurpg/setup-gpg@v3 @@ -320,8 +332,7 @@ jobs: - name: Configure AWS Credentials uses: aws-actions/configure-aws-credentials@v1 with: - aws-access-key-id: ${{ secrets.TERRAFORM_AWS_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.TERRAFORM_AWS_SECRET_ACCESS_KEY }} + role-to-assume: ${{ env.TERRAFORM_AWS_ASSUME_ROLE }} aws-region: us-west-2 - name: Cache sig @@ -358,14 +369,16 @@ jobs: working-directory: integration/terraform/ec2/localstack outputs: local_stack_host_name: ${{ steps.localstack.outputs.local_stack_host_name }} + permissions: + id-token: write + contents: read steps: - uses: actions/checkout@v2 - name: Configure AWS Credentials uses: aws-actions/configure-aws-credentials@v1 with: - aws-access-key-id: ${{ secrets.TERRAFORM_AWS_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.TERRAFORM_AWS_SECRET_ACCESS_KEY }} + role-to-assume: ${{ env.TERRAFORM_AWS_ASSUME_ROLE }} aws-region: us-west-2 - name: Verify Terraform version @@ -397,14 +410,16 @@ jobs: fail-fast: false matrix: arrays: ${{ fromJson(needs.GenerateTestMatrix.outputs.ec2_linux_matrix) }} + permissions: + id-token: write + contents: read steps: - uses: actions/checkout@v2 - name: Configure AWS Credentials uses: aws-actions/configure-aws-credentials@v1 with: - aws-access-key-id: ${{ secrets.TERRAFORM_AWS_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.TERRAFORM_AWS_SECRET_ACCESS_KEY }} + role-to-assume: ${{ env.TERRAFORM_AWS_ASSUME_ROLE }} aws-region: us-west-2 - name: Cache if success @@ -467,14 +482,16 @@ jobs: fail-fast: false matrix: arrays: ${{ fromJson(needs.GenerateTestMatrix.outputs.ec2_windows_matrix) }} + permissions: + id-token: write + contents: read steps: - uses: actions/checkout@v2 - name: Configure AWS Credentials uses: aws-actions/configure-aws-credentials@v1 with: - aws-access-key-id: ${{ secrets.TERRAFORM_AWS_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.TERRAFORM_AWS_SECRET_ACCESS_KEY }} + role-to-assume: ${{ env.TERRAFORM_AWS_ASSUME_ROLE }} aws-region: us-west-2 - name: Cache if success @@ -532,14 +549,16 @@ jobs: defaults: run: working-directory: integration/terraform/ec2/localstack + permissions: + id-token: write + contents: read steps: - uses: actions/checkout@v2 - name: Configure AWS Credentials uses: aws-actions/configure-aws-credentials@v1 with: - aws-access-key-id: ${{ secrets.TERRAFORM_AWS_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.TERRAFORM_AWS_SECRET_ACCESS_KEY }} + role-to-assume: ${{ env.TERRAFORM_AWS_ASSUME_ROLE }} aws-region: us-west-2 - name: Copy state @@ -562,14 +581,16 @@ jobs: fail-fast: false matrix: arrays: ${{ fromJson(needs.GenerateTestMatrix.outputs.ecs_fargate_matrix) }} + permissions: + id-token: write + contents: read steps: - uses: actions/checkout@v2 - name: Configure AWS Credentials uses: aws-actions/configure-aws-credentials@v1 with: - aws-access-key-id: ${{ secrets.TERRAFORM_AWS_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.TERRAFORM_AWS_SECRET_ACCESS_KEY }} + role-to-assume: ${{ env.TERRAFORM_AWS_ASSUME_ROLE }} aws-region: us-west-2 - name: Cache if success @@ -621,14 +642,16 @@ jobs: fail-fast: false matrix: arrays: ${{ fromJson(needs.GenerateTestMatrix.outputs.ec2_performance_matrix) }} + permissions: + id-token: write + contents: read steps: - uses: actions/checkout@v2 - name: Configure AWS Credentials uses: aws-actions/configure-aws-credentials@v1 with: - aws-access-key-id: ${{ secrets.TERRAFORM_AWS_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.TERRAFORM_AWS_SECRET_ACCESS_KEY }} + role-to-assume: ${{ env.TERRAFORM_AWS_ASSUME_ROLE }} aws-region: us-west-2 - name: Cache if success diff --git a/.github/workflows/internal-pipeline-dedicated-host-cleaner.yml b/.github/workflows/internal-pipeline-dedicated-host-cleaner.yml index b7d461174c..147da85fd2 100644 --- a/.github/workflows/internal-pipeline-dedicated-host-cleaner.yml +++ b/.github/workflows/internal-pipeline-dedicated-host-cleaner.yml @@ -12,6 +12,9 @@ on: jobs: clean-dedicated-hosts: runs-on: ubuntu-latest + permissions: + id-token: write + contents: read steps: - uses: actions/checkout@v2 - uses: actions/setup-go@v3 @@ -19,8 +22,7 @@ jobs: - name: Configure AWS Credentials uses: aws-actions/configure-aws-credentials@v1 with: - aws-access-key-id: ${{ secrets.INTERNAL_AWS_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.INTERNAL_AWS_SECRET_ACCESS_KEY }} + role-to-assume: ${{ secrets.INTERNAL_AWS_ASSUME_ROLE }} aws-region: us-west-2 - name: Clean old dedicated host diff --git a/.github/workflows/nightly-build.yml b/.github/workflows/nightly-build.yml index 91b2672aea..52e0466f15 100644 --- a/.github/workflows/nightly-build.yml +++ b/.github/workflows/nightly-build.yml @@ -10,6 +10,9 @@ jobs: build: name: Upload Nightly Binaries runs-on: ubuntu-latest + permissions: + id-token: write + contents: read steps: - name: Set up Go 1.x @@ -18,6 +21,12 @@ jobs: go-version: ~1.18.3 id: go + - name: Configure AWS Credentials + uses: aws-actions/configure-aws-credentials@v1 + with: + role-to-assume: ${{ secrets.S3_AWS_ASSUME_ROLE }} + aws-region: us-east-1 + #Need to install rpm so ubuntu can make rpm by default ubuntu can make deb - name: Install rpm run: sudo apt install rpm @@ -31,14 +40,5 @@ jobs: - name: Release run: make nightly-release - - name: Upload binaries to latest - uses: jakejarvis/s3-sync-action@master - with: - args: --acl public-read - env: - AWS_ACCESS_KEY_ID: ${{ secrets.S3_ACCESS_KEY }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.S3_SECRET_ACCESS_KEY}} - AWS_REGION: 'us-east-1' - AWS_S3_BUCKET: 'amazoncloudwatch-agent' - SOURCE_DIR: 'build/bin' - DEST_DIR: 'nightly-build/latest' + - name: Upload to S3 + run: aws s3 cp build/bin s3://amazoncloudwatch-agent/nightly-build/latest/ --recursive --acl public-read --source-region us-east-1 diff --git a/.github/workflows/releaseTest.yml b/.github/workflows/releaseTest.yml index ad67519e81..ae26d27339 100644 --- a/.github/workflows/releaseTest.yml +++ b/.github/workflows/releaseTest.yml @@ -4,8 +4,7 @@ name: Release Update env: PRIVATE_KEY: ${{ secrets.AWS_PRIVATE_KEY }} - TERRAFORM_AWS_ACCESS_KEY_ID: ${{ secrets.TERRAFORM_AWS_ACCESS_KEY_ID }} - TERRAFORM_AWS_SECRET_ACCESS_KEY: ${{ secrets.TERRAFORM_AWS_SECRET_ACCESS_KEY }} + TERRAFORM_AWS_ASSUME_ROLE: ${{ secrets.TERRAFORM_AWS_ASSUME_ROLE }} S3_INTEGRATION_BUCKET: ${{ secrets.S3_INTEGRATION_BUCKET }} KEY_NAME: ${{ secrets.KEY_NAME }} VPC_SECURITY_GROUPS_IDS: ${{ secrets.VPC_SECURITY_GROUPS_IDS }} @@ -29,6 +28,9 @@ jobs: UpdatePerformanceMetrics: name: "UpdatePerformanceMetrics" runs-on: ubuntu-latest + permissions: + id-token: write + contents: read steps: - uses: actions/checkout@v2 @@ -39,8 +41,7 @@ jobs: - name: Configure AWS Credentials uses: aws-actions/configure-aws-credentials@v1 with: - aws-access-key-id: ${{ secrets.TERRAFORM_AWS_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.TERRAFORM_AWS_SECRET_ACCESS_KEY }} + role-to-assume: ${{ secrets.TERRAFORM_AWS_ASSUME_ROLE }} aws-region: us-west-2 - name: Get Release Tag run: echo ${{ github.event.release.tag_name }} diff --git a/integration/terraform/ec2/README.md b/integration/terraform/ec2/README.md index abca08c951..d19bdf4f7d 100644 --- a/integration/terraform/ec2/README.md +++ b/integration/terraform/ec2/README.md @@ -10,13 +10,65 @@ Running integration tests This all assumes that you are creating resources in the `us-west-2` region, as that is currently the only region that supports the integration test AMIs. -#### Terraform IAM user permissions +#### Terraform IAM assume role permission For ease of use, here's a generated IAM policy based on resource usage that you can attach to your IAM user that Terraform will assume, with the required permissions. See docs on [Access Analyzer](https://docs.aws.amazon.com/IAM/latest/UserGuide/access-analyzer-policy-generation.html) for how to easily generate a new policy. +#### Creating assume role +[reference of how to create role](https://github.com/aws-actions/configure-aws-credentials) + +Cloud formation template. You only need to enter org and repo (ex aws amazon-cloudwatch-agent) +``` +Parameters: + GitHubOrg: + Type: String + RepositoryName: + Type: String + OIDCProviderArn: + Description: Arn for the GitHub OIDC Provider. + Default: "" + Type: String + +Conditions: + CreateOIDCProvider: !Equals + - !Ref OIDCProviderArn + - "" + +Resources: + Role: + Type: AWS::IAM::Role + Properties: + AssumeRolePolicyDocument: + Statement: + - Effect: Allow + Action: sts:AssumeRoleWithWebIdentity + Principal: + Federated: !If + - CreateOIDCProvider + - !Ref GithubOidc + - !Ref OIDCProviderArn + Condition: + StringLike: + token.actions.githubusercontent.com:sub: !Sub repo:${GitHubOrg}/${RepositoryName}:* + + GithubOidc: + Type: AWS::IAM::OIDCProvider + Condition: CreateOIDCProvider + Properties: + Url: https://token.actions.githubusercontent.com + ClientIdList: + - sts.amazonaws.com + ThumbprintList: + - 6938fd4d98bab03faadb97b34396831e3780aea1 + +Outputs: + Role: + Value: !GetAtt Role.Arn +``` + ```json { "Version": "2012-10-17", @@ -216,18 +268,17 @@ repository secret for the GitHub actions workflow. Follow [docs](https://docs.github.com/en/actions/security-guides/encrypted-secrets) on configuring GitHub Actions secrets. -| Key | Description | -|-----------------------------------|---------------------------------------------------------------------------------------------------------| -| `AWS_PRIVATE_KEY` | The contents of the `.pem` file (EC2 key pair) that is used to SSH onto EC2 instances | -| `TERRAFORM_AWS_ACCESS_KEY_ID` | IAM user access key | -| `TERRAFORM_AWS_SECRET_ACCESS_KEY` | IAM user secret key | -| `S3_INTEGRATION_BUCKET` | S3 bucket for dumping build artifacts | -| `KEY_NAME` | EC2 key pair name | -| `VPC_SECURITY_GROUPS_IDS` | Security groups for the integration test EC2 instances, in the form of `["sg-abc123"]` (note `"` chars) | -| `IAM_ROLE` | Name of the IAM role to attach to the EC2 instances | -| `GPG_PRIVATE_KEY` | The contents of your GPG private key | -| `PASSPHRASE` | The passphrase to use for GPG signing | -| `GPG_KEY_NAME` | The name of your GPG key, used as the default signing key | +| Key | Description | +|-----------------------------------|----------------------------------------------------------------------------------------------------------| +| `AWS_PRIVATE_KEY` | The contents of the `.pem` file (EC2 key pair) that is used to SSH onto EC2 instances | +| `TERRAFORM_AWS_ASSUME_ROLE` | IAM role to assume | +| `S3_INTEGRATION_BUCKET` | S3 bucket for dumping build artifacts | +| `KEY_NAME` | EC2 key pair name | +| `VPC_SECURITY_GROUPS_IDS` | Security groups for the integration test EC2 instances, in the form of `["sg-abc123"]` (note `"` chars) | +| `IAM_ROLE` | Name of the IAM role to attach to the EC2 instances | +| `GPG_PRIVATE_KEY` | The contents of your GPG private key | +| `PASSPHRASE` | The passphrase to use for GPG signing | +| `GPG_KEY_NAME` | The name of your GPG key, used as the default signing key | ### Run the integration test action on your fork diff --git a/integration/test/performancetest/performance_query_utils.go b/integration/test/performancetest/performance_query_utils.go index e32bd6d703..05a5475c7c 100644 --- a/integration/test/performancetest/performance_query_utils.go +++ b/integration/test/performancetest/performance_query_utils.go @@ -199,6 +199,12 @@ func GetPerformanceMetrics(instanceId string, agentRuntime, logNum, tps int, age //add actual test data with statistics for _, result := range metrics.MetricDataResults { + //convert memory bytes to MB + if (*result.Label == "procstat_memory_rss") { + for i, val := range(result.Values) { + result.Values[i] = val / (1000000) + } + } stats:= CalcStats(result.Values) testMetricResults[*result.Label] = stats }