From d1d22ddf2dc8b7d09af2cfa1f1537d2d07ff6423 Mon Sep 17 00:00:00 2001 From: Michael Levin Date: Wed, 28 Feb 2024 01:05:49 -0500 Subject: [PATCH] Replace CircleCI with github actions --- .circleci/{config.yml => config.yml.old} | 0 .github/workflows/ci.yml | 94 ++++++++++++++++++++++++ .github/workflows/deploy.yml | 91 +++++++++++++++++++++++ knexfile.js.cloudgov | 22 ++++++ manifest.yml | 36 +++++++++ sample.manifest.yml | 36 --------- src/config.js.cloudgov | 36 +++++++++ src/publish/s3.js | 2 +- 8 files changed, 280 insertions(+), 37 deletions(-) rename .circleci/{config.yml => config.yml.old} (100%) create mode 100644 .github/workflows/ci.yml create mode 100644 .github/workflows/deploy.yml create mode 100644 knexfile.js.cloudgov create mode 100644 manifest.yml delete mode 100644 sample.manifest.yml create mode 100644 src/config.js.cloudgov diff --git a/.circleci/config.yml b/.circleci/config.yml.old similarity index 100% rename from .circleci/config.yml rename to .circleci/config.yml.old diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 00000000..9ffb7db1 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,94 @@ +on: + push: + pull_request: + +jobs: + test: + runs-on: ubuntu-latest + # Start Postgres as a service, wait until healthy. Uses latest Postgres version. + services: + postgres: + image: postgres:latest + env: + POSTGRES_DB: analytics_reporter_test + POSTGRES_USER: analytics + POSTGRES_PASSWORD: 123abc + ports: + - 5432:5432 + options: + --health-cmd pg_isready + --health-interval 10s + --health-timeout 5s + --health-retries 5 + steps: + - name: Code checkout + uses: actions/checkout@v4 + - name: Install node + uses: actions/setup-node@v4 + with: + node-version: "lts/*" + cache: 'npm' + - name: Install node dependencies + run: npm ci + - name: Run tests + run: npm test + deploy_dev: + needs: + - test + if: github.ref == 'refs/heads/develop' + uses: 18F/analytics-reporter/.github/workflows/deploy.yml@develop + with: + ANALYTICS_KEY_FILE_NAME: ${{ vars.ANALYTICS_KEY_FILE_NAME }} + ANALYTICS_REPORT_EMAIL: ${{ vars.ANALYTICS_REPORT_EMAIL }} + APP_NAME: ${{ vars.APP_NAME_DEV }} + CF_ORGANIZATION_NAME: ${{ vars.CF_ORGANIZATION_NAME }} + CF_SPACE_NAME: ${{ vars.CF_SPACE_NAME_DEV }} + DB_SERVICE_NAME: ${{ vars.DB_SERVICE_NAME_DEV }} + NEW_RELIC_APP_NAME: ${{ vars.NEW_RELIC_APP_NAME_DEV }} + S3_SERVICE_NAME: ${{ vars.S3_SERVICE_NAME_DEV }} + secrets: + ANALYTICS_CREDENTIALS: ${{ secrets.ANALYTICS_CREDENTIALS }} + CF_USERNAME: ${{ secrets.CF_USERNAME_DEV }} + CF_PASSWORD: ${{ secrets.CF_PASSWORD_DEV }} + GA4_CREDS: ${{ secrets.GA4_CREDS }} + NEW_RELIC_LICENSE_KEY: ${{ secrets.NEW_RELIC_LICENSE_KEY_DEV }} + deploy_stg: + needs: + - test + if: github.ref == 'refs/heads/staging' + uses: 18F/analytics-reporter/.github/workflows/deploy.yml@develop + with: + ANALYTICS_KEY_FILE_NAME: ${{ vars.ANALYTICS_KEY_FILE_NAME }} + ANALYTICS_REPORT_EMAIL: ${{ vars.ANALYTICS_REPORT_EMAIL }} + APP_NAME: ${{ vars.APP_NAME_STG }} + CF_ORGANIZATION_NAME: ${{ vars.CF_ORGANIZATION_NAME }} + CF_SPACE_NAME: ${{ vars.CF_SPACE_NAME_STG }} + DB_SERVICE_NAME: ${{ vars.DB_SERVICE_NAME_STG }} + NEW_RELIC_APP_NAME: ${{ vars.NEW_RELIC_APP_NAME_STG }} + S3_SERVICE_NAME: ${{ vars.S3_SERVICE_NAME_STG }} + secrets: + ANALYTICS_CREDENTIALS: ${{ secrets.ANALYTICS_CREDENTIALS }} + CF_USERNAME: ${{ secrets.CF_USERNAME_STG }} + CF_PASSWORD: ${{ secrets.CF_PASSWORD_STG }} + GA4_CREDS: ${{ secrets.GA4_CREDS }} + NEW_RELIC_LICENSE_KEY: ${{ secrets.NEW_RELIC_LICENSE_KEY_STG }} + deploy_prd: + needs: + - test + if: github.ref == 'refs/heads/master' + uses: 18F/analytics-reporter/.github/workflows/deploy.yml@develop + with: + ANALYTICS_KEY_FILE_NAME: ${{ vars.ANALYTICS_KEY_FILE_NAME }} + ANALYTICS_REPORT_EMAIL: ${{ vars.ANALYTICS_REPORT_EMAIL }} + APP_NAME: ${{ vars.APP_NAME_PRD }} + CF_ORGANIZATION_NAME: ${{ vars.CF_ORGANIZATION_NAME }} + CF_SPACE_NAME: ${{ vars.CF_SPACE_NAME_PRD }} + DB_SERVICE_NAME: ${{ vars.DB_SERVICE_NAME_PRD }} + NEW_RELIC_APP_NAME: ${{ vars.NEW_RELIC_APP_NAME_PRD }} + S3_SERVICE_NAME: ${{ vars.S3_SERVICE_NAME_PRD }} + secrets: + ANALYTICS_CREDENTIALS: ${{ secrets.ANALYTICS_CREDENTIALS }} + CF_USERNAME: ${{ secrets.CF_USERNAME_PRD }} + CF_PASSWORD: ${{ secrets.CF_PASSWORD_PRD }} + GA4_CREDS: ${{ secrets.GA4_CREDS }} + NEW_RELIC_LICENSE_KEY: ${{ secrets.NEW_RELIC_LICENSE_KEY_PRD }} diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml new file mode 100644 index 00000000..ccccf015 --- /dev/null +++ b/.github/workflows/deploy.yml @@ -0,0 +1,91 @@ +on: + workflow_call: + inputs: + ANALYTICS_KEY_FILE_NAME: + required: true + type: string + ANALYTICS_REPORT_EMAIL: + required: true + type: string + APP_NAME: + required: true + type: string + CF_ORGANIZATION_NAME: + required: true + type: string + CF_SPACE_NAME: + required: true + type: string + DB_SERVICE_NAME: + required: true + type: string + NEW_RELIC_APP_NAME: + type: string + S3_SERVICE_NAME: + required: true + type: string + secrets: + ANALYTICS_CREDENTIALS: + required: true + CF_USERNAME: + required: true + CF_PASSWORD: + required: true + GA4_CREDS: + required: true + NEW_RELIC_LICENSE_KEY: + +env: + ANALYTICS_CREDENTIALS: ${{ secrets.ANALYTICS_CREDENTIALS }} + ANALYTICS_KEY_FILE_NAME: ${{ inputs.ANALYTICS_KEY_FILE_NAME }} + ANALYTICS_REPORT_EMAIL: ${{ inputs.ANALYTICS_REPORT_EMAIL }} + APP_NAME: ${{ inputs.APP_NAME }} + CF_USERNAME: ${{ secrets.CF_USERNAME }} + CF_PASSWORD: ${{ secrets.CF_PASSWORD }} + CF_ORGANIZATION_NAME: ${{ inputs.CF_ORGANIZATION_NAME }} + CF_SPACE_NAME: ${{ inputs.CF_SPACE_NAME }} + DB_SERVICE_NAME: ${{ inputs.DB_SERVICE_NAME }} + GA4_CREDS: ${{ secrets.GA4_CREDS }} + NEW_RELIC_APP_NAME: ${{ inputs.NEW_RELIC_APP_NAME }} + NEW_RELIC_LICENSE_KEY: ${{ secrets.NEW_RELIC_LICENSE_KEY }} + S3_SERVICE_NAME: ${{ inputs.S3_SERVICE_NAME }} + +jobs: + deploy_reporter: + runs-on: ubuntu-latest + steps: + - name: Code checkout + uses: actions/checkout@v4 + - name: Install node + uses: actions/setup-node@v4 + with: + node-version: "lts/*" + cache: 'npm' + - name: Install node dependencies + run: npm ci + - name: Install cloud foundry CLI for interacting with cloud.gov + run: | + sudo curl -v -L -o cf8-cli-installer_8.7.4_x86-64.deb 'https://packages.cloudfoundry.org/stable?release=debian64&version=8.7.4' + sudo dpkg -i cf8-cli-installer_8.7.4_x86-64.deb + - name: Write Google GA4 Credentails file from value in CircleCI env var. + run: | + echo $GA4_CREDS > ./my-analytics-ga4-65057af58daa.json + - name: Run envsubst on manifest.yml to set environment specific values + run: | + mv manifest.yml manifest.yml.src + envsubst < manifest.yml.src > manifest.yml + cat manifest.yml + - name: Replace config.js and knexfile.js with .cloudgov versions of those files + run: | + rm ./src/config.js + mv ./src/config.js.cloudgov ./src/config.js + rm knexfile.js + mv knexfile.js.cloudgov knexfile.js + - name: Login to cloud.gov and deploy + run: | + set -e + # Log into cloud.gov + cf api api.fr.cloud.gov + cf login -u $CF_USERNAME -p $CF_PASSWORD -o $CF_ORGANIZATION_NAME -s $CF_SPACE_NAME + cf push -f "./manifest.yml" + cf logout diff --git a/knexfile.js.cloudgov b/knexfile.js.cloudgov new file mode 100644 index 00000000..c75efa66 --- /dev/null +++ b/knexfile.js.cloudgov @@ -0,0 +1,22 @@ +const VCAP_SERVICES_JSON = JSON.parse(process.env.VCAP_SERVICES); + +module.exports = { + production: { + client: 'postgresql', + connection: { + host : VCAP_SERVICES_JSON["aws-rds"][0]["credentials"]["host"], + user : VCAP_SERVICES_JSON["aws-rds"][0]["credentials"]["username"], + password : VCAP_SERVICES_JSON["aws-rds"][0]["credentials"]["password"], + database : VCAP_SERVICES_JSON["aws-rds"][0]["credentials"]["db_name"], + port: 5432, + ssl : true + }, + pool: { + min: 2, + max: 10 + }, + migrations: { + tableName: 'knex_migrations' + } + } +}; diff --git a/manifest.yml b/manifest.yml new file mode 100644 index 00000000..ee0a54c7 --- /dev/null +++ b/manifest.yml @@ -0,0 +1,36 @@ +applications: +- name: ${APP_NAME} + instances: 1 + memory: 256M + disk_quota: 1024M + no-route: true + health-check-type: process + buildpacks: + - nodejs_buildpack + command: node deploy/cron.js + env: + # The default path for reports (used for gov-wide reports) + AWS_BUCKET_PATH: data/live + ANALYTICS_CREDENTIALS: '${ANALYTICS_CREDENTIALS}' + ANALYTICS_DEBUG: true + ANALYTICS_KEY_PATH: /home/vcap/app/${ANALYTICS_KEY_FILE_NAME} + # The default property ID for reports (used for gov-wide reports) + ANALYTICS_REPORT_IDS: 393249053 + # The default property ID for UA reports (used for gov-wide reports) + ANALYTICS_REPORT_UA_IDS: ga:96302018 + ANALYTICS_REPORT_EMAIL: ${ANALYTICS_REPORT_EMAIL} + ANALYTICS_REPORTS_PATH: /home/vcap/app/reports/usa.json + ANALYTICS_ROOT_PATH: /home/vcap/app + ANALYTICS_UA_ROOT_PATH: /home/vcap/app/ua + AWS_CACHE_TIME: 0 + GOOGLE_APPLICATION_CREDENTIALS: /home/vcap/app/${ANALYTICS_KEY_FILE_NAME} + NEW_RELIC_APP_NAME: ${NEW_RELIC_APP_NAME} + NEW_RELIC_LICENSE_KEY: ${NEW_RELIC_LICENSE_KEY} + NODE_ENV: production + PGSSLMODE: true + services: + - ${S3_SERVICE_NAME} + - ${DB_SERVICE_NAME} + stack: cflinuxfs4 + timeout: 180 + path: . diff --git a/sample.manifest.yml b/sample.manifest.yml deleted file mode 100644 index 231e5ce8..00000000 --- a/sample.manifest.yml +++ /dev/null @@ -1,36 +0,0 @@ -applications: -- name: analytics-reporter - instances: 1 - memory: 256M - disk_quota: 1024M - no-route: true - health-check-type: process - buildpacks: - - nodejs_buildpack - command: node deploy/cron.js - env: - ANALYTICS_ROOT_PATH: - ANALYTICS_REPORT_IDS: - ANALYTICS_REPORTS_PATH: - AWS_ACCESS_KEY_ID: - AWS_BUCKET: - AWS_BUCKET_PATH: - AWS_CACHE_TIME: - AWS_DEFAULT_REGION: - AWS_REGION: - AWS_SECRET_ACCESS_KEY: - CREDS: - BUCKET_NAME: - PGSSLMODE: true - POSTGRES_DATABASE: - POSTGRES_HOST: - POSTGRES_PASSWORD: - POSTGRES_USER: - NODE_ENV: production - ANALYTICS_DEBUG: true - services: - - analytics-s3 - - analytics-reporter-database - stack: cflinuxfs4 - timeout: 180 - path: . diff --git a/src/config.js.cloudgov b/src/config.js.cloudgov new file mode 100644 index 00000000..5b5f3135 --- /dev/null +++ b/src/config.js.cloudgov @@ -0,0 +1,36 @@ +const knexfile = require('../knexfile'); +const VCAP_SERVICES_JSON = JSON.parse(process.env.VCAP_SERVICES); + + +// Set environment variables to configure the application. +module.exports = { + email: process.env.ANALYTICS_REPORT_EMAIL, + key: process.env.ANALYTICS_KEY, + key_file: process.env.GOOGLE_APPLICATION_CREDENTIALS || process.env.ANALYTICS_KEY_PATH, + analytics_credentials: process.env.ANALYTICS_CREDENTIALS, + reports_file: process.env.ANALYTICS_REPORTS_PATH, + debug: (process.env.ANALYTICS_DEBUG ? true : false), + // AWS S3 information. + aws: { + bucket: VCAP_SERVICES_JSON["s3"][0]["credentials"]["bucket"], + path: process.env.AWS_BUCKET_PATH, + // HTTP cache time in seconds. Defaults to 0. + cache: process.env.AWS_CACHE_TIME, + endpoint: VCAP_SERVICES_JSON["s3"][0]["credentials"]["endpoint"], + accessKeyId: VCAP_SERVICES_JSON["s3"][0]["credentials"]["access_key_id"], + secretAccessKey: VCAP_SERVICES_JSON["s3"][0]["credentials"]["secret_access_key"], + s3ForcePathStyle: process.env.AWS_S3_FORCE_STYLE_PATH, + signatureVersion: process.env.AWS_SIGNATURE_VERSION + }, + account: { + ids: process.env.ANALYTICS_REPORT_IDS, + agency_name: process.env.AGENCY_NAME, + // needed for realtime reports which don't include hostname + // leave blank if your view includes hostnames + hostname: process.env.ANALYTICS_HOSTNAME || "", + }, + postgres: knexfile[process.env.NODE_ENV || "development"].connection, + static: { + path: '../analytics.usa.gov/', + }, +}; diff --git a/src/publish/s3.js b/src/publish/s3.js index 9fd9159b..4e54dc5e 100644 --- a/src/publish/s3.js +++ b/src/publish/s3.js @@ -7,7 +7,7 @@ const logger = require('../../src/logger').initialize(); const s3config = { accessKeyId: config.aws.accessKeyId, secretAccessKey: config.aws.secretAccessKey, - endpoint: config.aws.endpoint, + endpoint: `https://${config.aws.endpoint}`, s3ForcePathStyle: config.aws.s3ForcePathStyle, signatureVersion: config.aws.signatureVersion, };