[WIP] Add localstack and cf-mock for local development

.dockerignore

@@ -0,0 +1,53 @@
+# Local config
+config/local.js
+config/local-from-staging.js
+
+# Webpack-generated assets
+public/js/bundle.js
+public/js/bundle.*.js*
+public/styles/styles.css
+public/styles/styles.*.css*
+public/styles/uswds.*.css*
+webpack-manifest.json
+webpackAssets
+public/stats.json
+dist
+
+
+# Node.js / NPM
+lib-cov
+*.seed
+*.log
+*.out
+*.pid
+npm-debug.log
+node_modules/
+coverage
+.nyc_output
+.yarn-cache
+
+# Miscellaneous
+*~
+*#
+.git
+.DS_STORE
+.netbeans
+nbproject
+.idea
+.node_history
+cf-ssh.yml
+.env
+.sass-cache
+current-sites.csv
+.vscode
+credentials*.json
+tmp
+ci/vars/.*
+uaa/cloudfoundry-identity-uaa-4.19.0.war
+test-results/
+playwright-report/
+blob-report/
+playwright/.cache/
+playwright/.auth/*
+!playwright/.auth/.gitkeep
+volume

.gitignore

@@ -49,5 +49,7 @@ playwright-report/
 blob-report/
 playwright/.cache/
 playwright/.auth/*
-!playwright/.auth/.gitkeep 
+!playwright/.auth/.gitkeep
 database.json
+zscaler.crt
+volume

Dockerfile-localstack

@@ -0,0 +1,7 @@
+FROM localstack/localstack:latest
+
+COPY zscaler.crt /usr/local/share/ca-certificates/cert-bundle.crt
+RUN update-ca-certificates
+ENV CURL_CA_BUNDLE=/etc/ssl/certs/ca-certificates.crt
+ENV REQUESTS_CA_BUNDLE=/etc/ssl/certs/ca-certificates.crt
+ENV NODE_EXTRA_CA_CERTS=/etc/ssl/certs/ca-certificates.crt

api/services/S3Helper.js

@@ -17,6 +17,9 @@ class S3Client {
     this.bucket = credentials.bucket;
     this.client = new S3({
       region: credentials.region,
+      // https://docs.localstack.cloud/user-guide/integrations/sdks/javascript/
+      // TODO: this might break dev; don't merge without fixing
+      forcePathStyle: process.env.NODE_ENV === 'development',
       credentials: {
         accessKeyId: credentials.accessKeyId,
         secretAccessKey: credentials.secretAccessKey,
@@ -46,6 +49,7 @@ class S3Client {
         results.push(...page[property]);
       }
     }
+
     return results;
   }
 

api/services/S3PublishedFileLister.js

@@ -3,21 +3,6 @@ const CloudFoundryAPIClient = require('../utils/cfApiClient');
 
 const apiClient = new CloudFoundryAPIClient();
 
-const handleInvalidAccessKeyError = (error) => {
-  const validS3KeyUpdateEnv = process.env.NODE_ENV === 'development'
-    || process.env.NODE_ENV === 'test';
-
-  if (error.code === 'InvalidAccessKeyId' && validS3KeyUpdateEnv) {
-    const message = 'S3 keys out of date. Update them with `npm run update-local-config`';
-    throw {
-      message,
-      status: 400,
-    };
-  }
-
-  throw error;
-};
-
 function listTopLevelFolders(s3Client, path) {
   // Lists all top-level "folders" in the S3 bucket that start with
   // the given prefix path.
@@ -27,8 +12,7 @@ function listTopLevelFolders(s3Client, path) {
   return s3Client.listCommonPrefixes(path)
     .then(commonPrefixes => commonPrefixes.map(
       prefix => prefix.Prefix.split('/').slice(-2)[0]
-    ))
-    .catch(handleInvalidAccessKeyError);
+    ));
 }
 
 function listFilesPaged(s3Client, path, startAtKey = null) {
@@ -61,12 +45,10 @@ function listFilesPaged(s3Client, path, startAtKey = null) {
         files,
       };
     })
-    .catch(handleInvalidAccessKeyError);
 }
 
 function listPublishedPreviews(site) {
   const previewPath = `preview/${site.owner}/${site.repository}/`;
-
   return apiClient.fetchServiceInstanceCredentials(site.s3ServiceName)
     .then((credentials) => {
       const s3Client = new S3Helper.S3Client({

cf-mock/index.js

@@ -0,0 +1,16 @@
+const express = require('express');
+const router = require('./routers');
+const responses = require('../api/responses');
+const { expressLogger, expressErrorLogger } = require('../winston');
+
+const app = express();
+const port = 1234;
+
+app.use(expressLogger);
+app.use(expressErrorLogger);
+app.use(responses);
+app.use('/', router);
+
+app.listen(port, () => {
+  console.log(`Example app listening on port ${port}`);
+});

cf-mock/routers/index.js

@@ -0,0 +1,12 @@
+const express = require('express');
+
+const apiRouter = express.Router();
+const mainRouter = express.Router();
+
+apiRouter.use(require('./service-credential-binding'));
+// apiRouter.use(require('./build'));
+
+mainRouter.use('/v3', apiRouter);
+mainRouter.use(require('./oauth'));
+
+module.exports = mainRouter;

cf-mock/routers/oauth.js

@@ -0,0 +1,8 @@
+const jwt = require('jsonwebtoken');
+const router = require('express').Router();
+
+router.post('/oauth/token', (req, res) => res.send({
+  access_token: jwt.sign({ exp: (Date.now() / 1000) + 600 }, '123abc'),
+}));
+
+module.exports = router;

cf-mock/routers/service-credential-binding.js

@@ -0,0 +1,54 @@
+const crypto = require('crypto');
+const router = require('express').Router();
+
+const { createCFAPIResource, createCFAPIResourceList } = require('../../test/api/support/factory/cf-api-response');
+const { Site } = require('../../api/models');
+
+// TODO: move this, don't use a global
+// on init, make a service credential binding for each site
+const serviceCredentialBindings = [];
+async function initServiceCredentialBindings() {
+  const sites = await Site.findAll();
+  sites.forEach((site) => {
+    const serviceCredentialBinding = {
+      name: site.s3ServiceName,
+      guid: crypto.randomUUID(),
+      siteInfo: site,
+    };
+    serviceCredentialBindings.push(serviceCredentialBinding);
+  });
+}
+initServiceCredentialBindings();
+
+router.get('/service_credential_bindings', (req, res) => {
+  const name = req.query.service_instance_names;
+  const serviceCredentialBinding = serviceCredentialBindings.find(scb => scb.name === name);
+  if (!serviceCredentialBinding) {
+    return res.notFound();
+  }
+
+  const { guid } = serviceCredentialBinding;
+  const credentialsServiceInstance = createCFAPIResource({ name, guid });
+  const listCredentialServices = createCFAPIResourceList({
+    resources: [credentialsServiceInstance],
+  });
+  return res.send(listCredentialServices);
+});
+
+router.get('/service_credential_bindings/:guid/details', (req, res) => {
+  const serviceCredentialBinding = serviceCredentialBindings
+    .find(scb => scb.guid === req.params.guid);
+
+  if (!serviceCredentialBinding) {
+    return res.notFound();
+  }
+  const credentials = {
+    access_key_id: 'test',
+    secret_access_key: 'test',
+    region: serviceCredentialBinding.siteInfo.awsBucketRegion,
+    bucket: serviceCredentialBinding.siteInfo.awsBucketName,
+  };
+  return res.send({ credentials });
+});
+
+module.exports = router;

config/env/development.js

@@ -7,4 +7,10 @@ module.exports = {
   userEnvVar: {
     key: 'shhhhhhhhhhh',
   },
+  s3BuildLogs: {
+    accessKeyId: 'test',
+    secretAccessKey: 'test',
+    region: 'us-gov-west-1',
+    bucket: 'build-logs',
+  },
 };

docker-compose.yml

@@ -25,6 +25,7 @@ services:
     depends_on:
       - db
       - redis
+      - localstack
     environment:
       APP_HOSTNAME: http://localhost:1337
       DOMAIN: localhost:1337
@@ -34,11 +35,13 @@ services:
       UAA_HOST: http://localhost:9000
       UAA_HOST_DOCKER_URL: http://uaa:8080
       USER_AUDITOR: federalist
-      CLOUD_FOUNDRY_API_HOST: https://api.example.com
-      CLOUD_FOUNDRY_OAUTH_TOKEN_URL: https://login.example.com/oauth/token
+      CLOUD_FOUNDRY_API_HOST: http://cf-mock:1234
+      CLOUD_FOUNDRY_OAUTH_TOKEN_URL: http://cf-mock:1234/oauth/token
       CF_API_USERNAME: deploy_user
       CF_API_PASSWORD: deploy_pass
       PROXY_DOMAIN: localhost:1337
+      AWS_ENDPOINT_URL: http://localstack:4566
+      NODE_ENV: development
   bull-board:
     build:
       dockerfile: Dockerfile-app
@@ -110,3 +113,47 @@ services:
       HOST: 0.0.0.0
       PORT: 8989
     command: node app/scripts/echo.js
+  localstack:
+    build:
+      dockerfile: Dockerfile-localstack
+      context: .
+    ports:
+      - "127.0.0.1:4566:4566"            # LocalStack Gateway
+      - "127.0.0.1:4510-4559:4510-4559"  # external services port range
+    environment:
+      - DEBUG=${DEBUG-}
+      - DOCKER_HOST=unix:///var/run/docker.sock
+    volumes:
+      - "${LOCALSTACK_VOLUME_DIR:-./volume}:/var/lib/localstack"
+  localstack-setup:
+    build:
+      dockerfile: Dockerfile-app
+      context: .
+    volumes:
+      - yarn:/usr/local/share/.cache/yarn
+      - .:/app
+      - nm-app:/app/node_modules
+    command: ["scripts/wait-for-it.sh", "localstack:4566", "--", "yarn", "localstack-setup"]
+    depends_on:
+      - localstack
+      - db # race condition waiting to happen as we don't wait for DB
+    environment:
+      NODE_ENV: development
+      AWS_ENDPOINT_URL: http://localstack:4566
+  cf-mock:
+    build:
+      dockerfile: Dockerfile-app
+      context: .
+    volumes:
+      - yarn:/usr/local/share/.cache/yarn
+      - .:/app
+      - nm-app:/app/node_modules
+    command: ["scripts/wait-for-it.sh", "db:5432", "--", "yarn", "cf-mock"]
+    ports:
+      - "1234:1234"
+    depends_on:
+      - db
+      - localstack
+    environment:
+      NODE_ENV: development
+      AWS_ENDPOINT_URL: http://localstack:4566

package.json

@@ -105,7 +105,9 @@
     "queued-builds-check": "node ./scripts/queued-builds-check.js",
     "test:e2e": "yarn playwright test",
     "create-test-users": "DOTENV_CONFIG_PATH=.env node -r dotenv/config ./scripts/create-test-users.js",
-    "remove-test-users": "DOTENV_CONFIG_PATH=.env node -r dotenv/config ./scripts/remove-test-users.js"
+    "remove-test-users": "DOTENV_CONFIG_PATH=.env node -r dotenv/config ./scripts/remove-test-users.js",
+    "localstack-setup": "NODE_ENV=development node ./scripts/localstack-setup.js",
+    "cf-mock": "NODE_ENV=development nodemon cf-mock/index.js"
   },
   "main": "index.js",
   "repository": {

scripts/create-dev-data.js

@@ -1,6 +1,7 @@
 /* eslint-disable no-console */
 const { addDays, addMinutes } = require('date-fns');
 Promise.props = require('promise-props');
+const crypto = require('crypto');
 const BuildLogs = require('../api/services/build-logs');
 const { encrypt } = require('../api/services/Encryptor');
 const EventCreator = require('../api/services/EventCreator');
@@ -114,6 +115,10 @@ function socketIOError() {
   };
 }
 
+function randomGitSha() {
+  return crypto.createHash('sha1').update(crypto.randomBytes(30)).digest('hex');
+}
+
 // Chainable helpers
 async function createUAAIdentity(user) {
   await user.createUAAIdentity({
@@ -380,6 +385,7 @@ async function createData() {
       user: user1.id,
       username: user1.username,
       token: 'fake-token',
+      requestedCommitSha: randomGitSha(),
     }),
     Build.create({
       branch: site1.defaultBranch,
@@ -388,9 +394,8 @@ async function createData() {
       user: user1.id,
       username: user1.username,
       token: 'fake-token',
-    }).then(build => build.update({
-      requestedCommitSha: '57ce109dcc2cb8675ccbc2d023f40f82a2deabe1',
-    })),
+      requestedCommitSha: randomGitSha(),
+    }),
     Build.create({
       branch: site1.demoBranch,
       source: 'fake-build',