diff --git a/.dockerignore b/.dockerignore deleted file mode 120000 index 3e4e48b0b..000000000 --- a/.dockerignore +++ /dev/null @@ -1 +0,0 @@ -.gitignore \ No newline at end of file diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 000000000..07fbe4875 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,53 @@ +# Local config +config/local.js +config/local-from-staging.js + +# Webpack-generated assets +public/js/bundle.js +public/js/bundle.*.js* +public/styles/styles.css +public/styles/styles.*.css* +public/styles/uswds.*.css* +webpack-manifest.json +webpackAssets +public/stats.json +dist + + +# Node.js / NPM +lib-cov +*.seed +*.log +*.out +*.pid +npm-debug.log +node_modules/ +coverage +.nyc_output +.yarn-cache + +# Miscellaneous +*~ +*# +.git +.DS_STORE +.netbeans +nbproject +.idea +.node_history +cf-ssh.yml +.env +.sass-cache +current-sites.csv +.vscode +credentials*.json +tmp +ci/vars/.* +uaa/cloudfoundry-identity-uaa-4.19.0.war +test-results/ +playwright-report/ +blob-report/ +playwright/.cache/ +playwright/.auth/* +!playwright/.auth/.gitkeep +volume diff --git a/.gitignore b/.gitignore index 7a40d2550..3797523e2 100644 --- a/.gitignore +++ b/.gitignore @@ -49,5 +49,7 @@ playwright-report/ blob-report/ playwright/.cache/ playwright/.auth/* -!playwright/.auth/.gitkeep +!playwright/.auth/.gitkeep database.json +zscaler.crt +volume diff --git a/Dockerfile-localstack b/Dockerfile-localstack new file mode 100644 index 000000000..0543737cc --- /dev/null +++ b/Dockerfile-localstack @@ -0,0 +1,7 @@ +FROM localstack/localstack:latest + +COPY zscaler.crt /usr/local/share/ca-certificates/cert-bundle.crt +RUN update-ca-certificates +ENV CURL_CA_BUNDLE=/etc/ssl/certs/ca-certificates.crt +ENV REQUESTS_CA_BUNDLE=/etc/ssl/certs/ca-certificates.crt +ENV NODE_EXTRA_CA_CERTS=/etc/ssl/certs/ca-certificates.crt \ No newline at end of file diff --git a/api/services/S3Helper.js b/api/services/S3Helper.js index 4df7296d0..3457b2a6d 100644 --- a/api/services/S3Helper.js +++ b/api/services/S3Helper.js @@ -17,6 +17,9 @@ class S3Client { this.bucket = credentials.bucket; this.client = new S3({ region: credentials.region, + // https://docs.localstack.cloud/user-guide/integrations/sdks/javascript/ + // TODO: this might break dev; don't merge without fixing + forcePathStyle: process.env.NODE_ENV === 'development', credentials: { accessKeyId: credentials.accessKeyId, secretAccessKey: credentials.secretAccessKey, @@ -46,6 +49,7 @@ class S3Client { results.push(...page[property]); } } + return results; } diff --git a/api/services/S3PublishedFileLister.js b/api/services/S3PublishedFileLister.js index f74483271..1f0c9df35 100644 --- a/api/services/S3PublishedFileLister.js +++ b/api/services/S3PublishedFileLister.js @@ -3,21 +3,6 @@ const CloudFoundryAPIClient = require('../utils/cfApiClient'); const apiClient = new CloudFoundryAPIClient(); -const handleInvalidAccessKeyError = (error) => { - const validS3KeyUpdateEnv = process.env.NODE_ENV === 'development' - || process.env.NODE_ENV === 'test'; - - if (error.code === 'InvalidAccessKeyId' && validS3KeyUpdateEnv) { - const message = 'S3 keys out of date. Update them with `npm run update-local-config`'; - throw { - message, - status: 400, - }; - } - - throw error; -}; - function listTopLevelFolders(s3Client, path) { // Lists all top-level "folders" in the S3 bucket that start with // the given prefix path. @@ -27,8 +12,7 @@ function listTopLevelFolders(s3Client, path) { return s3Client.listCommonPrefixes(path) .then(commonPrefixes => commonPrefixes.map( prefix => prefix.Prefix.split('/').slice(-2)[0] - )) - .catch(handleInvalidAccessKeyError); + )); } function listFilesPaged(s3Client, path, startAtKey = null) { @@ -61,12 +45,10 @@ function listFilesPaged(s3Client, path, startAtKey = null) { files, }; }) - .catch(handleInvalidAccessKeyError); } function listPublishedPreviews(site) { const previewPath = `preview/${site.owner}/${site.repository}/`; - return apiClient.fetchServiceInstanceCredentials(site.s3ServiceName) .then((credentials) => { const s3Client = new S3Helper.S3Client({ diff --git a/cf-mock/index.js b/cf-mock/index.js new file mode 100644 index 000000000..150591a6f --- /dev/null +++ b/cf-mock/index.js @@ -0,0 +1,16 @@ +const express = require('express'); +const router = require('./routers'); +const responses = require('../api/responses'); +const { expressLogger, expressErrorLogger } = require('../winston'); + +const app = express(); +const port = 1234; + +app.use(expressLogger); +app.use(expressErrorLogger); +app.use(responses); +app.use('/', router); + +app.listen(port, () => { + console.log(`Example app listening on port ${port}`); +}); diff --git a/cf-mock/routers/index.js b/cf-mock/routers/index.js new file mode 100644 index 000000000..0e804de09 --- /dev/null +++ b/cf-mock/routers/index.js @@ -0,0 +1,12 @@ +const express = require('express'); + +const apiRouter = express.Router(); +const mainRouter = express.Router(); + +apiRouter.use(require('./service-credential-binding')); +// apiRouter.use(require('./build')); + +mainRouter.use('/v3', apiRouter); +mainRouter.use(require('./oauth')); + +module.exports = mainRouter; diff --git a/cf-mock/routers/oauth.js b/cf-mock/routers/oauth.js new file mode 100644 index 000000000..6da3dc0e6 --- /dev/null +++ b/cf-mock/routers/oauth.js @@ -0,0 +1,8 @@ +const jwt = require('jsonwebtoken'); +const router = require('express').Router(); + +router.post('/oauth/token', (req, res) => res.send({ + access_token: jwt.sign({ exp: (Date.now() / 1000) + 600 }, '123abc'), +})); + +module.exports = router; diff --git a/cf-mock/routers/service-credential-binding.js b/cf-mock/routers/service-credential-binding.js new file mode 100644 index 000000000..193192a19 --- /dev/null +++ b/cf-mock/routers/service-credential-binding.js @@ -0,0 +1,54 @@ +const crypto = require('crypto'); +const router = require('express').Router(); + +const { createCFAPIResource, createCFAPIResourceList } = require('../../test/api/support/factory/cf-api-response'); +const { Site } = require('../../api/models'); + +// TODO: move this, don't use a global +// on init, make a service credential binding for each site +const serviceCredentialBindings = []; +async function initServiceCredentialBindings() { + const sites = await Site.findAll(); + sites.forEach((site) => { + const serviceCredentialBinding = { + name: site.s3ServiceName, + guid: crypto.randomUUID(), + siteInfo: site, + }; + serviceCredentialBindings.push(serviceCredentialBinding); + }); +} +initServiceCredentialBindings(); + +router.get('/service_credential_bindings', (req, res) => { + const name = req.query.service_instance_names; + const serviceCredentialBinding = serviceCredentialBindings.find(scb => scb.name === name); + if (!serviceCredentialBinding) { + return res.notFound(); + } + + const { guid } = serviceCredentialBinding; + const credentialsServiceInstance = createCFAPIResource({ name, guid }); + const listCredentialServices = createCFAPIResourceList({ + resources: [credentialsServiceInstance], + }); + return res.send(listCredentialServices); +}); + +router.get('/service_credential_bindings/:guid/details', (req, res) => { + const serviceCredentialBinding = serviceCredentialBindings + .find(scb => scb.guid === req.params.guid); + + if (!serviceCredentialBinding) { + return res.notFound(); + } + const credentials = { + access_key_id: 'test', + secret_access_key: 'test', + region: serviceCredentialBinding.siteInfo.awsBucketRegion, + bucket: serviceCredentialBinding.siteInfo.awsBucketName, + }; + return res.send({ credentials }); +}); + +module.exports = router; diff --git a/config/env/development.js b/config/env/development.js index 671e4fe48..3470540d9 100644 --- a/config/env/development.js +++ b/config/env/development.js @@ -7,4 +7,10 @@ module.exports = { userEnvVar: { key: 'shhhhhhhhhhh', }, + s3BuildLogs: { + accessKeyId: 'test', + secretAccessKey: 'test', + region: 'us-gov-west-1', + bucket: 'build-logs', + }, }; diff --git a/docker-compose.yml b/docker-compose.yml index 8feeb29c1..7a08422c4 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -25,6 +25,7 @@ services: depends_on: - db - redis + - localstack environment: APP_HOSTNAME: http://localhost:1337 DOMAIN: localhost:1337 @@ -34,11 +35,13 @@ services: UAA_HOST: http://localhost:9000 UAA_HOST_DOCKER_URL: http://uaa:8080 USER_AUDITOR: federalist - CLOUD_FOUNDRY_API_HOST: https://api.example.com - CLOUD_FOUNDRY_OAUTH_TOKEN_URL: https://login.example.com/oauth/token + CLOUD_FOUNDRY_API_HOST: http://cf-mock:1234 + CLOUD_FOUNDRY_OAUTH_TOKEN_URL: http://cf-mock:1234/oauth/token CF_API_USERNAME: deploy_user CF_API_PASSWORD: deploy_pass PROXY_DOMAIN: localhost:1337 + AWS_ENDPOINT_URL: http://localstack:4566 + NODE_ENV: development bull-board: build: dockerfile: Dockerfile-app @@ -110,3 +113,47 @@ services: HOST: 0.0.0.0 PORT: 8989 command: node app/scripts/echo.js + localstack: + build: + dockerfile: Dockerfile-localstack + context: . + ports: + - "127.0.0.1:4566:4566" # LocalStack Gateway + - "127.0.0.1:4510-4559:4510-4559" # external services port range + environment: + - DEBUG=${DEBUG-} + - DOCKER_HOST=unix:///var/run/docker.sock + volumes: + - "${LOCALSTACK_VOLUME_DIR:-./volume}:/var/lib/localstack" + localstack-setup: + build: + dockerfile: Dockerfile-app + context: . + volumes: + - yarn:/usr/local/share/.cache/yarn + - .:/app + - nm-app:/app/node_modules + command: ["scripts/wait-for-it.sh", "localstack:4566", "--", "yarn", "localstack-setup"] + depends_on: + - localstack + - db # race condition waiting to happen as we don't wait for DB + environment: + NODE_ENV: development + AWS_ENDPOINT_URL: http://localstack:4566 + cf-mock: + build: + dockerfile: Dockerfile-app + context: . + volumes: + - yarn:/usr/local/share/.cache/yarn + - .:/app + - nm-app:/app/node_modules + command: ["scripts/wait-for-it.sh", "db:5432", "--", "yarn", "cf-mock"] + ports: + - "1234:1234" + depends_on: + - db + - localstack + environment: + NODE_ENV: development + AWS_ENDPOINT_URL: http://localstack:4566 \ No newline at end of file diff --git a/package.json b/package.json index 976d1e8cc..41ee82a04 100644 --- a/package.json +++ b/package.json @@ -105,7 +105,9 @@ "queued-builds-check": "node ./scripts/queued-builds-check.js", "test:e2e": "yarn playwright test", "create-test-users": "DOTENV_CONFIG_PATH=.env node -r dotenv/config ./scripts/create-test-users.js", - "remove-test-users": "DOTENV_CONFIG_PATH=.env node -r dotenv/config ./scripts/remove-test-users.js" + "remove-test-users": "DOTENV_CONFIG_PATH=.env node -r dotenv/config ./scripts/remove-test-users.js", + "localstack-setup": "NODE_ENV=development node ./scripts/localstack-setup.js", + "cf-mock": "NODE_ENV=development nodemon cf-mock/index.js" }, "main": "index.js", "repository": { diff --git a/scripts/create-dev-data.js b/scripts/create-dev-data.js index 0fe742355..fa0963f00 100644 --- a/scripts/create-dev-data.js +++ b/scripts/create-dev-data.js @@ -1,6 +1,7 @@ /* eslint-disable no-console */ const { addDays, addMinutes } = require('date-fns'); Promise.props = require('promise-props'); +const crypto = require('crypto'); const BuildLogs = require('../api/services/build-logs'); const { encrypt } = require('../api/services/Encryptor'); const EventCreator = require('../api/services/EventCreator'); @@ -114,6 +115,10 @@ function socketIOError() { }; } +function randomGitSha() { + return crypto.createHash('sha1').update(crypto.randomBytes(30)).digest('hex'); +} + // Chainable helpers async function createUAAIdentity(user) { await user.createUAAIdentity({ @@ -380,6 +385,7 @@ async function createData() { user: user1.id, username: user1.username, token: 'fake-token', + requestedCommitSha: randomGitSha(), }), Build.create({ branch: site1.defaultBranch, @@ -388,9 +394,8 @@ async function createData() { user: user1.id, username: user1.username, token: 'fake-token', - }).then(build => build.update({ - requestedCommitSha: '57ce109dcc2cb8675ccbc2d023f40f82a2deabe1', - })), + requestedCommitSha: randomGitSha(), + }), Build.create({ branch: site1.demoBranch, source: 'fake-build', diff --git a/scripts/localstack-setup.js b/scripts/localstack-setup.js new file mode 100755 index 000000000..f9e5abce6 --- /dev/null +++ b/scripts/localstack-setup.js @@ -0,0 +1,108 @@ +const { S3Client, CreateBucketCommand, PutObjectCommand } = require('@aws-sdk/client-s3'); +const moment = require('moment'); +const { Op } = require('sequelize'); + +const { + Build, + BuildTask, + Site, +} = require('../api/models'); +const BuildLogs = require('../api/services/build-logs'); + +const config = require('../config'); + +const s3 = new S3Client({ + region: 'us-gov-west-1', + forcePathStyle: true, + credentials: { + accessKeyId: 'test', + secretAccessKey: 'test', + }, +}); + +const fakeHtml = ` + +site + + +Website + + +`; + +async function initSitesAndBuckets() { + // create Build Logs bucket + const cmd = new CreateBucketCommand({ Bucket: config.s3BuildLogs.bucket }); + await s3.send(cmd); + + // create a bucket and some files for each Site + await Site.findAll() + .then(async (sites) => { + await Promise.all(sites.map(async (site) => { + const siteCmd = new CreateBucketCommand({ Bucket: site.awsBucketName }); + await s3.send(siteCmd); + const folders = ['site', 'demo']; + folders.forEach(async (folder) => { + const folderCmd = new PutObjectCommand({ + Body: fakeHtml, + Bucket: site.awsBucketName, + Key: `${folder}/${site.owner}/${site.repository}/index.html`, + }); + await s3.send(folderCmd); + }); + const fakePreviews = ['branch1', 'branch2', 'branch3']; + fakePreviews.forEach(async (branch) => { + const previewCmd = new PutObjectCommand({ + Body: fakeHtml, + Bucket: site.awsBucketName, + Key: `preview/${site.owner}/${site.repository}/${branch}/index.html`, + }); + await s3.send(previewCmd); + }); + })); + }) + .catch(err => console.error(err)); +} + +// archive older build logs +async function archiveLogs() { + const date = moment().subtract(3, 'days').startOf('day'); + + const builds = await Build.findAll({ + attributes: ['id'], + where: { + completedAt: { + [Op.lt]: date.toDate(), + }, + }, + }); + + console.log(`Found ${builds.length} builds.`); + + builds.forEach(build => BuildLogs.archiveBuildLogsForBuildId(build.id)); +} + +// create artifacts for Build Tasks +async function createBuildTaskArtifacts() { + const artifactedTasks = await BuildTask.findAll({ + where: { + artifact: { + [Op.not]: null, + }, + }, + include: [{ model: Build, include: [Site] }], + }); + artifactedTasks.forEach((task) => { + console.log(task.Build.Site) + const artifactCmd = new PutObjectCommand({ + Body: fakeHtml, + Bucket: task.Build.Site.awsBucketName, + Key: task.artifact, + }); + s3.send(artifactCmd); + }); +} + +initSitesAndBuckets() + .then(archiveLogs) + .then(createBuildTaskArtifacts); diff --git a/test/api/support/factory/site.js b/test/api/support/factory/site.js index 58df6baa2..aeb5fc78f 100644 --- a/test/api/support/factory/site.js +++ b/test/api/support/factory/site.js @@ -8,6 +8,8 @@ function generateUniqueAtts() { const res = { owner: `repo-owner-${siteAttsStep}`, repository: `repo-name-${siteAttsStep}`, + awsBucketName: `cg-bucket-name-${siteAttsStep}`, + s3ServiceName: `s3-service-${siteAttsStep}`, }; siteAttsStep += 1; return res; @@ -20,14 +22,16 @@ function makeAttributes(overrides = {}) { users = Promise.all([userFactory()]); } - const { owner, repository } = generateUniqueAtts(); + const { + owner, repository, awsBucketName, s3ServiceName, + } = generateUniqueAtts(); return { owner, repository, engine: 'jekyll', - s3ServiceName: 'federalist-dev-s3', - awsBucketName: 'cg-123456789', + s3ServiceName, + awsBucketName, awsBucketRegion: 'us-gov-west-1', defaultBranch: 'main', subdomain: generateSubdomain(owner, repository),