diff --git a/Makefile b/Makefile index ab56ec661..c3834651e 100644 --- a/Makefile +++ b/Makefile @@ -34,44 +34,44 @@ lint-client: ## Lint admin client code lint: lint-server lint-client ## Lint project lint-fix: ## lint and fix - docker compose run --rm app yarn format:lint + docker compose --env-file ./services/local/docker.env run --rm app yarn format:lint format: - docker compose run --rm app yarn format + docker compose --env-file ./services/local/docker.env run --rm app yarn format format-check: - docker compose run --rm app yarn format:check + docker compose --env-file ./services/local/docker.env run --rm app yarn format:check migrate: ## Run database migrations - docker compose run --rm app yarn migrate:up + docker compose --env-file ./services/local/docker.env run --rm app yarn migrate:up rebuild: ## Rebuild docker images and database volumes docker volume rm pages-core_db-data - docker compose -f ./docker-compose.yml -f ./docker-compose.uaa.yml --env-file ./services/local/local-docker.env build + docker compose -f ./docker-compose.yml -f ./docker-compose.uaa.yml --env-file ./services/local/docker.env build seed: ## (Re)Create seed data - docker compose run --rm app yarn create-dev-data + docker compose --env-file ./services/local/docker.env run --rm app yarn create-dev-data set-pipeline: ## Set Concourse `web` pipeline fly -t pages-staging sp -p web -c ci/pipeline.yml start: ## Start - docker compose -f ./docker-compose.yml -f ./docker-compose.uaa.yml --env-file ./services/local/local-docker.env up + docker compose -f ./docker-compose.yml -f ./docker-compose.uaa.yml --env-file ./services/local/docker.env up start-workers: ## Start with workers - docker compose -f ./docker-compose.yml -f ./docker-compose.uaa.yml --env-file ./services/local/local-docker.env up + docker compose -f ./docker-compose.yml -f ./docker-compose.uaa.yml --env-file ./services/local/docker.env up test-client: ## Run client tests - docker compose run --rm app yarn test:client + docker compose --env-file ./services/local/docker.env run --rm app yarn test:rtl test-server: ## Run server tests - docker compose run --rm app yarn test:server + docker compose --env-file ./services/local/docker.env run --rm app yarn test:server test-all: ## Run all tests - docker compose run --rm app yarn test + docker compose --env-file ./services/local/docker.env run --rm app yarn test everything: # When you switch to a new branch and need to rebuild everything - docker compose -f ./docker-compose.yml -f ./docker-compose.uaa.yml --env-file ./services/local/local-docker.env down + docker compose -f ./docker-compose.yml -f ./docker-compose.uaa.yml --env-file ./services/local/docker.env down make rebuild make install make migrate diff --git a/docs/DEVELOPMENT.md b/docs/DEVELOPMENT.md index 484646e0e..4d6c2ada3 100644 --- a/docs/DEVELOPMENT.md +++ b/docs/DEVELOPMENT.md @@ -63,16 +63,16 @@ If local UAA authentication is not needed, Docker can be set up and started with 1. Run `docker compose build`. 1. Run `docker compose run --rm app yarn` to install dependencies. 1. Run `docker compose run --rm admin-client yarn` to install dependencies. -1. Run `docker compose run --rm app yarn migrate:up` to initialize the local database. -1. Run `docker compose run --rm app yarn create-dev-data` to create some fake development data for your local database. +1. Run `docker compose --env-file ./services/local/docker.env run --rm app yarn migrate:up` to initialize the local database. +1. Run `docker compose --env-file ./services/local/docker.env run --rm app yarn create-dev-data` to create some fake development data for your local database. 1. Run `docker compose up` to start the development environment. Any time the node dependencies are changed (like from a recently completed new feature), `docker compose run --rm app yarn` will need to be re-run to install updated dependencies after pulling the new code from GitHub. In order to make it possible to log in with local UAA authentication in a development environment it is necessary to also build and start the UAA container, which requires specifying a second docker compose configuration file when executing the docker compose commands which build containers or start the development environment, e.g.: -1. `docker compose -f ./docker-compose.yml -f ./docker-compose.uaa.yml --env-file ./services/local/local-docker.env build` -1. `docker compose -f ./docker-compose.yml -f ./docker-compose.uaa.yml --env-file ./services/local/local-docker.env up` +1. `docker compose -f ./docker-compose.yml -f ./docker-compose.uaa.yml --env-file ./services/local/docker.env build` +1. `docker compose -f ./docker-compose.yml -f ./docker-compose.uaa.yml --env-file ./services/local/docker.env up` #### Check to see if everything is working correctly @@ -85,7 +85,7 @@ In our Docker Compose environment, `app` is the name of the container where the For example: -- Use `docker compose run --rm app yarn test` to run local testing on the app. +- Use `docker compose --env-file ./services/local/docker.env build run --rm app yarn test` to run local testing on the app. - Use `docker compose run --rm app yarn lint` to check that your local changes meet our linting standards. - Use `docker compose run --rm app yarn format` to format your local changes based on our standards. @@ -315,11 +315,11 @@ docker compose run --rm app yarn test You can also just run back or front end tests via: ```sh -docker compose run --rm app yarn test:server # for all back end tests -docker compose run --rm app yarn test:server:file ./test/api/ # to run a single back end test file -docker compose run --rm app yarn test:client # for all front end tests -docker compose run --rm app yarn test:client:watch # to watch and re-run front end tests -docker compose run --rm app yarn test:client:file ./test/frontend/ # to run a single front end test file +docker compose --env-file ./services/local/docker.env build run --rm app yarn test:server # for all back end tests +docker compose --env-file ./services/local/docker.env build run --rm app yarn test:server:file ./test/api/ # to run a single back end test file +docker compose --env-file ./services/local/docker.env build run --rm app yarn test:rtl # for all front end tests +docker compose --env-file ./services/local/docker.env build run --rm app yarn test:rtl:watch # to watch and re-run front end tests +docker compose --env-file ./services/local/docker.env build run --rm app yarn test:rtl:file ./test/frontend/ # to run a single front end test file ``` To view coverage reports as HTML: diff --git a/package.json b/package.json index 0c6c3a341..1690db13c 100644 --- a/package.json +++ b/package.json @@ -108,7 +108,7 @@ "export:sites": "node ./scripts/exportSitesAsCsv.js", "serve-coverage": "serve -n -l 8080 ./coverage", "update-local-config": "node ./scripts/update-local-config.js", - "create-dev-data": "NODE_ENV=development DOTENV_CONFIG_PATH=./services/local/local-docker.env node -r dotenv/config ./scripts/create-dev-data.js", + "create-dev-data": "NODE_ENV=development DOTENV_CONFIG_PATH=./services/local/docker.env node -r dotenv/config ./scripts/create-dev-data.js", "update-proxy-db": "node ./scripts/proxy-edge.js", "archive-build-logs": "node ./scripts/archive-build-logs.js", "migrate-build-logs": "node ./scripts/migrate-build-logs.js", @@ -123,7 +123,7 @@ "create-test-users": "DOTENV_CONFIG_PATH=.env node -r dotenv/config ./scripts/create-test-users.js", "remove-test-users": "DOTENV_CONFIG_PATH=.env node -r dotenv/config ./scripts/remove-test-users.js", "run-scans-for-build": "node ./scripts/run-scans-for-build.js", - "mock-cf-api": "DOTENV_CONFIG_PATH=./services/local/local-docker.env node -r dotenv/config --watch-path=./services/local/mock-cf-api ./services/local/mock-cf-api/index.js" + "mock-cf-api": "DOTENV_CONFIG_PATH=./services/local/docker.env node -r dotenv/config --watch-path=./services/local/mock-cf-api ./services/local/mock-cf-api/index.js" }, "main": "index.js", "repository": { diff --git a/scripts/create-dev-data.js b/scripts/create-dev-data.js index 4839e0751..7db38f584 100644 --- a/scripts/create-dev-data.js +++ b/scripts/create-dev-data.js @@ -23,6 +23,8 @@ const { UserAction, } = require('../api/models'); const { site: siteFactory } = require('../test/api/support/factory'); +const { cleanFileStorage, runFileStorageSeed } = require('./local/populate-file-storage'); +const fileStorageStructure = require('../services/local/file-storage/data-structure'); const localSiteBuildTasks = []; const localSiteBuildTasksFile = path.join( @@ -154,6 +156,15 @@ async function addSiteToOrg(site, org) { return site; } +// Get Site Bucket Names +const siteServicesStrings = process.env.SITES_SERVICE_NAMES; + +if (!siteServicesStrings) { + throw 'The SITES_SERVICE_NAMES env var is not defined'; +} + +const serviceList = siteServicesStrings.split(','); + /** ***************************************** * Where the magic happens!! */ @@ -161,6 +172,9 @@ async function createData() { console.log('Cleaning database...'); await cleanDatabase(); + console.log('Cleaning file storage buckets...'); + await Promise.all(serviceList.map((bucket) => cleanFileStorage(bucket))); + /** ***************************************** * Action Types */ @@ -337,13 +351,6 @@ async function createData() { * Sites */ console.log('Creating sites...'); - const siteServicesStrings = process.env.SITES_SERVICE_NAMES; - - if (!siteServicesStrings) { - throw 'The SITES_SERVICE_NAMES env var is not defined'; - } - - const serviceList = siteServicesStrings.split(','); const [site1, nodeSite, goSite, goSite2, nodeSite2] = await Promise.all([ siteFactory({ @@ -407,7 +414,7 @@ async function createData() { * File Storage Services */ console.log('Creating file storage services...'); - await Promise.all([ + const [fs1, fs2, fs3, fs4] = await Promise.all([ FileStorageService.create({ siteId: nodeSite.id, organizationId: agency1.id, @@ -438,6 +445,16 @@ async function createData() { }), ]); + // Seed file storage + // *Note: file storage service name is the same as the bucket name locally + + await Promise.all([ + runFileStorageSeed(fs1.id, user1.id, fs1.serviceName, fileStorageStructure), + runFileStorageSeed(fs2.id, user1.id, fs2.serviceName, fileStorageStructure), + runFileStorageSeed(fs3.id, user3.id, fs3.serviceName, fileStorageStructure), + runFileStorageSeed(fs4.id, user4.id, fs4.serviceName, fileStorageStructure), + ]); + /** ***************************************** * Builds */ diff --git a/scripts/local/minio-bootstrap.sh b/scripts/local/minio-bootstrap.sh index c122a312e..43cbe7f5a 100755 --- a/scripts/local/minio-bootstrap.sh +++ b/scripts/local/minio-bootstrap.sh @@ -13,7 +13,7 @@ build_buckets() { local bucket_exists="Your previous request to create the named bucket succeeded and you already own it" # Load the .env file - local env_file="/app/services/local/local-docker.env" + local env_file="/app/services/local/docker.env" if [ -f "$env_file" ]; then while IFS='=' read -r key value; do diff --git a/scripts/local/populate-file-storage.js b/scripts/local/populate-file-storage.js new file mode 100644 index 000000000..5f56c1ad9 --- /dev/null +++ b/scripts/local/populate-file-storage.js @@ -0,0 +1,190 @@ +const fs = require('node:fs/promises'); +const crypto = require('node:crypto'); +const path = require('node:path'); +const { ListObjectsV2Command, DeleteObjectsCommand } = require('@aws-sdk/client-s3'); +const { FileStorageFile, FileStorageUserAction } = require('../../api/models'); +const S3Helper = require('../../api/services/S3Helper'); +const { slugify } = require('../../api/utils'); + +class SeedFileStorage { + constructor(fileStorageServiceId, userId, bucket) { + this.fileStorageServiceId = fileStorageServiceId; + this.userId = userId; + this.bucket = bucket; + this.s3 = new S3Helper.S3Client({ + accessKeyId: process.env.MINIO_ROOT_USER, + secretAccessKey: process.env.MINIO_ROOT_PASSWORD, + bucket, + region: 'auto', + }); + } + + async seedFileStorage(dirArray, parent = '.') { + return Promise.all( + dirArray.map(async (item) => { + if (item.type === 'directory') { + const key = path.join(parent, item.name); + await this.createDirectory(item.name, key); + + if (item?.children?.length > 0) { + return this.seedFileStorage(item.children, key); + } + } + + if (item.type === 'generated') { + const promises = Array.from({ length: item.number }, () => + this.generateFile(parent), + ); + + return Promise.all(promises); + } + + if (item.type === 'file') { + const filePath = path.join(__dirname, item.path); + const file = await fs.readFile(filePath); + const key = path.join(parent, slugify(item.name)); + const fileStat = await fs.stat(filePath); + const ext = path.extname(filePath); + const type = this.#getMimeType(ext); + const metadata = { size: fileStat.size }; + + return this.createFile(item.name, file, type, key, metadata); + } + }), + ); + } + + async createDirectory(name, key) { + const dirBuffer = Buffer.from(''); + await this.s3.putObject(dirBuffer, key); + + const fsf = await FileStorageFile.create({ + name, + key, + type: 'directory', + fileStorageServiceId: this.fileStorageServiceId, + description: 'directory', + }); + + await FileStorageUserAction.create({ + userId: this.userId, + fileStorageServiceId: this.fileStorageServiceId, + fileStorageFileId: fsf.id, + method: FileStorageUserAction.METHODS.POST, + description: FileStorageUserAction.ACTION_TYPES.CREATE_DIRECTORY, + }); + } + + async createFile(name, fileBuffer, type, key, metadata = {}) { + await this.s3.putObject(fileBuffer, key); + + const fsf = await FileStorageFile.create({ + name, + key, + type: type, + metadata, + fileStorageServiceId: this.fileStorageServiceId, + }); + + await FileStorageUserAction.create({ + userId: this.userId, + fileStorageServiceId: this.fileStorageServiceId, + fileStorageFileId: fsf.id, + method: FileStorageUserAction.METHODS.POST, + description: FileStorageUserAction.ACTION_TYPES.UPLOAD_FILE, + }); + + return fsf; + } + + async generateFile(parent) { + const name = this.#generateRandomString(10); + const fileName = `${name}.txt`; + const type = 'text/plain'; + const key = path.join(parent, fileName); + const fileBuffer = this.#generateRandomString(500); + const size = this.#generateRandomNumber(10000, 100000); + const metadata = { + size, + }; + + await this.createFile(fileName, fileBuffer, type, key, metadata); + } + + #generateRandomNumber(min, max) { + return Math.floor(Math.random() * (max - min + 1)) + min; + } + + #generateRandomString(len) { + return crypto.randomBytes(len).toString('hex').slice(0, len); + } + + #getMimeType(ext) { + if (ext === 'pdf') 'application/pdf'; + if (ext === 'png') return 'image/png'; + + return 'application/octet-stream'; + } +} + +async function runFileStorageSeed(fileStorageServiceId, userId, bucket, fileStructure) { + const seed = new SeedFileStorage(fileStorageServiceId, userId, bucket); + + await seed.seedFileStorage(fileStructure); +} + +async function cleanFileStorage(bucket) { + const s3 = new S3Helper.S3Client({ + accessKeyId: process.env.MINIO_ROOT_USER, + secretAccessKey: process.env.MINIO_ROOT_PASSWORD, + bucket, + region: 'auto', + }); + + async function emptyBucket() { + try { + // Step 1: List objects in the bucket + const listParams = { + Bucket: bucket, + }; + + const listedObjects = await s3.client.send(new ListObjectsV2Command(listParams)); + + if (!listedObjects.Contents) { + return; + } + + if (listedObjects?.Contents?.length === 0) { + return; + } + + // Step 2: Prepare objects for deletion + const objectsToDelete = listedObjects.Contents.map((object) => ({ + Key: object.Key, + })); + + // Step 3: Delete objects + const deleteParams = { + Bucket: bucket, + Delete: { + Objects: objectsToDelete, + Quiet: false, + }, + }; + + await s3.client.send(new DeleteObjectsCommand(deleteParams)); + + // If there are more objects, you may need to repeat the process + if (listedObjects.IsTruncated) { + await emptyBucket(); // Recursively call to delete all objects + } + } catch (err) { + // eslint-disable-next-line no-console + console.error('Error emptying bucket:', err); + } + } + + await emptyBucket(); +} + +module.exports = { cleanFileStorage, runFileStorageSeed, SeedFileStorage }; diff --git a/services/local/local-docker.env b/services/local/docker.env similarity index 100% rename from services/local/local-docker.env rename to services/local/docker.env diff --git a/services/local/file-storage/cloudgov-customers.pdf b/services/local/file-storage/cloudgov-customers.pdf new file mode 100644 index 000000000..c64da29e5 Binary files /dev/null and b/services/local/file-storage/cloudgov-customers.pdf differ diff --git a/services/local/file-storage/cloudgov-overview.pdf b/services/local/file-storage/cloudgov-overview.pdf new file mode 100644 index 000000000..4ffda444a Binary files /dev/null and b/services/local/file-storage/cloudgov-overview.pdf differ diff --git a/services/local/file-storage/data-structure.js b/services/local/file-storage/data-structure.js new file mode 100644 index 000000000..ecbd754de --- /dev/null +++ b/services/local/file-storage/data-structure.js @@ -0,0 +1,69 @@ +module.exports = [ + { + name: '~assets/', + type: 'directory', + children: [ + { + type: 'generated', + number: 100, + }, + { + name: 'images/', + type: 'directory', + children: [ + { + type: 'file', + name: 'pages-screenshot.png', + path: '../../services/local/file-storage/pages-screenshot.png', + }, + { + type: 'file', + name: 'PagesSettings.png', + path: '../../services/local/file-storage/pages-site-settings.png', + }, + ], + }, + { + name: 'pdfs/', + type: 'directory', + children: [ + { + type: 'file', + name: 'cloudgov-customers.pdf', + path: '../../services/local/file-storage/cloudgov-customers.pdf', + }, + { + type: 'file', + name: 'Cloudgov Overview.pdf', + path: '../../services/local/file-storage/cloudgov-overview.pdf', + }, + ], + }, + { + name: 'dir-1/', + type: 'directory', + children: [ + { + name: 'subdir/', + type: 'directory', + children: [ + { + type: 'generated', + number: 10, + }, + ], + }, + { + name: 'subdir-empty/', + type: 'directory', + children: [], + }, + { + type: 'generated', + number: 100, + }, + ], + }, + ], + }, +]; diff --git a/services/local/file-storage/pages-screenshot.png b/services/local/file-storage/pages-screenshot.png new file mode 100644 index 000000000..d774d338d Binary files /dev/null and b/services/local/file-storage/pages-screenshot.png differ diff --git a/services/local/file-storage/pages-site-settings.png b/services/local/file-storage/pages-site-settings.png new file mode 100644 index 000000000..03098b7e5 Binary files /dev/null and b/services/local/file-storage/pages-site-settings.png differ