chore: Add file storage seeding to create-dev-data #4733

Makefile

@@ -34,44 +34,44 @@ lint-client: ## Lint admin client code
 lint: lint-server lint-client ## Lint project
 
 lint-fix: ## lint and fix
-	docker compose run --rm app yarn format:lint
+	docker compose --env-file ./services/local/docker.env run --rm app yarn format:lint
 
 format:
-	docker compose run --rm app yarn format
+	docker compose --env-file ./services/local/docker.env run --rm app yarn format
 
 format-check:
-	docker compose run --rm app yarn format:check
+	docker compose --env-file ./services/local/docker.env run --rm app yarn format:check
 
 migrate: ## Run database migrations
-	docker compose run --rm app yarn migrate:up
+	docker compose --env-file ./services/local/docker.env run  --rm app yarn migrate:up
 
 rebuild: ## Rebuild docker images and database volumes
 	docker volume rm pages-core_db-data
-	docker compose -f ./docker-compose.yml -f ./docker-compose.uaa.yml --env-file ./services/local/local-docker.env build
+	docker compose -f ./docker-compose.yml -f ./docker-compose.uaa.yml --env-file ./services/local/docker.env build
 
 seed: ## (Re)Create seed data
-	docker compose run --rm app yarn create-dev-data
+	docker compose --env-file ./services/local/docker.env run --rm app yarn create-dev-data
 
 set-pipeline: ## Set Concourse `web` pipeline
 	fly -t pages-staging sp -p web -c ci/pipeline.yml
 
 start: ## Start
-	docker compose -f ./docker-compose.yml -f ./docker-compose.uaa.yml --env-file ./services/local/local-docker.env up
+	docker compose -f ./docker-compose.yml -f ./docker-compose.uaa.yml --env-file ./services/local/docker.env up
 
 start-workers: ## Start with workers
-	docker compose -f ./docker-compose.yml -f ./docker-compose.uaa.yml --env-file ./services/local/local-docker.env up
+	docker compose -f ./docker-compose.yml -f ./docker-compose.uaa.yml --env-file ./services/local/docker.env up
 
 test-client: ## Run client tests
-	docker compose run --rm app yarn test:client
+	docker compose --env-file ./services/local/docker.env  run --rm app yarn test:rtl
 
 test-server: ## Run server tests
-	docker compose run --rm app yarn test:server
+	docker compose --env-file ./services/local/docker.env  run --rm app yarn test:server
 
 test-all: ## Run all tests
-	docker compose run --rm app yarn test
+	docker compose --env-file ./services/local/docker.env run --rm app yarn test
 
 everything: # When you switch to a new branch and need to rebuild everything
-	docker compose -f ./docker-compose.yml -f ./docker-compose.uaa.yml --env-file ./services/local/local-docker.env down
+	docker compose -f ./docker-compose.yml -f ./docker-compose.uaa.yml --env-file ./services/local/docker.env down
 	make rebuild
 	make install
 	make migrate

docs/DEVELOPMENT.md

@@ -63,16 +63,16 @@ If local UAA authentication is not needed, Docker can be set up and started with
 1. Run `docker compose build`.
 1. Run `docker compose run --rm app yarn` to install dependencies.
 1. Run `docker compose run --rm admin-client yarn` to install dependencies.
-1. Run `docker compose run --rm app yarn migrate:up` to initialize the local database.
-1. Run `docker compose run --rm app yarn create-dev-data` to create some fake development data for your local database.
+1. Run `docker compose --env-file ./services/local/docker.env run --rm app yarn migrate:up` to initialize the local database.
+1. Run `docker compose --env-file ./services/local/docker.env run --rm app yarn create-dev-data` to create some fake development data for your local database.
 1. Run `docker compose up` to start the development environment.
 
 Any time the node dependencies are changed (like from a recently completed new feature), `docker compose run --rm app yarn` will need to be re-run to install updated dependencies after pulling the new code from GitHub.
 
 In order to make it possible to log in with local UAA authentication in a development environment it is necessary to also build and start the UAA container, which requires specifying a second docker compose configuration file when executing the docker compose commands which build containers or start the development environment, e.g.:
 
-1. `docker compose -f ./docker-compose.yml -f ./docker-compose.uaa.yml --env-file ./services/local/local-docker.env build`
-1. `docker compose -f ./docker-compose.yml -f ./docker-compose.uaa.yml --env-file ./services/local/local-docker.env up`
+1. `docker compose -f ./docker-compose.yml -f ./docker-compose.uaa.yml --env-file ./services/local/docker.env build`
+1. `docker compose -f ./docker-compose.yml -f ./docker-compose.uaa.yml --env-file ./services/local/docker.env up`
 
 #### Check to see if everything is working correctly
 
@@ -85,7 +85,7 @@ In our Docker Compose environment, `app` is the name of the container where the
 
 For example:
 
-- Use `docker compose run --rm app yarn test` to run local testing on the app.
+- Use `docker compose --env-file ./services/local/docker.env build run --rm app yarn test` to run local testing on the app.
 - Use `docker compose run --rm app yarn lint` to check that your local changes meet our linting standards.
 - Use `docker compose run --rm app yarn format` to format your local changes based on our standards.
 
@@ -315,11 +315,11 @@ docker compose run --rm app yarn test
 You can also just run back or front end tests via:
 
 ```sh
-docker compose run --rm app yarn test:server  # for all back end tests
-docker compose run --rm app yarn test:server:file ./test/api/<path/to/test.js> # to run a single back end test file
-docker compose run --rm app yarn test:client  # for all front end tests
-docker compose run --rm app yarn test:client:watch  # to watch and re-run front end tests
-docker compose run --rm app yarn test:client:file ./test/frontend/<path/to/test.js> # to run a single front end test file
+docker compose --env-file ./services/local/docker.env build run --rm app yarn test:server  # for all back end tests
+docker compose --env-file ./services/local/docker.env build run --rm app yarn test:server:file ./test/api/<path/to/test.js> # to run a single back end test file
+docker compose --env-file ./services/local/docker.env build run --rm app yarn test:rtl  # for all front end tests
+docker compose --env-file ./services/local/docker.env build run --rm app yarn test:rtl:watch  # to watch and re-run front end tests
+docker compose --env-file ./services/local/docker.env build run --rm app yarn test:rtl:file ./test/frontend/<path/to/test.js> # to run a single front end test file
 ```
 
 To view coverage reports as HTML:

package.json

@@ -108,7 +108,7 @@
     "export:sites": "node ./scripts/exportSitesAsCsv.js",
     "serve-coverage": "serve -n -l 8080 ./coverage",
     "update-local-config": "node ./scripts/update-local-config.js",
-    "create-dev-data": "NODE_ENV=development DOTENV_CONFIG_PATH=./services/local/local-docker.env node -r dotenv/config ./scripts/create-dev-data.js",
+    "create-dev-data": "NODE_ENV=development DOTENV_CONFIG_PATH=./services/local/docker.env node -r dotenv/config ./scripts/create-dev-data.js",
     "update-proxy-db": "node ./scripts/proxy-edge.js",
     "archive-build-logs": "node ./scripts/archive-build-logs.js",
     "migrate-build-logs": "node ./scripts/migrate-build-logs.js",
@@ -123,7 +123,7 @@
     "create-test-users": "DOTENV_CONFIG_PATH=.env node -r dotenv/config ./scripts/create-test-users.js",
     "remove-test-users": "DOTENV_CONFIG_PATH=.env node -r dotenv/config ./scripts/remove-test-users.js",
     "run-scans-for-build": "node ./scripts/run-scans-for-build.js",
-    "mock-cf-api": "DOTENV_CONFIG_PATH=./services/local/local-docker.env node -r dotenv/config --watch-path=./services/local/mock-cf-api ./services/local/mock-cf-api/index.js"
+    "mock-cf-api": "DOTENV_CONFIG_PATH=./services/local/docker.env node -r dotenv/config --watch-path=./services/local/mock-cf-api ./services/local/mock-cf-api/index.js"
   },
   "main": "index.js",
   "repository": {

scripts/create-dev-data.js

@@ -23,6 +23,8 @@ const {
   UserAction,
 } = require('../api/models');
 const { site: siteFactory } = require('../test/api/support/factory');
+const { cleanFileStorage, runFileStorageSeed } = require('./local/populate-file-storage');
+const fileStorageStructure = require('../services/local/file-storage/data-structure');
 
 const localSiteBuildTasks = [];
 const localSiteBuildTasksFile = path.join(
@@ -154,13 +156,25 @@ async function addSiteToOrg(site, org) {
   return site;
 }
 
+// Get Site Bucket Names
+const siteServicesStrings = process.env.SITES_SERVICE_NAMES;
+
+if (!siteServicesStrings) {
+  throw 'The SITES_SERVICE_NAMES env var is not defined';
+}
+
+const serviceList = siteServicesStrings.split(',');
+
 /** *****************************************
  *        Where the magic happens!!
  */
 async function createData() {
   console.log('Cleaning database...');
   await cleanDatabase();
 
+  console.log('Cleaning file storage buckets...');
+  await Promise.all(serviceList.map((bucket) => cleanFileStorage(bucket)));
+
   /** *****************************************
    *              Action Types
    */
@@ -337,13 +351,6 @@ async function createData() {
    *                 Sites
    */
   console.log('Creating sites...');
-  const siteServicesStrings = process.env.SITES_SERVICE_NAMES;
-
-  if (!siteServicesStrings) {
-    throw 'The SITES_SERVICE_NAMES env var is not defined';
-  }
-
-  const serviceList = siteServicesStrings.split(',');
 
   const [site1, nodeSite, goSite, goSite2, nodeSite2] = await Promise.all([
     siteFactory({
@@ -407,7 +414,7 @@ async function createData() {
    *           File Storage Services
    */
   console.log('Creating file storage services...');
-  await Promise.all([
+  const [fs1, fs2, fs3, fs4] = await Promise.all([
     FileStorageService.create({
       siteId: nodeSite.id,
       organizationId: agency1.id,
@@ -438,6 +445,16 @@ async function createData() {
     }),
   ]);
 
+  // Seed file storage
+  // *Note: file storage service name is the same as the bucket name locally
+
+  await Promise.all([
+    runFileStorageSeed(fs1.id, user1.id, fs1.serviceName, fileStorageStructure),
+    runFileStorageSeed(fs2.id, user1.id, fs2.serviceName, fileStorageStructure),
+    runFileStorageSeed(fs3.id, user3.id, fs3.serviceName, fileStorageStructure),
+    runFileStorageSeed(fs4.id, user4.id, fs4.serviceName, fileStorageStructure),
+  ]);
+
   /** *****************************************
    *                 Builds
    */

scripts/local/minio-bootstrap.sh

@@ -13,7 +13,7 @@ build_buckets() {
   local bucket_exists="Your previous request to create the named bucket succeeded and you already own it"
 
   # Load the .env file
-  local env_file="/app/services/local/local-docker.env"
+  local env_file="/app/services/local/docker.env"
 
   if [ -f "$env_file" ]; then
     while IFS='=' read -r key value; do

scripts/local/populate-file-storage.js

@@ -0,0 +1,190 @@
+const fs = require('node:fs/promises');
+const crypto = require('node:crypto');
+const path = require('node:path');
+const { ListObjectsV2Command, DeleteObjectsCommand } = require('@aws-sdk/client-s3');
+const { FileStorageFile, FileStorageUserAction } = require('../../api/models');
+const S3Helper = require('../../api/services/S3Helper');
+const { slugify } = require('../../api/utils');
+
+class SeedFileStorage {
+  constructor(fileStorageServiceId, userId, bucket) {
+    this.fileStorageServiceId = fileStorageServiceId;
+    this.userId = userId;
+    this.bucket = bucket;
+    this.s3 = new S3Helper.S3Client({
+      accessKeyId: process.env.MINIO_ROOT_USER,
+      secretAccessKey: process.env.MINIO_ROOT_PASSWORD,
+      bucket,
+      region: 'auto',
+    });
+  }
+
+  async seedFileStorage(dirArray, parent = '.') {
+    return Promise.all(
+      dirArray.map(async (item) => {
+        if (item.type === 'directory') {
+          const key = path.join(parent, item.name);
+          await this.createDirectory(item.name, key);
+
+          if (item?.children?.length > 0) {
+            return this.seedFileStorage(item.children, key);
+          }
+        }
+
+        if (item.type === 'generated') {
+          const promises = Array.from({ length: item.number }, () =>
+            this.generateFile(parent),
+          );
+
+          return Promise.all(promises);
+        }
+
+        if (item.type === 'file') {
+          const filePath = path.join(__dirname, item.path);
+          const file = await fs.readFile(filePath);
+          const key = path.join(parent, slugify(item.name));
+          const fileStat = await fs.stat(filePath);
+          const ext = path.extname(filePath);
+          const type = this.#getMimeType(ext);
+          const metadata = { size: fileStat.size };
+
+          return this.createFile(item.name, file, type, key, metadata);
+        }
+      }),
+    );
+  }
+
+  async createDirectory(name, key) {
+    const dirBuffer = Buffer.from('');
+    await this.s3.putObject(dirBuffer, key);
+
+    const fsf = await FileStorageFile.create({
+      name,
+      key,
+      type: 'directory',
+      fileStorageServiceId: this.fileStorageServiceId,
+      description: 'directory',
+    });
+
+    await FileStorageUserAction.create({
+      userId: this.userId,
+      fileStorageServiceId: this.fileStorageServiceId,
+      fileStorageFileId: fsf.id,
+      method: FileStorageUserAction.METHODS.POST,
+      description: FileStorageUserAction.ACTION_TYPES.CREATE_DIRECTORY,
+    });
+  }
+
+  async createFile(name, fileBuffer, type, key, metadata = {}) {
+    await this.s3.putObject(fileBuffer, key);
+
+    const fsf = await FileStorageFile.create({
+      name,
+      key,
+      type: type,
+      metadata,
+      fileStorageServiceId: this.fileStorageServiceId,
+    });
+
+    await FileStorageUserAction.create({
+      userId: this.userId,
+      fileStorageServiceId: this.fileStorageServiceId,
+      fileStorageFileId: fsf.id,
+      method: FileStorageUserAction.METHODS.POST,
+      description: FileStorageUserAction.ACTION_TYPES.UPLOAD_FILE,
+    });
+
+    return fsf;
+  }
+
+  async generateFile(parent) {
+    const name = this.#generateRandomString(10);
+    const fileName = `${name}.txt`;
+    const type = 'text/plain';
+    const key = path.join(parent, fileName);
+    const fileBuffer = this.#generateRandomString(500);
+    const size = this.#generateRandomNumber(10000, 100000);
+    const metadata = {
+      size,
+    };
+
+    await this.createFile(fileName, fileBuffer, type, key, metadata);
+  }
+
+  #generateRandomNumber(min, max) {
+    return Math.floor(Math.random() * (max - min + 1)) + min;
+  }
+
+  #generateRandomString(len) {
+    return crypto.randomBytes(len).toString('hex').slice(0, len);
+  }
+
+  #getMimeType(ext) {
+    if (ext === 'pdf') 'application/pdf';
+    if (ext === 'png') return 'image/png';
+
+    return 'application/octet-stream';
+  }
+}
+
+async function runFileStorageSeed(fileStorageServiceId, userId, bucket, fileStructure) {
+  const seed = new SeedFileStorage(fileStorageServiceId, userId, bucket);
+
+  await seed.seedFileStorage(fileStructure);
+}
+
+async function cleanFileStorage(bucket) {
+  const s3 = new S3Helper.S3Client({
+    accessKeyId: process.env.MINIO_ROOT_USER,
+    secretAccessKey: process.env.MINIO_ROOT_PASSWORD,
+    bucket,
+    region: 'auto',
+  });
+
+  async function emptyBucket() {
+    try {
+      // Step 1: List objects in the bucket
+      const listParams = {
+        Bucket: bucket,
+      };
+
+      const listedObjects = await s3.client.send(new ListObjectsV2Command(listParams));
+
+      if (!listedObjects.Contents) {
+        return;
+      }
+
+      if (listedObjects?.Contents?.length === 0) {
+        return;
+      }
+
+      // Step 2: Prepare objects for deletion
+      const objectsToDelete = listedObjects.Contents.map((object) => ({
+        Key: object.Key,
+      }));
+
+      // Step 3: Delete objects
+      const deleteParams = {
+        Bucket: bucket,
+        Delete: {
+          Objects: objectsToDelete,
+          Quiet: false,
+        },
+      };
+
+      await s3.client.send(new DeleteObjectsCommand(deleteParams));
+
+      // If there are more objects, you may need to repeat the process
+      if (listedObjects.IsTruncated) {
+        await emptyBucket(); // Recursively call to delete all objects
+      }
+    } catch (err) {
+      // eslint-disable-next-line no-console
+      console.error('Error emptying bucket:', err);
+    }
+  }
+
+  await emptyBucket();
+}
+
+module.exports = { cleanFileStorage, runFileStorageSeed, SeedFileStorage };