Add queue to delete site builds older than 180 days

OPERATIONS.md

@@ -33,6 +33,15 @@ $ cf run-task pages-<env> --command "yarn migrate-site-repo 1 user@agency.gov ag
 $ cf logs --recent pages-<env>
 ```
 
+## Queues
+
+### Nightly deletion of older builds
+
+This is a job that runs nightly and removes all builds older than 180 days, based on `completedAt` timestamps.
+
+Queue name: `delete-older-builds`
+Queue worker: [`deleteOlderBuilds.js`](./api/workers/jobProcessors/deleteOlderBuilds.js)
+
 ## CI
 
 ### Nightly site bucket key rotations

api/bull-board/app.js

@@ -11,6 +11,7 @@ const slowDown = require('express-slow-down');
 const {
   ArchiveBuildLogsQueue,
   BuildTasksQueue,
+  DeleteOlderBuildsQueue,
   DomainQueue,
   FailStuckBuildsQueue,
   MailQueue,
@@ -40,6 +41,7 @@ createBullBoard({
     new BullAdapter(new SiteBuildQueue(connection)),
     new BullMQAdapter(new ArchiveBuildLogsQueue(connection)),
     new BullMQAdapter(new BuildTasksQueue(connection)),
+    new BullMQAdapter(new DeleteOlderBuildsQueue(connection)),
     new BullMQAdapter(new DomainQueue(connection)),
     new BullMQAdapter(new FailStuckBuildsQueue(connection)),
     new BullMQAdapter(new MailQueue(connection)),

api/models/build.js

@@ -329,6 +329,7 @@ module.exports = (sequelize, DataTypes) => {
         afterCreate,
         afterUpdate,
       },
+      paranoid: true,
     }
   );
 

api/queues/DeleteOlderBuildsQueue.js

@@ -0,0 +1,14 @@
+const { Queue } = require('bullmq');
+
+const DeleteOlderBuildsQueueName = 'delete-older-builds';
+
+class DeleteOlderBuildsQueue extends Queue {
+  constructor(connection) {
+    super(DeleteOlderBuildsQueueName, { connection });
+  }
+}
+
+module.exports = {
+  DeleteOlderBuildsQueue,
+  DeleteOlderBuildsQueueName,
+};

api/queues/index.js

@@ -1,4 +1,5 @@
 const { ArchiveBuildLogsQueue, ArchiveBuildLogsQueueName } = require('./ArchiveBuildLogsQueue');
+const { DeleteOlderBuildsQueue, DeleteOlderBuildsQueueName } = require('./DeleteOlderBuildsQueue');
 const { BuildTasksQueue, BuildTasksQueueName } = require('./BuildTasksQueue');
 const { DomainQueue, DomainQueueName } = require('./DomainQueue');
 const { FailStuckBuildsQueue, FailStuckBuildsQueueName } = require('./FailStuckBuildsQueue');
@@ -15,6 +16,8 @@ module.exports = {
   ArchiveBuildLogsQueueName,
   BuildTasksQueue,
   BuildTasksQueueName,
+  DeleteOlderBuildsQueue,
+  DeleteOlderBuildsQueueName,
   DomainQueue,
   DomainQueueName,
   FailStuckBuildsQueue,

api/workers/index.js

@@ -16,6 +16,8 @@ const {
   ArchiveBuildLogsQueueName,
   BuildTasksQueue,
   BuildTasksQueueName,
+  DeleteOlderBuildsQueue,
+  DeleteOlderBuildsQueueName,
   DomainQueueName,
   FailStuckBuildsQueue,
   FailStuckBuildsQueueName,
@@ -62,6 +64,8 @@ function pagesWorker(connection) {
 
   const buildTasksProcessor = job => Processors.buildTaskRunner(job);
 
+  const deleteOlderBuildsProcessor = job => Processors.deleteOlderBuilds(job);
+
   const failBuildsProcessor = job => Processors.failStuckBuilds(job);
 
   const siteDeletionProcessor = job => Processors.destroySiteInfra(job.data);
@@ -85,6 +89,7 @@ function pagesWorker(connection) {
     new QueueWorker(ArchiveBuildLogsQueueName, connection,
       path.join(__dirname, 'jobProcessors', 'archiveBuildLogsDaily.js')),
     new QueueWorker(BuildTasksQueueName, connection, buildTasksProcessor),
+    new QueueWorker(DeleteOlderBuildsQueueName, connection, deleteOlderBuildsProcessor),
     new QueueWorker(DomainQueueName, connection, domainJobProcessor),
     new QueueWorker(FailStuckBuildsQueueName, connection, failBuildsProcessor),
     new QueueWorker(MailQueueName, connection, mailJobProcessor),
@@ -97,23 +102,24 @@ function pagesWorker(connection) {
 
   const archiveBuildLogsQueue = new ArchiveBuildLogsQueue(connection);
   const buildTasksQueue = new BuildTasksQueue(connection);
+  const deleteOlderBuildsQueue = new DeleteOlderBuildsQueue(connection);
   const failStuckBuildsQueue = new FailStuckBuildsQueue(connection);
   const nightlyBuildsQueue = new NightlyBuildsQueue(connection);
   const scheduledQueue = new ScheduledQueue(connection);
   const timeoutBuildTasksQueue = new TimeoutBuildTasksQueue(connection);
   const queues = [
     archiveBuildLogsQueue,
     buildTasksQueue,
+    deleteOlderBuildsQueue,
     failStuckBuildsQueue,
     nightlyBuildsQueue,
     scheduledQueue,
     timeoutBuildTasksQueue,
   ];
 
   const jobs = () => Promise.all([
-    appEnv === 'production'
-      ? archiveBuildLogsQueue.add('archiveBuildLogsDaily', {}, nightlyJobConfig)
-      : Promise.resolve(),
+    archiveBuildLogsQueue.add('archiveBuildLogsDaily', {}, nightlyJobConfig),
+    deleteOlderBuildsQueue.add('deleteOlderBuilds', {}, nightlyJobConfig),
     failStuckBuildsQueue.add('failStuckBuilds', {}, everyTenMinutesJobConfig),
     nightlyBuildsQueue.add('nightlyBuilds', {}, nightlyJobConfig),
     scheduledQueue.add('sandboxNotifications', {}, nightlyJobConfig),

api/workers/jobProcessors/deleteOlderBuilds.js

@@ -0,0 +1,37 @@
+const moment = require('moment');
+const { Op } = require('sequelize');
+const { Build } = require('../../models');
+const Mailer = require('../../services/mailer');
+const Slacker = require('../../services/slacker');
+const { createJobLogger } = require('./utils');
+
+async function deleteOlderBuilds(job) {
+  const logger = createJobLogger(job);
+  const cutoffDate = moment().subtract(180, 'days').startOf('day');
+
+  logger.log(`Deleting all builds completed before ${cutoffDate}.`);
+
+  try {
+    const numDeleted = await Build.destroy({
+      where: {
+        completedAt: {
+          [Op.lt]: cutoffDate.toDate(),
+        },
+      },
+    });
+    logger.log(`Deleted ${numDeleted} builds.`);
+    await logger.flush();
+    return 'OK';
+  } catch (error) {
+    const errMsg = `Delete builds before ${cutoffDate.format('YYYY-MM-DD')} completed with error`;
+    logger.log(errMsg, error);
+    await logger.flush();
+    Mailer.init();
+    Slacker.init();
+    Mailer.sendAlert(errMsg, error);
+    Slacker.sendAlert(errMsg, error);
+    throw new Error(errMsg);
+  }
+}
+
+module.exports = deleteOlderBuilds;

api/workers/jobProcessors/index.js

@@ -1,5 +1,6 @@
 const archiveBuildLogsDaily = require('./archiveBuildLogsDaily');
 const buildTaskRunner = require('./buildTaskRunner');
+const deleteOlderBuilds = require('./deleteOlderBuilds');
 const destroySiteInfra = require('./destroySiteInfra');
 const failStuckBuilds = require('./failStuckBuilds');
 const multiJobProcessor = require('./multiJobProcessor');
@@ -11,6 +12,7 @@ const cleanSandboxOrganizations = require('./cleanSandboxOrganizations');
 module.exports = {
   archiveBuildLogsDaily,
   buildTaskRunner,
+  deleteOlderBuilds,
   destroySiteInfra,
   failStuckBuilds,
   multiJobProcessor,

frontend/components/site/siteBuilds.jsx

@@ -158,7 +158,7 @@ function SiteBuilds() {
   }, []);
 
   if (!builds.isLoading && !builds.data.length) {
-    const header = 'This site does not yet have any builds.';
+    const header = 'This site has not had any builds in the past 180 days.';
     const message = 'If this site was just added, the first build should be available within a few minutes.';
     return (
       <AlertBanner status="info" header={header} message={message}>

migrations/20240103173046-add-paranoid-to-build.js

@@ -0,0 +1,9 @@
+const TABLE = 'build';
+
+exports.up = async db => {
+  await db.addColumn(TABLE, 'deletedAt', { type: 'date', allowNull: true });
+};
+
+exports.down = async db => {
+  db.removeColumn(TABLE, 'deletedAt');
+};

test/api/workers/jobProcessors.test.js

@@ -1,6 +1,7 @@
 const { expect } = require('chai');
 const sinon = require('sinon');
 const moment = require('moment');
+const { Build } = require('../../../api/models');
 const BuildLogs = require('../../../api/services/build-logs');
 const TimeoutBuilds = require('../../../api/services/TimeoutBuilds');
 const NightlyBuildsHelper = require('../../../api/services/NightlyBuildsHelper');
@@ -86,6 +87,78 @@ describe('job processors', () => {
     });
   });
 
+  context('deleteOlderBuilds', () => {
+    it('handles successful deletion properly', async () => {
+      sinon.stub(Build, 'destroy').resolves();
+      const result = await jobProcessors.deleteOlderBuilds(job);
+      expect(result).to.not.be.an('error');
+    });
+
+    it('handles failed deletion properly', async () => {
+      sinon.stub(Build, 'destroy').rejects('nope');
+      const result = await jobProcessors.deleteOlderBuilds(job).catch(e => e);
+      expect(result).to.be.an('error');
+      const cutoffDate = moment().subtract(180, 'days').startOf('day');
+
+      expect(result.message.split(',')[0]).to
+        .equal(`Delete builds before ${cutoffDate.format('YYYY-MM-DD')} completed with error`);
+    });
+
+    it('deletes the correct builds', async () => {
+      const newerCompletedAt = moment().subtract(179, 'days').startOf('day');;
+      const newerBuild1 = await factory.build({ completedAt: newerCompletedAt });
+      const newerBuild2 = await factory.build({ completedAt: newerCompletedAt });
+
+      const olderCompletedAt = moment().subtract(181, 'days').startOf('day');;
+      const olderBuild1 = await factory.build({ completedAt: olderCompletedAt });
+      const olderBuild2 = await factory.build({ completedAt: olderCompletedAt });
+
+      const buildsBefore = await Build.count();
+      const result = await jobProcessors.deleteOlderBuilds(job);
+      expect(result).to.not.be.an('error');
+
+      const buildsAfter = await Build.count();
+      const buildsDeleted = buildsBefore - buildsAfter;
+      expect(buildsDeleted).to.equal(2);
+
+      const foundNewerBuild1 =  await Build.findOne( { where: { id: newerBuild1.id } });
+      const foundNewerBuild2 =  await Build.findOne( { where: { id: newerBuild2.id } });
+      expect(foundNewerBuild1).to.not.be.null;
+      expect(foundNewerBuild2).to.not.be.null;
+
+      const foundOlderBuild1 =  await Build.findOne( { where: { id: olderBuild1.id } });
+      const foundOlderBuild2 =  await Build.findOne( { where: { id: olderBuild2.id } });
+      expect(foundOlderBuild1).to.be.null;
+      expect(foundOlderBuild2).to.be.null;
+
+      Build.truncate({ force: true, cascade: true });
+    });
+
+    it('deletes nothing when there are no older builds', async () => {
+      const completedAt = moment().subtract(30, 'days').startOf('day');;
+      await factory.build({ completedAt });
+      await factory.build({ completedAt });
+      await factory.build({ completedAt });
+
+      const buildsBefore = await Build.count();
+      const result = await jobProcessors.deleteOlderBuilds(job);
+      expect(result).to.not.be.an('error');
+
+      const buildsAfter = await Build.count();
+      const buildsDeleted = buildsBefore - buildsAfter;
+      expect(buildsDeleted).to.equal(0);
+
+      Build.truncate({ force: true, cascade: true });
+    });
+
+    it('runs harmlessly when there are no builds at all', async () => {
+      Build.truncate({ force: true, cascade: true });
+
+      const result = await jobProcessors.deleteOlderBuilds(job);
+      expect(result).to.not.be.an('error');
+    });
+  });
+
   context('sandboxNotifications', () => {
     it('failed to notify all sandbox organization members', async () => {
       sinon.stub(SandboxHelper, 'notifyOrganizations').resolves([

test/frontend/components/site/siteBuilds.test.js

@@ -176,7 +176,7 @@ describe('<SiteBuilds/>', () => {
     const wrapper = mountRouter(<SiteBuilds />, '/site/:id/builds', '/site/5/builds', state);
 
     expect(wrapper.find('table')).to.have.length(0);
-    expect(wrapper.find('AlertBanner').prop('header')).to.equal('This site does not yet have any builds.');
+    expect(wrapper.find('AlertBanner').prop('header')).to.equal('This site has not had any builds in the past 180 days.');
     expect(wrapper.find('AlertBanner').prop('message')).to.equal(
       'If this site was just added, the first build should be available within a few minutes.'
     );