Merge pull request #680 from FreeFeed/media-files

New media processing pipeline

.github/workflows/checks.yml

@@ -29,10 +29,11 @@ jobs:
         with:
           redis-version: ${{ matrix.redis-version }}
 
-      - name: install GraphicsMagick
+      - name: Install media tools
         run: |
           sudo apt-get update
-          sudo apt-get install graphicsmagick
+          sudo apt-get install imagemagick
+          sudo apt-get install ffmpeg
 
       - uses: actions/checkout@v3
 
@@ -56,9 +57,6 @@ jobs:
       - name: create directories for attachments
         run: |
           mkdir -p /tmp/pepyatka-media/attachments
-          mkdir /tmp/pepyatka-media/attachments/thumbnails
-          mkdir /tmp/pepyatka-media/attachments/thumbnails2
-          mkdir /tmp/pepyatka-media/attachments/anotherTestSize
 
       - name: Install dependencies
         run: yarn

API_VERSIONS.md

@@ -5,6 +5,40 @@ All backward-incompatible FreeFeed API changes will be documented in this file.
 See the [About API versions](#about-api-versions) section in the end of this
 file for the general versioning information.
 
+## [4] - 2025-02-01
+### Changed
+- The attachment serialization is changed. The new format contains the following
+  fields:
+  - _id_ (string) - the UUID of the attachment
+  - _mediaType_ (string) - the media type of the attachment, one of 'image',
+    'video', 'audio', 'general'
+  - _fileName_ (string) - the original filename of the attachment
+  - _fileSize_ (number) - the size of the attachment's original file in bytes
+  - _previewTypes_ (array of string) - the array of available preview types of
+    the attachment, can be empty or contains the following values: 'image',
+    'video', 'audio'
+  - _meta_ (object) - optional field with temporary or not essential media
+    metadata (all fields are optional):
+    - _dc:title_: the audio/video title
+    - _dc:creator_: the audio/video author name
+    - _animatedImage_: true if the video was created from an animated image
+    - _silent_: true if the video has no audio track
+    - _inProgress_: true if the media file is currently being processed
+  - _width_ and _height_ (number) - the size of the original image/video file in
+    pixels, presents only for 'image' and 'video' attachments, and when the
+    processing is done
+  - _duration_ (number) - the duration of the audio/video file in seconds,
+    present only for 'audio' and 'video' attachments, and when the processing is
+    done
+  - _previewWidth_ and _previewHeight_ (number) - the size of the maximum
+    available image/video preview in pixels, presents only when different from
+    the _width_ and _height_
+  - _postId_ (string|null) - the UUID of the post to which the attachment is
+    attached
+  - _createdBy_ (string) - the UUID of the user who uploaded the attachment
+  - _createdAt_ (string) - the ISO 8601 datetime when the attachment was created
+  - _updatedAt_ (string) - the ISO 8601 datetime when the attachment was updated
+
 ## [3] - 2024-06-21
 
 ### Changed

CHANGELOG.md

@@ -6,6 +6,71 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
 ## [2.23.0] - Not released
+### Changed
+- The media files (attachments) handling algorithm has been changed. There are
+  four media types now: 'image', 'video', 'audio' and 'general'. Images are
+  accepted in JPEG, PNG, WebP, GIF, HEIC/HEIF and AVIF formats. Also now we
+  accept and process arbitrary formats of video and audio files (detected with
+  ffmpeg).
+
+  For the visual files (images and videos), the multiple preview sizes are
+  created, in addition to the legacy 'thumbnail' and 'thumbnail2'.
+
+  The animated GIF images are now treated as video files and the video previews
+  are created for them.
+
+  We don't keep the originals for the truly (not from animated images) video
+  files. After the preview creation, the largest preview is kept as the
+  'original'.
+
+  Some media files (the truly video ones for now) are processed asynchronously.
+  Right after they are uploaded to the server, the asynchronous job is
+  scheduled, and after the job finishes, the 'attachment:update' realtime event
+  is sent to the 'user:{ownerId}', 'attachment:{attachmentId}' and
+  'post:{postId}' (if the file is attached to a post) channels.
+
+  The `attachments` table now has a few new columns:
+  - `width` and `height`: size of the original image or video file in pixels
+    (null for non-visual files)
+  - `duration`: duration of the video or audio in seconds (null for non-playable
+    files)
+  - `previews`: JSON object with preview types and sizes, see the
+    _MediaPreviews_ type in the
+    [app/support/media-files/types.ts](app/support/media-files/types.ts) file.
+  - `meta`: JSON object with temporary or not essential media metadata. It can
+    contain the audio/video title and author name (in 'dc:title' and
+    'dc:creator' fields, respectively) and some special flags:
+    - `animatedImage`: true if the video was created from an animated image
+    - `silent`: true if the video has no audio track
+    - `inProgress`: true if the media file is currently being processed
+### Added
+- The new V4 API version is introduced, to support the new attachment features.
+  See the new serialized attachment type `SerializedAttachmentV4` in the
+  [app/serializers/v2/attachment.ts](app/serializers/v2/attachment.ts) file.
+- The new `GET /vN/attachments/:attId` API endpoint returns the attachment by
+  its ID.
+- The new `GET /vN/attachments/:attId/:type` API endpoint returns the preview or
+  the original of the attachment. The _type_ parameter can be 'original',
+  'image', 'video' or 'audio'. The returned data is a JSON object with the following
+  fields:
+  - _url_ - the URL of the preview/original file
+  - _mimeType_ - the MIME type of the preview/original file
+  - _width_ and _height_ (optional) - the size in pixels if the file is visual
+
+  This endpoint accepts the following query parameters (all optional):
+  - _width_ and _height_ - the desired 'image'/'video' preview size in pixels
+  - _format_ - the desired 'image' preview format: 'jpeg', 'webp', 'avif'
+  - _redirect_ - if present, the response will be a 302 redirect to the file
+
+  The server will choose the best available preview to fill the given size. It
+  is not guaranteed that the returned preview will be of the requested size and
+  format, but it will be as close as possible.
+- Allow to limit the number of simultaneous executions for some job types.
+
+  The JobManager now has a `limitedJobs` parameter of type `Record<string,
+  number>`, that defines the maximum number of simultaneous executions for each
+  job of given type (name). Other jobs, that are not listed in `limitedJobs` are
+  executed without limits.
 
 ## [2.22.5] - 2025-01-05
 ### Added

Dockerfile

@@ -2,7 +2,8 @@ FROM node:18-bookworm
 
 RUN apt-get update && \
     apt-get install -y \
-    graphicsmagick \
+    imagemagick \
+    ffmpeg \
     g++ \
     git \
     make
@@ -12,8 +13,7 @@ WORKDIR /server
 
 RUN rm -rf node_modules && \
     rm -f log/*.log && \
-    mkdir -p ./public/files/attachments/thumbnails && \
-    mkdir -p ./public/files/attachments/thumbnails2 && \
+    mkdir -p ./public/files/attachments && \
     yarn install
 
 ENV NODE_ENV production

README.md

@@ -48,7 +48,11 @@ mkdir ./public/files/attachments/thumbnails/ && mkdir ./public/files/attachments
 ```
 mkdir -p /tmp/pepyatka-media/attachments/thumbnails
 mkdir -p /tmp/pepyatka-media/attachments/thumbnails2
-mkdir -p /tmp/pepyatka-media/attachments/anotherTestSize
+mkdir -p /tmp/pepyatka-media/attachments/p1
+mkdir -p /tmp/pepyatka-media/attachments/p2
+mkdir -p /tmp/pepyatka-media/attachments/p3
+mkdir -p /tmp/pepyatka-media/attachments/p4
+mkdir -p /tmp/pepyatka-media/attachments/a1
 ```
 
 3. Create config `config/local.json` with some random secret string: `{   "secret": "myverysecretstring"  }`.

app/api-versions.ts

@@ -1,5 +1,6 @@
 export const API_VERSION_2 = 2;
 export const API_VERSION_3 = 3;
+export const API_VERSION_4 = 4;
 
-export const API_VERSION_ACTUAL = API_VERSION_3;
+export const API_VERSION_ACTUAL = API_VERSION_4;
 export const API_VERSION_MINIMAL = API_VERSION_2;

app/controllers/api/v1/AttachmentsController.js

@@ -1,13 +1,22 @@
 import createDebug from 'debug';
 import compose from 'koa-compose';
 import { isInt } from 'validator';
+import { lookup } from 'mime-types';
+import { mediaType } from '@hapi/accept';
 
-import { reportError, BadRequestException, ValidationException } from '../../../support/exceptions';
-import { serializeAttachment } from '../../../serializers/v2/post';
+import {
+  reportError,
+  BadRequestException,
+  ValidationException,
+  NotFoundException,
+} from '../../../support/exceptions';
+import { serializeAttachment } from '../../../serializers/v2/attachment';
 import { serializeUsersByIds } from '../../../serializers/v2/user';
 import { authRequired } from '../../middlewares';
-import { dbAdapter } from '../../../models';
+import { dbAdapter, Attachment } from '../../../models';
 import { startAttachmentsSanitizeJob } from '../../../jobs/attachments-sanitize';
+import { currentConfig } from '../../../support/app-async-context';
+import { getBestVariant } from '../../../support/media-files/geometry';
 
 export default class AttachmentsController {
   app;
@@ -23,20 +32,17 @@ export default class AttachmentsController {
     async (ctx) => {
       // Accept one file-type field with any name
       const [file] = Object.values(ctx.request.files || []);
-      const { user } = ctx.state;
+      const { user, apiVersion } = ctx.state;
 
       if (!file) {
         throw new BadRequestException('No file provided');
       }
 
       try {
-        const newAttachment = await user.newAttachment({
-          file: { ...file, path: file.filepath, name: file.originalFilename },
-        });
-        await newAttachment.create();
+        const newAttachment = await Attachment.create(file.filepath, file.originalFilename, user);
 
         ctx.body = {
-          attachments: serializeAttachment(newAttachment),
+          attachments: serializeAttachment(newAttachment, apiVersion),
           users: await serializeUsersByIds([newAttachment.userId], user.id),
         };
       } catch (e) {
@@ -64,7 +70,7 @@ export default class AttachmentsController {
   my = compose([
     authRequired(),
     async (ctx) => {
-      const { user } = ctx.state;
+      const { user, apiVersion } = ctx.state;
       const { limit: qLimit, page: qPage } = ctx.request.query;
 
       const DEFAULT_LIMIT = 30;
@@ -106,7 +112,7 @@ export default class AttachmentsController {
       }
 
       ctx.body = {
-        attachments: attachments.map(serializeAttachment),
+        attachments: attachments.map((a) => serializeAttachment(a, apiVersion)),
         users: await serializeUsersByIds([user.id], user.id),
         hasMore,
       };
@@ -138,4 +144,139 @@ export default class AttachmentsController {
       };
     },
   ]);
+
+  async getById(ctx) {
+    const { attId } = ctx.params;
+    const { user, apiVersion } = ctx.state;
+
+    const attachment = await dbAdapter.getAttachmentById(attId);
+
+    if (!attachment) {
+      throw new NotFoundException('Attachment not found');
+    }
+
+    const serAttachment = serializeAttachment(attachment, apiVersion);
+    const users = await serializeUsersByIds([attachment.userId], user?.id);
+
+    ctx.body = {
+      attachments: serAttachment,
+      users,
+    };
+  }
+
+  /**
+   * @param {import('koa').Context} ctx
+   */
+  async getPreview(ctx) {
+    const { attId, type } = ctx.params;
+    const { query } = ctx.request;
+    const { useImgProxy } = currentConfig().attachments;
+    const imageFormats = ['jpeg', 'webp', 'avif'];
+    const formatExtensions = {
+      jpeg: 'jpg',
+      webp: 'webp',
+      avif: 'avif',
+    };
+
+    if (!['original', 'image', 'video', 'audio'].includes(type)) {
+      throw new NotFoundException('Invalid preview type');
+    }
+
+    if ('format' in query && !imageFormats.includes(query.format)) {
+      throw new ValidationException('Invalid format value');
+    }
+
+    const width = 'width' in query ? Number.parseInt(query.width, 10) : undefined;
+    const height = 'height' in query ? Number.parseInt(query.height, 10) : undefined;
+
+    if (
+      (width && (!Number.isFinite(width) || width <= 0)) ||
+      (height && (!Number.isFinite(height) || height <= 0))
+    ) {
+      throw new ValidationException('Invalid width/height values');
+    }
+
+    const asRedirect = 'redirect' in query;
+
+    const attachment = await dbAdapter.getAttachmentById(attId);
+
+    if (!attachment) {
+      throw new NotFoundException('Attachment not found');
+    }
+
+    if (type !== 'original' && !(type in attachment.previews)) {
+      throw new NotFoundException('Preview of specified type not found');
+    }
+
+    const response = {};
+
+    if (type === 'original') {
+      response.url = attachment.getFileUrl('');
+      response.mimeType = attachment.mimeType;
+
+      if (attachment.width && attachment.height) {
+        response.width = attachment.width;
+        response.height = attachment.height;
+      }
+    } else if (type === 'audio') {
+      // We always have one audio preview
+      const [[variant, { ext }]] = Object.entries(attachment.previews.audio);
+      response.url = attachment.getFileUrl(variant);
+      response.mimeType = lookup(ext) || 'application/octet-stream';
+    } else {
+      // Visual types, 'image' and 'video'
+
+      const previews = attachment.previews[type];
+      const {
+        variant,
+        width: resWidth,
+        height: resHeight,
+      } = getBestVariant(previews, width, height);
+      const prv = previews[variant];
+
+      response.url = attachment.getFileUrl(variant);
+      response.mimeType = lookup(prv.ext) || 'application/octet-stream';
+      response.width = prv.w;
+      response.height = prv.h;
+
+      // With imgproxy, we can resize images and change their format
+      if (type === 'image' && useImgProxy) {
+        let { format } = query;
+
+        if (!format) {
+          const acceptedTypes = imageFormats.map((f) => `image/${f}`);
+          format = mediaType(ctx.headers.accept ?? 'image/jpeg', acceptedTypes);
+
+          if (acceptedTypes.includes(format)) {
+            format = format.replace('image/', '');
+          } else {
+            format = 'jpeg';
+          }
+        }
+
+        const fileUrl = new URL(response.url);
+
+        if (prv.ext !== formatExtensions[format]) {
+          fileUrl.searchParams.set('format', format);
+          response.mimeType = `image/${format}`;
+        }
+
+        if (resWidth !== prv.w || resHeight !== prv.h) {
+          fileUrl.searchParams.set('width', width.toString());
+          fileUrl.searchParams.set('height', height.toString());
+          response.width = resWidth;
+          response.height = resHeight;
+        }
+
+        response.url = fileUrl.toString();
+      }
+    }
+
+    if (asRedirect) {
+      ctx.redirect(response.url);
+      ctx.body = `Redirecting to ${response.url}`;
+    } else {
+      ctx.body = response;
+    }
+  }
 }