refactor(api): ⚡ Move media processing to background job

classes/database/attachment.ts

@@ -11,9 +11,13 @@ import {
     eq,
     inArray,
 } from "drizzle-orm";
+import sharp from "sharp";
 import { z } from "zod";
 import { MediaBackendType } from "~/packages/config-manager/config.type";
 import { config } from "~/packages/config-manager/index.ts";
+import { ApiError } from "../errors/api-error.ts";
+import { MediaManager } from "../media/media-manager.ts";
+import { MediaJobType, mediaQueue } from "../queues/media.ts";
 import { BaseInterface } from "./base.ts";
 
 type AttachmentType = InferSelectModel<typeof Attachments>;
@@ -150,6 +154,75 @@ export class Attachment extends BaseInterface<typeof Attachments> {
         return attachment;
     }
 
+    public static async fromFile(
+        file: File,
+        options?: {
+            description?: string;
+            thumbnail?: File;
+        },
+    ): Promise<Attachment> {
+        if (file.size > config.validation.max_media_size) {
+            throw new ApiError(
+                413,
+                `File too large, max size is ${config.validation.max_media_size} bytes`,
+            );
+        }
+
+        if (
+            config.validation.enforce_mime_types &&
+            !config.validation.allowed_mime_types.includes(file.type)
+        ) {
+            throw new ApiError(
+                415,
+                `File type ${file.type} is not allowed`,
+                `Allowed types: ${config.validation.allowed_mime_types.join(", ")}`,
+            );
+        }
+
+        const sha256 = new Bun.SHA256();
+
+        const isImage = file.type.startsWith("image/");
+
+        const metadata = isImage
+            ? await sharp(await file.arrayBuffer()).metadata()
+            : null;
+
+        const mediaManager = new MediaManager(config);
+
+        const { path, blurhash } = await mediaManager.addFile(file);
+
+        const url = Attachment.getUrl(path);
+
+        let thumbnailUrl = "";
+
+        if (options?.thumbnail) {
+            const { path } = await mediaManager.addFile(options.thumbnail);
+
+            thumbnailUrl = Attachment.getUrl(path);
+        }
+
+        const newAttachment = await Attachment.insert({
+            url,
+            thumbnailUrl: thumbnailUrl || undefined,
+            sha256: sha256.update(await file.arrayBuffer()).digest("hex"),
+            mimeType: file.type,
+            description: options?.description ?? "",
+            size: file.size,
+            blurhash: blurhash ?? undefined,
+            width: metadata?.width ?? undefined,
+            height: metadata?.height ?? undefined,
+        });
+
+        if (config.media.conversion.convert_images) {
+            await mediaQueue.add(MediaJobType.ConvertMedia, {
+                attachmentId: newAttachment.id,
+                filename: file.name,
+            });
+        }
+
+        return newAttachment;
+    }
+
     public get id(): string {
         return this.data.id;
     }

classes/media/media-manager.ts

@@ -8,7 +8,6 @@ import { DiskMediaDriver } from "./drivers/disk.ts";
 import type { MediaDriver } from "./drivers/media-driver.ts";
 import { S3MediaDriver } from "./drivers/s3.ts";
 import { BlurhashPreprocessor } from "./preprocessors/blurhash.ts";
-import { ImageConversionPreprocessor } from "./preprocessors/image-conversion.ts";
 import type { MediaPreprocessor } from "./preprocessors/media-preprocessor.ts";
 
 /**
@@ -58,11 +57,6 @@ export class MediaManager {
      * Initializes the preprocessors based on the configuration.
      */
     private initializePreprocessors(): void {
-        if (this.config.media.conversion.convert_images) {
-            this.preprocessors.push(
-                new ImageConversionPreprocessor(this.config),
-            );
-        }
         this.preprocessors.push(new BlurhashPreprocessor());
         // Add other preprocessors here as needed
     }
@@ -87,6 +81,7 @@ export class MediaManager {
         }
 
         const uploadResult = await this.driver.addFile(processedFile);
+
         return { ...uploadResult, blurhash };
     }
     /**

classes/queues/media.ts

@@ -0,0 +1,15 @@
+import { Queue } from "bullmq";
+import { connection } from "~/utils/redis.ts";
+
+export enum MediaJobType {
+    ConvertMedia = "convertMedia",
+}
+
+export type MediaJobData = {
+    attachmentId: string;
+    filename: string;
+};
+
+export const mediaQueue = new Queue<MediaJobData, void, MediaJobType>("media", {
+    connection,
+});

classes/workers/media.ts

@@ -0,0 +1,78 @@
+import { Attachment } from "@versia/kit/db";
+import { Worker } from "bullmq";
+import { config } from "~/packages/config-manager";
+import { connection } from "~/utils/redis.ts";
+import { MediaManager } from "../media/media-manager.ts";
+import { ImageConversionPreprocessor } from "../media/preprocessors/image-conversion.ts";
+import {
+    type MediaJobData,
+    MediaJobType,
+    mediaQueue,
+} from "../queues/media.ts";
+
+export const getMediaWorker = (): Worker<MediaJobData, void, MediaJobType> =>
+    new Worker<MediaJobData, void, MediaJobType>(
+        mediaQueue.name,
+        async (job) => {
+            switch (job.name) {
+                case MediaJobType.ConvertMedia: {
+                    const { attachmentId, filename } = job.data;
+
+                    const attachment = await Attachment.fromId(attachmentId);
+
+                    if (!attachment) {
+                        throw new Error(
+                            `Attachment not found: [${attachmentId}]`,
+                        );
+                    }
+
+                    const processor = new ImageConversionPreprocessor(config);
+
+                    const hash = attachment?.data.sha256;
+
+                    if (!hash) {
+                        throw new Error(
+                            `Attachment [${attachmentId}] has no hash, cannot process.`,
+                        );
+                    }
+
+                    // Download the file and process it.
+                    const blob = await (
+                        await fetch(attachment.data.url)
+                    ).blob();
+
+                    const file = new File([blob], filename);
+
+                    const { file: processedFile } =
+                        await processor.process(file);
+
+                    const mediaManager = new MediaManager(config);
+
+                    const { path, uploadedFile } =
+                        await mediaManager.addFile(processedFile);
+
+                    const url = Attachment.getUrl(path);
+
+                    const sha256 = new Bun.SHA256();
+
+                    await attachment.update({
+                        url,
+                        sha256: sha256
+                            .update(await uploadedFile.arrayBuffer())
+                            .digest("hex"),
+                        mimeType: uploadedFile.type,
+                        size: uploadedFile.size,
+                    });
+                }
+            }
+        },
+        {
+            connection,
+            removeOnComplete: {
+                age: config.queues.media.remove_on_complete,
+            },
+            removeOnFail: {
+                age: config.queues.media.remove_on_failure,
+            },
+        },
+    );