refactor(database): ♻️ Use ContentFormat to store media data

api/api/v1/media/:id/index.ts

@@ -107,26 +107,35 @@ export default apiRoute((app) => {
             throw new ApiError(404, "Media not found");
         }
 
-        const { description, thumbnail } = context.req.valid("form");
+        const { description, thumbnail: thumbnailFile } =
-
+            context.req.valid("form");
-        let thumbnailUrl = attachment.data.thumbnailUrl;
 
         const mediaManager = new MediaManager(config);
 
-        if (thumbnail) {
+        // TODO: Generate thumbnail if not provided
-            const { path } = await mediaManager.addFile(thumbnail);
+        if (thumbnailFile) {
-            thumbnailUrl = Media.getUrl(path);
+            const { path } = await mediaManager.addFile(thumbnailFile);
+            const thumbnail = attachment.data.thumbnail;
+
+            // FIXME: Also update thumbnail if it hasn't been set
+            if (thumbnail) {
+                thumbnail[Object.keys(thumbnail)[0]].content =
+                    Media.getUrl(path);
+            }
+
+            attachment.data.thumbnail = thumbnail;
         }
 
-        const descriptionText = description || attachment.data.description;
+        if (description) {
+            for (const type of Object.keys(attachment.data.content)) {
+                attachment.data.content[type].description = description;
+            }
+        }
 
-        if (
+        if (description || thumbnailFile) {
-            descriptionText !== attachment.data.description ||
-            thumbnailUrl !== attachment.data.thumbnailUrl
-        ) {
             await attachment.update({
-                description: descriptionText,
+                content: attachment.data.content,
-                thumbnailUrl,
+                thumbnail: attachment.data.thumbnail,
             });
 
             return context.json(attachment.toApi(), 200);

classes/database/media.ts

@@ -3,6 +3,7 @@ import type { Attachment as ApiAttachment } from "@versia/client/types";
 import type { ContentFormat } from "@versia/federation/types";
 import { db } from "@versia/kit/db";
 import { Medias } from "@versia/kit/tables";
+import { SHA256 } from "bun";
 import {
     type InferInsertModel,
     type InferSelectModel,
@@ -175,14 +176,6 @@ export class Media extends BaseInterface<typeof Medias> {
             );
         }
 
-        const sha256 = new Bun.SHA256();
-
-        const isImage = file.type.startsWith("image/");
-
-        const metadata = isImage
-            ? await sharp(await file.arrayBuffer()).metadata()
-            : null;
-
         const mediaManager = new MediaManager(config);
 
         const { path } = await mediaManager.addFile(file);
@@ -197,15 +190,18 @@ export class Media extends BaseInterface<typeof Medias> {
             thumbnailUrl = Media.getUrl(path);
         }
 
+        const content = await Media.fileToContentFormat(file, url, {
+            description: options?.description,
+        });
+        const thumbnailContent = options?.thumbnail
+            ? await Media.fileToContentFormat(options.thumbnail, thumbnailUrl, {
+                  description: options?.description,
+              })
+            : undefined;
+
         const newAttachment = await Media.insert({
-            url,
+            content,
-            thumbnailUrl: thumbnailUrl || undefined,
+            thumbnail: thumbnailContent,
-            sha256: sha256.update(await file.arrayBuffer()).digest("hex"),
-            mimeType: file.type,
-            description: options?.description ?? "",
-            size: file.size,
-            width: metadata?.width ?? undefined,
-            height: metadata?.height ?? undefined,
         });
 
         if (config.media.conversion.convert_images) {
@@ -215,6 +211,11 @@ export class Media extends BaseInterface<typeof Medias> {
             });
         }
 
+        await mediaQueue.add(MediaJobType.CalculateMetadata, {
+            attachmentId: newAttachment.id,
+            filename: file.name,
+        });
+
         return newAttachment;
     }
 
@@ -232,105 +233,165 @@ export class Media extends BaseInterface<typeof Medias> {
         return "";
     }
 
+    public getUrl(): string {
+        const type = this.getPreferredMimeType();
+
+        return this.data.content[type]?.content;
+    }
+
+    /**
+     * Gets favourite MIME type for the attachment
+     * Uses a hardcoded list of preferred types, for images
+     *
+     * @returns {string} Preferred MIME type
+     */
+    public getPreferredMimeType(): string {
+        return Media.getPreferredMimeType(Object.keys(this.data.content));
+    }
+
+    /**
+     * Gets favourite MIME type from a list
+     * Uses a hardcoded list of preferred types, for images
+     *
+     * @returns {string} Preferred MIME type
+     */
+    public static getPreferredMimeType(types: string[]): string {
+        const ranking = [
+            "image/svg+xml",
+            "image/avif",
+            "image/jxl",
+            "image/webp",
+            "image/heif",
+            "image/heif-sequence",
+            "image/heic",
+            "image/heic-sequence",
+            "image/apng",
+            "image/gif",
+            "image/png",
+            "image/jpeg",
+            "image/bmp",
+        ];
+
+        return ranking.find((type) => types.includes(type)) ?? types[0];
+    }
+
+    /**
+     * Maps MIME type to Mastodon attachment type
+     *
+     * @returns
+     */
     public getMastodonType(): ApiAttachment["type"] {
-        if (this.data.mimeType.startsWith("image/")) {
+        const type = this.getPreferredMimeType();
+
+        if (type.startsWith("image/")) {
             return "image";
         }
-        if (this.data.mimeType.startsWith("video/")) {
+        if (type.startsWith("video/")) {
             return "video";
         }
-        if (this.data.mimeType.startsWith("audio/")) {
+        if (type.startsWith("audio/")) {
             return "audio";
         }
 
         return "unknown";
     }
 
-    public toApiMeta(): ApiAttachment["meta"] {
+    /**
+     * Extracts metadata from a file and outputs as ContentFormat
+     *
+     * Does not calculate thumbhash (do this in a worker)
+     * @param file
+     * @param uri Uploaded file URI
+     * @param options Extra metadata, such as description
+     * @returns
+     */
+    public static async fileToContentFormat(
+        file: File,
+        uri: string,
+        options?: Partial<{
+            description: string;
+        }>,
+    ): Promise<ContentFormat> {
+        const buffer = await file.arrayBuffer();
+        const isImage = file.type.startsWith("image/");
+        const { width, height } = isImage ? await sharp(buffer).metadata() : {};
+        const hash = new SHA256().update(file).digest("hex");
+
+        // Missing: fps, duration
+        // Thumbhash should be added in a worker after the file is uploaded
         return {
-            width: this.data.width || undefined,
+            [file.type]: {
-            height: this.data.height || undefined,
+                content: uri,
-            fps: this.data.fps || undefined,
+                remote: true,
-            size:
+                hash: {
-                this.data.width && this.data.height
+                    sha256: hash,
-                    ? `${this.data.width}x${this.data.height}`
+                },
-                    : undefined,
+                width,
-            duration: this.data.duration || undefined,
+                height,
-            length: undefined,
+                description: options?.description,
-            aspect:
+                size: file.size,
-                this.data.width && this.data.height
-                    ? this.data.width / this.data.height
-                    : undefined,
-            original: {
-                width: this.data.width || undefined,
-                height: this.data.height || undefined,
-                size:
-                    this.data.width && this.data.height
-                        ? `${this.data.width}x${this.data.height}`
-                        : undefined,
-                aspect:
-                    this.data.width && this.data.height
-                        ? this.data.width / this.data.height
-                        : undefined,
             },
+        };
+    }
+
+    public toApiMeta(): ApiAttachment["meta"] {
+        const type = this.getPreferredMimeType();
+        const data = this.data.content[type];
+        const size =
+            data.width && data.height
+                ? `${data.width}x${data.height}`
+                : undefined;
+        const aspect =
+            data.width && data.height ? data.width / data.height : undefined;
+
+        return {
+            width: data.width || undefined,
+            height: data.height || undefined,
+            fps: data.fps || undefined,
+            size,
             // Idk whether size or length is the right value
+            duration: data.duration || undefined,
+            // Versia doesn't have a concept of length in ContentFormat
+            length: undefined,
+            aspect,
+            original: {
+                width: data.width || undefined,
+                height: data.height || undefined,
+                size,
+                aspect,
+            },
         };
     }
 
     public toApi(): ApiAttachment {
+        const type = this.getPreferredMimeType();
+        const data = this.data.content[type];
+
+        // Thumbnail should only have a single MIME type
+        const thumbnailData =
+            this.data.thumbnail?.[Object.keys(this.data.thumbnail)[0]];
+
         return {
             id: this.data.id,
             type: this.getMastodonType(),
-            url: proxyUrl(this.data.url) ?? "",
+            url: proxyUrl(data.content) ?? "",
-            remote_url: proxyUrl(this.data.remoteUrl),
+            remote_url: null,
-            preview_url: proxyUrl(this.data.thumbnailUrl || this.data.url),
+            preview_url: proxyUrl(thumbnailData?.content),
             text_url: null,
             meta: this.toApiMeta(),
-            description: this.data.description,
+            description: data.description || null,
             blurhash: this.data.blurhash,
         };
     }
 
     public toVersia(): ContentFormat {
-        return {
+        return this.data.content;
-            [this.data.mimeType]: {
-                content: this.data.url,
-                remote: true,
-                // TODO: Replace BlurHash with thumbhash
-                // thumbhash: this.data.blurhash ?? undefined,
-                description: this.data.description ?? undefined,
-                duration: this.data.duration ?? undefined,
-                fps: this.data.fps ?? undefined,
-                height: this.data.height ?? undefined,
-                size: this.data.size ?? undefined,
-                hash: this.data.sha256
-                    ? {
-                          sha256: this.data.sha256,
-                      }
-                    : undefined,
-                width: this.data.width ?? undefined,
-            },
-        };
     }
 
-    public static fromVersia(
+    public static fromVersia(contentFormat: ContentFormat): Promise<Media> {
-        attachmentToConvert: ContentFormat,
-    ): Promise<Media> {
-        const key = Object.keys(attachmentToConvert)[0];
-        const value = attachmentToConvert[key];
-
         return Media.insert({
-            mimeType: key,
+            content: contentFormat,
-            url: value.content,
+            originalContent: contentFormat,
-            description: value.description || undefined,
-            duration: value.duration || undefined,
-            fps: value.fps || undefined,
-            height: value.height || undefined,
-            // biome-ignore lint/style/useExplicitLengthCheck: Biome thinks we're checking if size is not zero
-            size: value.size || undefined,
-            width: value.width || undefined,
-            sha256: value.hash?.sha256 || undefined,
-            // blurhash: value.blurhash || undefined,
         });
     }
 }

classes/database/note.ts

@@ -44,9 +44,9 @@ import {
 import { config } from "~/packages/config-manager";
 import { DeliveryJobType, deliveryQueue } from "../queues/delivery.ts";
 import { Application } from "./application.ts";
-import { Media } from "./attachment.ts";
 import { BaseInterface } from "./base.ts";
 import { Emoji } from "./emoji.ts";
+import { Media } from "./media.ts";
 import { User } from "./user.ts";
 
 type NoteType = InferSelectModel<typeof Notes>;

classes/queues/media.ts

@@ -3,6 +3,7 @@ import { connection } from "~/utils/redis.ts";
 
 export enum MediaJobType {
     ConvertMedia = "convertMedia",
+    CalculateMetadata = "calculateMetadata",
 }
 
 export type MediaJobData = {

classes/workers/media.ts

@@ -30,24 +30,15 @@ export const getMediaWorker = (): Worker<MediaJobData, void, MediaJobType> =>
                     }
 
                     const processor = new ImageConversionPreprocessor(config);
-                    const blurhashProcessor = new BlurhashPreprocessor();
-
-                    const hash = attachment?.data.sha256;
-
-                    if (!hash) {
-                        throw new Error(
-                            `Attachment [${attachmentId}] has no hash, cannot process.`,
-                        );
-                    }
 
                     await job.log(`Processing attachment [${attachmentId}]`);
                     await job.log(
-                        `Fetching file from [${attachment.data.url}]`,
+                        `Fetching file from [${attachment.getUrl()}]`,
                     );
 
                     // Download the file and process it.
                     const blob = await (
-                        await fetch(attachment.data.url)
+                        await fetch(attachment.getUrl())
                     ).blob();
 
                     const file = new File([blob], filename);
@@ -57,10 +48,6 @@ export const getMediaWorker = (): Worker<MediaJobData, void, MediaJobType> =>
                     const { file: processedFile } =
                         await processor.process(file);
 
-                    await job.log(`Generating blurhash for [${attachmentId}]`);
-
-                    const { blurhash } = await blurhashProcessor.process(file);
-
                     const mediaManager = new MediaManager(config);
 
                     await job.log(`Uploading attachment [${attachmentId}]`);
@@ -70,21 +57,66 @@ export const getMediaWorker = (): Worker<MediaJobData, void, MediaJobType> =>
 
                     const url = Media.getUrl(path);
 
-                    const sha256 = new Bun.SHA256();
+                    await attachment.update({
+                        content: await Media.fileToContentFormat(
+                            uploadedFile,
+                            url,
+                            {
+                                description:
+                                    attachment.data.content[0].description ||
+                                    undefined,
+                            },
+                        ),
+                    });
+
+                    await job.log(
+                        `✔ Finished processing attachment [${attachmentId}]`,
+                    );
+
+                    break;
+                }
+                case MediaJobType.CalculateMetadata: {
+                    // Calculate blurhash
+                    const { attachmentId } = job.data;
+
+                    await job.log(`Fetching attachment ID [${attachmentId}]`);
+
+                    const attachment = await Media.fromId(attachmentId);
+
+                    if (!attachment) {
+                        throw new Error(
+                            `Attachment not found: [${attachmentId}]`,
+                        );
+                    }
+
+                    const blurhashProcessor = new BlurhashPreprocessor();
+
+                    await job.log(`Processing attachment [${attachmentId}]`);
+                    await job.log(
+                        `Fetching file from [${attachment.getUrl()}]`,
+                    );
+
+                    // Download the file and process it.
+                    const blob = await (
+                        await fetch(attachment.getUrl())
+                    ).blob();
+
+                    // Filename is not important for blurhash
+                    const file = new File([blob], "");
+
+                    await job.log(`Generating blurhash for [${attachmentId}]`);
+
+                    const { blurhash } = await blurhashProcessor.process(file);
 
                     await attachment.update({
-                        url,
-                        sha256: sha256
-                            .update(await uploadedFile.arrayBuffer())
-                            .digest("hex"),
-                        mimeType: uploadedFile.type,
-                        size: uploadedFile.size,
                         blurhash,
                     });
 
                     await job.log(
                         `✔ Finished processing attachment [${attachmentId}]`,
                     );
+
+                    break;
                 }
             }
         },

drizzle/schema.ts

@@ -305,18 +305,10 @@ export const Tokens = pgTable("Tokens", {
 
 export const Medias = pgTable("Medias", {
     id: id(),
-    url: text("url").notNull(),
+    content: jsonb("content").notNull().$type<ContentFormat>(),
-    remoteUrl: text("remote_url"),
+    originalContent: jsonb("original_content").$type<ContentFormat>(),
-    thumbnailUrl: text("thumbnail_url"),
+    thumbnail: jsonb("thumbnail").$type<ContentFormat>(),
-    mimeType: text("mime_type").notNull(),
-    description: text("description"),
     blurhash: text("blurhash"),
-    sha256: text("sha256"),
-    fps: integer("fps"),
-    duration: integer("duration"),
-    width: integer("width"),
-    height: integer("height"),
-    size: integer("size"),
     noteId: uuid("noteId").references(() => Notes.id, {
         onDelete: "cascade",
         onUpdate: "cascade",

packages/plugin-kit/exports/db.ts

@@ -1,7 +1,7 @@
 // biome-ignore lint/performance/noBarrelFile: <explanation>
 export { User } from "~/classes/database/user.ts";
 export { Role } from "~/classes/database/role.ts";
-export { Media } from "~/classes/database/attachment.ts";
+export { Media } from "~/classes/database/media";
 export { Emoji } from "~/classes/database/emoji.ts";
 export { Instance } from "~/classes/database/instance.ts";
 export { Note } from "~/classes/database/note.ts";