diff --git a/api/api/v1/media/:id/index.ts b/api/api/v1/media/:id/index.ts index 83621c69..65fc11aa 100644 --- a/api/api/v1/media/:id/index.ts +++ b/api/api/v1/media/:id/index.ts @@ -107,26 +107,35 @@ export default apiRoute((app) => { throw new ApiError(404, "Media not found"); } - const { description, thumbnail } = context.req.valid("form"); - - let thumbnailUrl = attachment.data.thumbnailUrl; + const { description, thumbnail: thumbnailFile } = + context.req.valid("form"); const mediaManager = new MediaManager(config); - if (thumbnail) { - const { path } = await mediaManager.addFile(thumbnail); - thumbnailUrl = Media.getUrl(path); + // TODO: Generate thumbnail if not provided + if (thumbnailFile) { + const { path } = await mediaManager.addFile(thumbnailFile); + const thumbnail = attachment.data.thumbnail; + + // FIXME: Also update thumbnail if it hasn't been set + if (thumbnail) { + thumbnail[Object.keys(thumbnail)[0]].content = + Media.getUrl(path); + } + + attachment.data.thumbnail = thumbnail; } - const descriptionText = description || attachment.data.description; + if (description) { + for (const type of Object.keys(attachment.data.content)) { + attachment.data.content[type].description = description; + } + } - if ( - descriptionText !== attachment.data.description || - thumbnailUrl !== attachment.data.thumbnailUrl - ) { + if (description || thumbnailFile) { await attachment.update({ - description: descriptionText, - thumbnailUrl, + content: attachment.data.content, + thumbnail: attachment.data.thumbnail, }); return context.json(attachment.toApi(), 200); diff --git a/classes/database/attachment.ts b/classes/database/media.ts similarity index 62% rename from classes/database/attachment.ts rename to classes/database/media.ts index 4d1c3bf9..4c5003ae 100644 --- a/classes/database/attachment.ts +++ b/classes/database/media.ts @@ -3,6 +3,7 @@ import type { Attachment as ApiAttachment } from "@versia/client/types"; import type { ContentFormat } from "@versia/federation/types"; import { db } from "@versia/kit/db"; import { Medias } from "@versia/kit/tables"; +import { SHA256 } from "bun"; import { type InferInsertModel, type InferSelectModel, @@ -175,14 +176,6 @@ export class Media extends BaseInterface { ); } - const sha256 = new Bun.SHA256(); - - const isImage = file.type.startsWith("image/"); - - const metadata = isImage - ? await sharp(await file.arrayBuffer()).metadata() - : null; - const mediaManager = new MediaManager(config); const { path } = await mediaManager.addFile(file); @@ -197,15 +190,18 @@ export class Media extends BaseInterface { thumbnailUrl = Media.getUrl(path); } + const content = await Media.fileToContentFormat(file, url, { + description: options?.description, + }); + const thumbnailContent = options?.thumbnail + ? await Media.fileToContentFormat(options.thumbnail, thumbnailUrl, { + description: options?.description, + }) + : undefined; + const newAttachment = await Media.insert({ - url, - thumbnailUrl: thumbnailUrl || undefined, - sha256: sha256.update(await file.arrayBuffer()).digest("hex"), - mimeType: file.type, - description: options?.description ?? "", - size: file.size, - width: metadata?.width ?? undefined, - height: metadata?.height ?? undefined, + content, + thumbnail: thumbnailContent, }); if (config.media.conversion.convert_images) { @@ -215,6 +211,11 @@ export class Media extends BaseInterface { }); } + await mediaQueue.add(MediaJobType.CalculateMetadata, { + attachmentId: newAttachment.id, + filename: file.name, + }); + return newAttachment; } @@ -232,105 +233,165 @@ export class Media extends BaseInterface { return ""; } + public getUrl(): string { + const type = this.getPreferredMimeType(); + + return this.data.content[type]?.content; + } + + /** + * Gets favourite MIME type for the attachment + * Uses a hardcoded list of preferred types, for images + * + * @returns {string} Preferred MIME type + */ + public getPreferredMimeType(): string { + return Media.getPreferredMimeType(Object.keys(this.data.content)); + } + + /** + * Gets favourite MIME type from a list + * Uses a hardcoded list of preferred types, for images + * + * @returns {string} Preferred MIME type + */ + public static getPreferredMimeType(types: string[]): string { + const ranking = [ + "image/svg+xml", + "image/avif", + "image/jxl", + "image/webp", + "image/heif", + "image/heif-sequence", + "image/heic", + "image/heic-sequence", + "image/apng", + "image/gif", + "image/png", + "image/jpeg", + "image/bmp", + ]; + + return ranking.find((type) => types.includes(type)) ?? types[0]; + } + + /** + * Maps MIME type to Mastodon attachment type + * + * @returns + */ public getMastodonType(): ApiAttachment["type"] { - if (this.data.mimeType.startsWith("image/")) { + const type = this.getPreferredMimeType(); + + if (type.startsWith("image/")) { return "image"; } - if (this.data.mimeType.startsWith("video/")) { + if (type.startsWith("video/")) { return "video"; } - if (this.data.mimeType.startsWith("audio/")) { + if (type.startsWith("audio/")) { return "audio"; } return "unknown"; } - public toApiMeta(): ApiAttachment["meta"] { + /** + * Extracts metadata from a file and outputs as ContentFormat + * + * Does not calculate thumbhash (do this in a worker) + * @param file + * @param uri Uploaded file URI + * @param options Extra metadata, such as description + * @returns + */ + public static async fileToContentFormat( + file: File, + uri: string, + options?: Partial<{ + description: string; + }>, + ): Promise { + const buffer = await file.arrayBuffer(); + const isImage = file.type.startsWith("image/"); + const { width, height } = isImage ? await sharp(buffer).metadata() : {}; + const hash = new SHA256().update(file).digest("hex"); + + // Missing: fps, duration + // Thumbhash should be added in a worker after the file is uploaded return { - width: this.data.width || undefined, - height: this.data.height || undefined, - fps: this.data.fps || undefined, - size: - this.data.width && this.data.height - ? `${this.data.width}x${this.data.height}` - : undefined, - duration: this.data.duration || undefined, - length: undefined, - aspect: - this.data.width && this.data.height - ? this.data.width / this.data.height - : undefined, - original: { - width: this.data.width || undefined, - height: this.data.height || undefined, - size: - this.data.width && this.data.height - ? `${this.data.width}x${this.data.height}` - : undefined, - aspect: - this.data.width && this.data.height - ? this.data.width / this.data.height - : undefined, + [file.type]: { + content: uri, + remote: true, + hash: { + sha256: hash, + }, + width, + height, + description: options?.description, + size: file.size, }, + }; + } + + public toApiMeta(): ApiAttachment["meta"] { + const type = this.getPreferredMimeType(); + const data = this.data.content[type]; + const size = + data.width && data.height + ? `${data.width}x${data.height}` + : undefined; + const aspect = + data.width && data.height ? data.width / data.height : undefined; + + return { + width: data.width || undefined, + height: data.height || undefined, + fps: data.fps || undefined, + size, // Idk whether size or length is the right value + duration: data.duration || undefined, + // Versia doesn't have a concept of length in ContentFormat + length: undefined, + aspect, + original: { + width: data.width || undefined, + height: data.height || undefined, + size, + aspect, + }, }; } public toApi(): ApiAttachment { + const type = this.getPreferredMimeType(); + const data = this.data.content[type]; + + // Thumbnail should only have a single MIME type + const thumbnailData = + this.data.thumbnail?.[Object.keys(this.data.thumbnail)[0]]; + return { id: this.data.id, type: this.getMastodonType(), - url: proxyUrl(this.data.url) ?? "", - remote_url: proxyUrl(this.data.remoteUrl), - preview_url: proxyUrl(this.data.thumbnailUrl || this.data.url), + url: proxyUrl(data.content) ?? "", + remote_url: null, + preview_url: proxyUrl(thumbnailData?.content), text_url: null, meta: this.toApiMeta(), - description: this.data.description, + description: data.description || null, blurhash: this.data.blurhash, }; } public toVersia(): ContentFormat { - return { - [this.data.mimeType]: { - content: this.data.url, - remote: true, - // TODO: Replace BlurHash with thumbhash - // thumbhash: this.data.blurhash ?? undefined, - description: this.data.description ?? undefined, - duration: this.data.duration ?? undefined, - fps: this.data.fps ?? undefined, - height: this.data.height ?? undefined, - size: this.data.size ?? undefined, - hash: this.data.sha256 - ? { - sha256: this.data.sha256, - } - : undefined, - width: this.data.width ?? undefined, - }, - }; + return this.data.content; } - public static fromVersia( - attachmentToConvert: ContentFormat, - ): Promise { - const key = Object.keys(attachmentToConvert)[0]; - const value = attachmentToConvert[key]; - + public static fromVersia(contentFormat: ContentFormat): Promise { return Media.insert({ - mimeType: key, - url: value.content, - description: value.description || undefined, - duration: value.duration || undefined, - fps: value.fps || undefined, - height: value.height || undefined, - // biome-ignore lint/style/useExplicitLengthCheck: Biome thinks we're checking if size is not zero - size: value.size || undefined, - width: value.width || undefined, - sha256: value.hash?.sha256 || undefined, - // blurhash: value.blurhash || undefined, + content: contentFormat, + originalContent: contentFormat, }); } } diff --git a/classes/database/note.ts b/classes/database/note.ts index b732b70c..f7fd22cb 100644 --- a/classes/database/note.ts +++ b/classes/database/note.ts @@ -44,9 +44,9 @@ import { import { config } from "~/packages/config-manager"; import { DeliveryJobType, deliveryQueue } from "../queues/delivery.ts"; import { Application } from "./application.ts"; -import { Media } from "./attachment.ts"; import { BaseInterface } from "./base.ts"; import { Emoji } from "./emoji.ts"; +import { Media } from "./media.ts"; import { User } from "./user.ts"; type NoteType = InferSelectModel; diff --git a/classes/queues/media.ts b/classes/queues/media.ts index a9005eaf..e3177bf1 100644 --- a/classes/queues/media.ts +++ b/classes/queues/media.ts @@ -3,6 +3,7 @@ import { connection } from "~/utils/redis.ts"; export enum MediaJobType { ConvertMedia = "convertMedia", + CalculateMetadata = "calculateMetadata", } export type MediaJobData = { diff --git a/classes/workers/media.ts b/classes/workers/media.ts index e42597ed..ac116907 100644 --- a/classes/workers/media.ts +++ b/classes/workers/media.ts @@ -30,24 +30,15 @@ export const getMediaWorker = (): Worker => } const processor = new ImageConversionPreprocessor(config); - const blurhashProcessor = new BlurhashPreprocessor(); - - const hash = attachment?.data.sha256; - - if (!hash) { - throw new Error( - `Attachment [${attachmentId}] has no hash, cannot process.`, - ); - } await job.log(`Processing attachment [${attachmentId}]`); await job.log( - `Fetching file from [${attachment.data.url}]`, + `Fetching file from [${attachment.getUrl()}]`, ); // Download the file and process it. const blob = await ( - await fetch(attachment.data.url) + await fetch(attachment.getUrl()) ).blob(); const file = new File([blob], filename); @@ -57,10 +48,6 @@ export const getMediaWorker = (): Worker => const { file: processedFile } = await processor.process(file); - await job.log(`Generating blurhash for [${attachmentId}]`); - - const { blurhash } = await blurhashProcessor.process(file); - const mediaManager = new MediaManager(config); await job.log(`Uploading attachment [${attachmentId}]`); @@ -70,21 +57,66 @@ export const getMediaWorker = (): Worker => const url = Media.getUrl(path); - const sha256 = new Bun.SHA256(); + await attachment.update({ + content: await Media.fileToContentFormat( + uploadedFile, + url, + { + description: + attachment.data.content[0].description || + undefined, + }, + ), + }); + + await job.log( + `✔ Finished processing attachment [${attachmentId}]`, + ); + + break; + } + case MediaJobType.CalculateMetadata: { + // Calculate blurhash + const { attachmentId } = job.data; + + await job.log(`Fetching attachment ID [${attachmentId}]`); + + const attachment = await Media.fromId(attachmentId); + + if (!attachment) { + throw new Error( + `Attachment not found: [${attachmentId}]`, + ); + } + + const blurhashProcessor = new BlurhashPreprocessor(); + + await job.log(`Processing attachment [${attachmentId}]`); + await job.log( + `Fetching file from [${attachment.getUrl()}]`, + ); + + // Download the file and process it. + const blob = await ( + await fetch(attachment.getUrl()) + ).blob(); + + // Filename is not important for blurhash + const file = new File([blob], ""); + + await job.log(`Generating blurhash for [${attachmentId}]`); + + const { blurhash } = await blurhashProcessor.process(file); await attachment.update({ - url, - sha256: sha256 - .update(await uploadedFile.arrayBuffer()) - .digest("hex"), - mimeType: uploadedFile.type, - size: uploadedFile.size, blurhash, }); await job.log( `✔ Finished processing attachment [${attachmentId}]`, ); + + break; } } }, diff --git a/drizzle/schema.ts b/drizzle/schema.ts index aad1e069..b3295607 100644 --- a/drizzle/schema.ts +++ b/drizzle/schema.ts @@ -305,18 +305,10 @@ export const Tokens = pgTable("Tokens", { export const Medias = pgTable("Medias", { id: id(), - url: text("url").notNull(), - remoteUrl: text("remote_url"), - thumbnailUrl: text("thumbnail_url"), - mimeType: text("mime_type").notNull(), - description: text("description"), + content: jsonb("content").notNull().$type(), + originalContent: jsonb("original_content").$type(), + thumbnail: jsonb("thumbnail").$type(), blurhash: text("blurhash"), - sha256: text("sha256"), - fps: integer("fps"), - duration: integer("duration"), - width: integer("width"), - height: integer("height"), - size: integer("size"), noteId: uuid("noteId").references(() => Notes.id, { onDelete: "cascade", onUpdate: "cascade", diff --git a/packages/plugin-kit/exports/db.ts b/packages/plugin-kit/exports/db.ts index 4b00aa20..5f3ad757 100644 --- a/packages/plugin-kit/exports/db.ts +++ b/packages/plugin-kit/exports/db.ts @@ -1,7 +1,7 @@ // biome-ignore lint/performance/noBarrelFile: export { User } from "~/classes/database/user.ts"; export { Role } from "~/classes/database/role.ts"; -export { Media } from "~/classes/database/attachment.ts"; +export { Media } from "~/classes/database/media"; export { Emoji } from "~/classes/database/emoji.ts"; export { Instance } from "~/classes/database/instance.ts"; export { Note } from "~/classes/database/note.ts";