versia-pub/server
2
0
Fork 0
mirror of https://github.com/versia-pub/server.git synced 2025-12-06 08:28:19 +01:00
Code Issues Actions Packages Projects Releases Wiki Activity

refactor(database): ♻️ Use ContentFormat to store media data

Browse source
This commit is contained in:
Jesse Wierzbinski 2025-01-23 19:37:17 +01:00
parent 2f61cd8f0a
commit 9c30dacda7
No known key found for this signature in database
7 changed files with 227 additions and 132 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

33
api/api/v1/media/:id/index.ts
View file

@ -107,26 +107,35 @@ export default apiRoute((app) => {
throw new ApiError(404, "Media not found");
}
const { description, thumbnail } = context.req.valid("form");
let thumbnailUrl = attachment.data.thumbnailUrl;
const { description, thumbnail: thumbnailFile } =
context.req.valid("form");
const mediaManager = new MediaManager(config);
// TODO: Generate thumbnail if not provided
if (thumbnailFile) {
const { path } = await mediaManager.addFile(thumbnailFile);
const thumbnail = attachment.data.thumbnail;
// FIXME: Also update thumbnail if it hasn't been set
if (thumbnail) {
const { path } = await mediaManager.addFile(thumbnail);
thumbnailUrl = Media.getUrl(path);
thumbnail[Object.keys(thumbnail)[0]].content =
Media.getUrl(path);
}
const descriptionText = description || attachment.data.description;
attachment.data.thumbnail = thumbnail;
}
if (
descriptionText !== attachment.data.description ||
thumbnailUrl !== attachment.data.thumbnailUrl
) {
if (description) {
for (const type of Object.keys(attachment.data.content)) {
attachment.data.content[type].description = description;
}
}
if (description || thumbnailFile) {
await attachment.update({
description: descriptionText,
thumbnailUrl,
content: attachment.data.content,
thumbnail: attachment.data.thumbnail,
});
return context.json(attachment.toApi(), 200);

229
classes/database/attachment.ts → classes/database/media.ts
View file

@ -3,6 +3,7 @@ import type { Attachment as ApiAttachment } from "@versia/client/types";
import type { ContentFormat } from "@versia/federation/types";
import { db } from "@versia/kit/db";
import { Medias } from "@versia/kit/tables";
import { SHA256 } from "bun";
import {
type InferInsertModel,
type InferSelectModel,
@ -175,14 +176,6 @@ export class Media extends BaseInterface<typeof Medias> {
);
}
const sha256 = new Bun.SHA256();
const isImage = file.type.startsWith("image/");
const metadata = isImage
? await sharp(await file.arrayBuffer()).metadata()
: null;
const mediaManager = new MediaManager(config);
const { path } = await mediaManager.addFile(file);
@ -197,15 +190,18 @@ export class Media extends BaseInterface<typeof Medias> {
thumbnailUrl = Media.getUrl(path);
}
const content = await Media.fileToContentFormat(file, url, {
description: options?.description,
});
const thumbnailContent = options?.thumbnail
? await Media.fileToContentFormat(options.thumbnail, thumbnailUrl, {
description: options?.description,
})
: undefined;
const newAttachment = await Media.insert({
url,
thumbnailUrl: thumbnailUrl || undefined,
sha256: sha256.update(await file.arrayBuffer()).digest("hex"),
mimeType: file.type,
description: options?.description ?? "",
size: file.size,
width: metadata?.width ?? undefined,
height: metadata?.height ?? undefined,
content,
thumbnail: thumbnailContent,
});
if (config.media.conversion.convert_images) {
@ -215,6 +211,11 @@ export class Media extends BaseInterface<typeof Medias> {
});
}
await mediaQueue.add(MediaJobType.CalculateMetadata, {
attachmentId: newAttachment.id,
filename: file.name,
});
return newAttachment;
}
@ -232,105 +233,165 @@ export class Media extends BaseInterface<typeof Medias> {
return "";
}
public getUrl(): string {
const type = this.getPreferredMimeType();
return this.data.content[type]?.content;
}
/**
* Gets favourite MIME type for the attachment
* Uses a hardcoded list of preferred types, for images
*
* @returns {string} Preferred MIME type
*/
public getPreferredMimeType(): string {
return Media.getPreferredMimeType(Object.keys(this.data.content));
}
/**
* Gets favourite MIME type from a list
* Uses a hardcoded list of preferred types, for images
*
* @returns {string} Preferred MIME type
*/
public static getPreferredMimeType(types: string[]): string {
const ranking = [
"image/svg+xml",
"image/avif",
"image/jxl",
"image/webp",
"image/heif",
"image/heif-sequence",
"image/heic",
"image/heic-sequence",
"image/apng",
"image/gif",
"image/png",
"image/jpeg",
"image/bmp",
];
return ranking.find((type) => types.includes(type)) ?? types[0];
}
/**
* Maps MIME type to Mastodon attachment type
*
* @returns
*/
public getMastodonType(): ApiAttachment["type"] {
if (this.data.mimeType.startsWith("image/")) {
const type = this.getPreferredMimeType();
if (type.startsWith("image/")) {
return "image";
}
if (this.data.mimeType.startsWith("video/")) {
if (type.startsWith("video/")) {
return "video";
}
if (this.data.mimeType.startsWith("audio/")) {
if (type.startsWith("audio/")) {
return "audio";
}
return "unknown";
}
public toApiMeta(): ApiAttachment["meta"] {
/**
* Extracts metadata from a file and outputs as ContentFormat
*
* Does not calculate thumbhash (do this in a worker)
* @param file
* @param uri Uploaded file URI
* @param options Extra metadata, such as description
* @returns
*/
public static async fileToContentFormat(
file: File,
uri: string,
options?: Partial<{
description: string;
}>,
): Promise<ContentFormat> {
const buffer = await file.arrayBuffer();
const isImage = file.type.startsWith("image/");
const { width, height } = isImage ? await sharp(buffer).metadata() : {};
const hash = new SHA256().update(file).digest("hex");
// Missing: fps, duration
// Thumbhash should be added in a worker after the file is uploaded
return {
width: this.data.width || undefined,
height: this.data.height || undefined,
fps: this.data.fps || undefined,
size:
this.data.width && this.data.height
? `${this.data.width}x${this.data.height}`
: undefined,
duration: this.data.duration || undefined,
length: undefined,
aspect:
this.data.width && this.data.height
? this.data.width / this.data.height
: undefined,
original: {
width: this.data.width || undefined,
height: this.data.height || undefined,
size:
this.data.width && this.data.height
? `${this.data.width}x${this.data.height}`
: undefined,
aspect:
this.data.width && this.data.height
? this.data.width / this.data.height
: undefined,
[file.type]: {
content: uri,
remote: true,
hash: {
sha256: hash,
},
width,
height,
description: options?.description,
size: file.size,
},
};
}
public toApiMeta(): ApiAttachment["meta"] {
const type = this.getPreferredMimeType();
const data = this.data.content[type];
const size =
data.width && data.height
? `${data.width}x${data.height}`
: undefined;
const aspect =
data.width && data.height ? data.width / data.height : undefined;
return {
width: data.width || undefined,
height: data.height || undefined,
fps: data.fps || undefined,
size,
// Idk whether size or length is the right value
duration: data.duration || undefined,
// Versia doesn't have a concept of length in ContentFormat
length: undefined,
aspect,
original: {
width: data.width || undefined,
height: data.height || undefined,
size,
aspect,
},
};
}
public toApi(): ApiAttachment {
const type = this.getPreferredMimeType();
const data = this.data.content[type];
// Thumbnail should only have a single MIME type
const thumbnailData =
this.data.thumbnail?.[Object.keys(this.data.thumbnail)[0]];
return {
id: this.data.id,
type: this.getMastodonType(),
url: proxyUrl(this.data.url) ?? "",
remote_url: proxyUrl(this.data.remoteUrl),
preview_url: proxyUrl(this.data.thumbnailUrl || this.data.url),
url: proxyUrl(data.content) ?? "",
remote_url: null,
preview_url: proxyUrl(thumbnailData?.content),
text_url: null,
meta: this.toApiMeta(),
description: this.data.description,
description: data.description || null,
blurhash: this.data.blurhash,
};
}
public toVersia(): ContentFormat {
return {
[this.data.mimeType]: {
content: this.data.url,
remote: true,
// TODO: Replace BlurHash with thumbhash
// thumbhash: this.data.blurhash ?? undefined,
description: this.data.description ?? undefined,
duration: this.data.duration ?? undefined,
fps: this.data.fps ?? undefined,
height: this.data.height ?? undefined,
size: this.data.size ?? undefined,
hash: this.data.sha256
? {
sha256: this.data.sha256,
}
: undefined,
width: this.data.width ?? undefined,
},
};
return this.data.content;
}
public static fromVersia(
attachmentToConvert: ContentFormat,
): Promise<Media> {
const key = Object.keys(attachmentToConvert)[0];
const value = attachmentToConvert[key];
public static fromVersia(contentFormat: ContentFormat): Promise<Media> {
return Media.insert({
mimeType: key,
url: value.content,
description: value.description || undefined,
duration: value.duration || undefined,
fps: value.fps || undefined,
height: value.height || undefined,
// biome-ignore lint/style/useExplicitLengthCheck: Biome thinks we're checking if size is not zero
size: value.size || undefined,
width: value.width || undefined,
sha256: value.hash?.sha256 || undefined,
// blurhash: value.blurhash || undefined,
content: contentFormat,
originalContent: contentFormat,
});
}
}

2
classes/database/note.ts
View file

@ -44,9 +44,9 @@ import {
import { config } from "~/packages/config-manager";
import { DeliveryJobType, deliveryQueue } from "../queues/delivery.ts";
import { Application } from "./application.ts";
import { Media } from "./attachment.ts";
import { BaseInterface } from "./base.ts";
import { Emoji } from "./emoji.ts";
import { Media } from "./media.ts";
import { User } from "./user.ts";
type NoteType = InferSelectModel<typeof Notes>;

1
classes/queues/media.ts
View file

@ -3,6 +3,7 @@ import { connection } from "~/utils/redis.ts";
export enum MediaJobType {
ConvertMedia = "convertMedia",
CalculateMetadata = "calculateMetadata",
}
export type MediaJobData = {

76
classes/workers/media.ts
View file

@ -30,24 +30,15 @@ export const getMediaWorker = (): Worker<MediaJobData, void, MediaJobType> =>
}
const processor = new ImageConversionPreprocessor(config);
const blurhashProcessor = new BlurhashPreprocessor();
const hash = attachment?.data.sha256;
if (!hash) {
throw new Error(
`Attachment [${attachmentId}] has no hash, cannot process.`,
);
}
await job.log(`Processing attachment [${attachmentId}]`);
await job.log(
`Fetching file from [${attachment.data.url}]`,
`Fetching file from [${attachment.getUrl()}]`,
);
// Download the file and process it.
const blob = await (
await fetch(attachment.data.url)
await fetch(attachment.getUrl())
).blob();
const file = new File([blob], filename);
@ -57,10 +48,6 @@ export const getMediaWorker = (): Worker<MediaJobData, void, MediaJobType> =>
const { file: processedFile } =
await processor.process(file);
await job.log(`Generating blurhash for [${attachmentId}]`);
const { blurhash } = await blurhashProcessor.process(file);
const mediaManager = new MediaManager(config);
await job.log(`Uploading attachment [${attachmentId}]`);
@ -70,21 +57,66 @@ export const getMediaWorker = (): Worker<MediaJobData, void, MediaJobType> =>
const url = Media.getUrl(path);
const sha256 = new Bun.SHA256();
await attachment.update({
content: await Media.fileToContentFormat(
uploadedFile,
url,
{
description:
attachment.data.content[0].description ||
undefined,
},
),
});
await job.log(
`✔ Finished processing attachment [${attachmentId}]`,
);
break;
}
case MediaJobType.CalculateMetadata: {
// Calculate blurhash
const { attachmentId } = job.data;
await job.log(`Fetching attachment ID [${attachmentId}]`);
const attachment = await Media.fromId(attachmentId);
if (!attachment) {
throw new Error(
`Attachment not found: [${attachmentId}]`,
);
}
const blurhashProcessor = new BlurhashPreprocessor();
await job.log(`Processing attachment [${attachmentId}]`);
await job.log(
`Fetching file from [${attachment.getUrl()}]`,
);
// Download the file and process it.
const blob = await (
await fetch(attachment.getUrl())
).blob();
// Filename is not important for blurhash
const file = new File([blob], "");
await job.log(`Generating blurhash for [${attachmentId}]`);
const { blurhash } = await blurhashProcessor.process(file);
await attachment.update({
url,
sha256: sha256
.update(await uploadedFile.arrayBuffer())
.digest("hex"),
mimeType: uploadedFile.type,
size: uploadedFile.size,
blurhash,
});
await job.log(
`✔ Finished processing attachment [${attachmentId}]`,
);
break;
}
}
},

14
drizzle/schema.ts
View file

@ -305,18 +305,10 @@ export const Tokens = pgTable("Tokens", {
export const Medias = pgTable("Medias", {
id: id(),
url: text("url").notNull(),
remoteUrl: text("remote_url"),
thumbnailUrl: text("thumbnail_url"),
mimeType: text("mime_type").notNull(),
description: text("description"),
content: jsonb("content").notNull().$type<ContentFormat>(),
originalContent: jsonb("original_content").$type<ContentFormat>(),
thumbnail: jsonb("thumbnail").$type<ContentFormat>(),
blurhash: text("blurhash"),
sha256: text("sha256"),
fps: integer("fps"),
duration: integer("duration"),
width: integer("width"),
height: integer("height"),
size: integer("size"),
noteId: uuid("noteId").references(() => Notes.id, {
onDelete: "cascade",
onUpdate: "cascade",

2
packages/plugin-kit/exports/db.ts
View file

@ -1,7 +1,7 @@
// biome-ignore lint/performance/noBarrelFile: <explanation>
export { User } from "~/classes/database/user.ts";
export { Role } from "~/classes/database/role.ts";
export { Media } from "~/classes/database/attachment.ts";
export { Media } from "~/classes/database/media";
export { Emoji } from "~/classes/database/emoji.ts";
export { Instance } from "~/classes/database/instance.ts";
export { Note } from "~/classes/database/note.ts";