versia-pub/server
2
0
Fork 0
mirror of https://github.com/versia-pub/server.git synced 2025-12-06 08:28:19 +01:00
Code Issues Actions Packages Projects Releases Wiki Activity

refactor(media): ♻️ Massively simplify media pipeline with Bun.S3

Browse source
This commit is contained in:
Jesse Wierzbinski 2025-01-29 17:21:40 +01:00
parent 29cbe7d293
commit 9ba6237f13
No known key found for this signature in database
21 changed files with 197 additions and 1005 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
.vscode/settings.json vendored
View file

@ -7,7 +7,8 @@
"federation",
"config",
"plugin",
"worker"
"worker",
"media"
],
"languageToolLinter.languageTool.ignoredWordsInWorkspace": ["versia"]
}

12
api/api/v1/emojis/:id/index.ts
View file

@ -1,12 +1,10 @@
import { apiRoute, auth, emojiValidator, jsonOrForm } from "@/api";
import { mimeLookup } from "@/content_types";
import { createRoute } from "@hono/zod-openapi";
import { Emoji, db } from "@versia/kit/db";
import { Emojis, RolePermissions } from "@versia/kit/tables";
import { eq } from "drizzle-orm";
import { Emoji } from "@versia/kit/db";
import { RolePermissions } from "@versia/kit/tables";
import { z } from "zod";
import { ApiError } from "~/classes/errors/api-error";
import { MediaManager } from "~/classes/media/media-manager";
import { config } from "~/packages/config-manager";
import { ErrorSchema } from "~/types/api";
@ -306,11 +304,7 @@ export default apiRoute((app) => {
);
}
const mediaManager = new MediaManager(config);
await mediaManager.deleteFileByUrl(emoji.media.getUrl());
await db.delete(Emojis).where(eq(Emojis.id, id));
await emoji.delete();
return context.body(null, 204);
});

62
classes/database/media.ts
View file

@ -1,10 +1,11 @@
import { join } from "node:path";
import { mimeLookup } from "@/content_types.ts";
import { proxyUrl } from "@/response";
import type { Attachment as ApiAttachment } from "@versia/client/types";
import type { ContentFormat } from "@versia/federation/types";
import { db } from "@versia/kit/db";
import { Medias } from "@versia/kit/tables";
import { SHA256 } from "bun";
import { S3Client, SHA256, randomUUIDv7, write } from "bun";
import {
type InferInsertModel,
type InferSelectModel,
@ -18,7 +19,7 @@ import { z } from "zod";
import { MediaBackendType } from "~/packages/config-manager/config.type";
import { config } from "~/packages/config-manager/index.ts";
import { ApiError } from "../errors/api-error.ts";
import { MediaManager } from "../media/media-manager.ts";
import { getMediaHash } from "../media/media-hasher.ts";
import { MediaJobType, mediaQueue } from "../queues/media.ts";
import { BaseInterface } from "./base.ts";
@ -154,6 +155,47 @@ export class Media extends BaseInterface<typeof Medias> {
return attachment;
}
private static async upload(file: File): Promise<{
path: string;
}> {
const fileName = file.name ?? randomUUIDv7();
const hash = await getMediaHash(file);
switch (config.media.backend) {
case MediaBackendType.Local: {
const path = join(
config.media.local_uploads_folder,
hash,
fileName,
);
await write(path, file);
return { path: join(hash, fileName) };
}
case MediaBackendType.S3: {
const path = join(hash, fileName);
if (!config.s3) {
throw new ApiError(500, "S3 configuration missing");
}
const client = new S3Client({
endpoint: config.s3.endpoint,
region: config.s3.region,
bucket: config.s3.bucket_name,
accessKeyId: config.s3.access_key,
secretAccessKey: config.s3.secret_access_key,
});
await client.write(path, file);
return { path };
}
}
}
public static async fromFile(
file: File,
options?: {
@ -163,16 +205,14 @@ export class Media extends BaseInterface<typeof Medias> {
): Promise<Media> {
Media.checkFile(file);
const mediaManager = new MediaManager(config);
const { path } = await mediaManager.addFile(file);
const { path } = await Media.upload(file);
const url = Media.getUrl(path);
let thumbnailUrl = "";
if (options?.thumbnail) {
const { path } = await mediaManager.addFile(options.thumbnail);
const { path } = await Media.upload(options.thumbnail);
thumbnailUrl = Media.getUrl(path);
}
@ -259,9 +299,7 @@ export class Media extends BaseInterface<typeof Medias> {
public async updateFromFile(file: File): Promise<void> {
Media.checkFile(file);
const mediaManager = new MediaManager(config);
const { path } = await mediaManager.addFile(file);
const { path } = await Media.upload(file);
const url = Media.getUrl(path);
@ -307,9 +345,7 @@ export class Media extends BaseInterface<typeof Medias> {
public async updateThumbnail(file: File): Promise<void> {
Media.checkFile(file);
const mediaManager = new MediaManager(config);
const { path } = await mediaManager.addFile(file);
const { path } = await Media.upload(file);
const url = Media.getUrl(path);
@ -346,7 +382,7 @@ export class Media extends BaseInterface<typeof Medias> {
return new URL(`/media/${name}`, config.http.base_url).toString();
}
if (config.media.backend === MediaBackendType.S3) {
return new URL(`/${name}`, config.s3.public_url).toString();
return new URL(`/${name}`, config.s3?.public_url).toString();
}
return "";
}

12
classes/database/notification.ts
View file

@ -10,8 +10,6 @@ import {
inArray,
} from "drizzle-orm";
import { z } from "zod";
import { MediaBackendType } from "~/packages/config-manager/config.type";
import { config } from "~/packages/config-manager/index.ts";
import {
transformOutputToUserWithRelations,
userExtrasTemplate,
@ -215,16 +213,6 @@ export class Notification extends BaseInterface<
return this.data.id;
}
public static getUrl(name: string): string {
if (config.media.backend === MediaBackendType.Local) {
return new URL(`/media/${name}`, config.http.base_url).toString();
}
if (config.media.backend === MediaBackendType.S3) {
return new URL(`/${name}`, config.s3.public_url).toString();
}
return "";
}
public async toApi(): Promise<APINotification> {
const account = new User(this.data.account);

136
classes/media/drivers/disk.test.ts
View file

@ -1,136 +0,0 @@
/**
* @packageDocumentation
* @module Tests/DiskMediaDriver
*/
import {
type Mock,
beforeEach,
describe,
expect,
it,
mock,
spyOn,
} from "bun:test";
import { rm } from "node:fs/promises";
import { join } from "node:path";
import type { Config } from "~/packages/config-manager/config.type";
import type { getMediaHash } from "../media-hasher.ts";
import { DiskMediaDriver } from "./disk.ts";
describe("DiskMediaDriver", () => {
let diskDriver: DiskMediaDriver;
let mockConfig: Config;
let mockMediaHasher: Mock<typeof getMediaHash>;
let bunWriteSpy: Mock<typeof Bun.write>;
beforeEach(() => {
mockConfig = {
media: {
local_uploads_folder: "/test/uploads",
},
http: {
base_url: "http://localhost:3000",
},
} as Config;
mockMediaHasher = mock(() => Promise.resolve("testhash"));
mock.module("../media-hasher", () => ({
getMediaHash: mockMediaHasher,
}));
diskDriver = new DiskMediaDriver(mockConfig);
// @ts-expect-error: Replacing private property for testing
diskDriver.mediaHasher = mockMediaHasher;
// Mock fs.promises methods
mock.module("node:fs/promises", () => ({
writeFile: mock(() => Promise.resolve()),
rm: mock(() => {
return Promise.resolve();
}),
}));
spyOn(Bun, "file").mockImplementation(
mock(() => ({
exists: mock(() => Promise.resolve(true)),
arrayBuffer: mock(() => Promise.resolve(new ArrayBuffer(8))),
type: "image/webp",
lastModified: Date.now(),
})) as unknown as typeof Bun.file,
);
bunWriteSpy = spyOn(Bun, "write").mockImplementation(
mock(() => Promise.resolve(0)),
);
});
it("should add a file", async () => {
const file = new File(["test"], "test.webp", { type: "image/webp" });
const result = await diskDriver.addFile(file);
expect(mockMediaHasher).toHaveBeenCalledWith(file);
expect(bunWriteSpy).toHaveBeenCalledWith(
join("/test/uploads", "testhash", "test.webp"),
expect.any(ArrayBuffer),
);
expect(result).toEqual({
uploadedFile: file,
path: join("testhash", "test.webp"),
hash: "testhash",
});
});
it("should properly handle a Blob instead of a File", async () => {
const file = new Blob(["test"], { type: "image/webp" });
const result = await diskDriver.addFile(file as File);
expect(mockMediaHasher).toHaveBeenCalledWith(file);
expect(bunWriteSpy).toHaveBeenCalledWith(
expect.stringContaining("testhash"),
expect.any(ArrayBuffer),
);
expect(result).toEqual({
uploadedFile: expect.any(Blob),
path: expect.stringContaining("testhash"),
hash: "testhash",
});
});
it("should get a file by hash", async () => {
const hash = "testhash";
const databaseHashFetcher = mock(() => Promise.resolve("test.webp"));
const result = await diskDriver.getFileByHash(
hash,
databaseHashFetcher,
);
expect(databaseHashFetcher).toHaveBeenCalledWith(hash);
expect(Bun.file).toHaveBeenCalledWith(
join("/test/uploads", "test.webp"),
);
expect(result).toBeInstanceOf(File);
expect(result?.name).toBe("test.webp");
expect(result?.type).toBe("image/webp");
});
it("should get a file by filename", async () => {
const filename = "test.webp";
const result = await diskDriver.getFile(filename);
expect(Bun.file).toHaveBeenCalledWith(join("/test/uploads", filename));
expect(result).toBeInstanceOf(File);
expect(result?.name).toBe(filename);
expect(result?.type).toBe("image/webp");
});
it("should delete a file by URL", async () => {
const url = "http://localhost:3000/uploads/testhash/test.webp";
await diskDriver.deleteFileByUrl(url);
expect(rm).toHaveBeenCalledWith(join("/test/uploads", "testhash"), {
recursive: true,
});
});
});

96
classes/media/drivers/disk.ts
View file

@ -1,96 +0,0 @@
/**
* @packageDocumentation
* @module MediaManager/Drivers
*/
import { rm } from "node:fs/promises";
import { join } from "node:path";
import type { Config } from "~/packages/config-manager/config.type";
import { getMediaHash } from "../media-hasher.ts";
import type { UploadedFileMetadata } from "../media-manager.ts";
import type { MediaDriver } from "./media-driver.ts";
/**
* Implements the MediaDriver interface for disk storage.
*/
export class DiskMediaDriver implements MediaDriver {
/**
* Creates a new DiskMediaDriver instance.
* @param config - The configuration object.
*/
public constructor(private config: Config) {}
/**
* @inheritdoc
*/
public async addFile(
file: File,
): Promise<Omit<UploadedFileMetadata, "blurhash">> {
// Sometimes the file name is not available, so we generate a random name
const fileName = file.name ?? crypto.randomUUID();
const hash = await getMediaHash(file);
const path = join(hash, fileName);
const fullPath = join(this.config.media.local_uploads_folder, path);
await Bun.write(fullPath, await file.arrayBuffer());
return {
uploadedFile: file,
path,
hash,
};
}
/**
* @inheritdoc
*/
public async getFileByHash(
hash: string,
databaseHashFetcher: (sha256: string) => Promise<string | null>,
): Promise<File | null> {
const filename = await databaseHashFetcher(hash);
if (!filename) {
return null;
}
return this.getFile(filename);
}
/**
* @inheritdoc
*/
public async getFile(filename: string): Promise<File | null> {
const fullPath = join(this.config.media.local_uploads_folder, filename);
try {
const file = Bun.file(fullPath);
if (await file.exists()) {
return new File([await file.arrayBuffer()], filename, {
type: file.type,
lastModified: file.lastModified,
});
}
} catch {
// File doesn't exist or can't be read
}
return null;
}
/**
* @inheritdoc
*/
public async deleteFileByUrl(url: string): Promise<void> {
const urlObj = new URL(url);
// Check if URL is from the local uploads folder
if (urlObj.host !== new URL(this.config.http.base_url).host) {
return Promise.resolve();
}
const hash = urlObj.pathname.split("/").at(-2);
if (!hash) {
throw new Error("Invalid URL");
}
const dirPath = join(this.config.media.local_uploads_folder, hash);
await rm(dirPath, { recursive: true });
}
}

43
classes/media/drivers/media-driver.ts
View file

@ -1,43 +0,0 @@
/**
* @packageDocumentation
* @module MediaManager/Drivers
*/
import type { UploadedFileMetadata } from "../media-manager.ts";
/**
* Represents a media storage driver.
*/
export interface MediaDriver {
/**
* Adds a file to the media storage.
* @param file - The file to add.
* @returns A promise that resolves to the metadata of the uploaded file.
*/
addFile(file: File): Promise<Omit<UploadedFileMetadata, "blurhash">>;
/**
* Retrieves a file from the media storage by its hash.
* @param hash - The hash of the file to retrieve.
* @param databaseHashFetcher - A function to fetch the filename from the database.
* @returns A promise that resolves to the file or null if not found.
*/
getFileByHash(
hash: string,
databaseHashFetcher: (sha256: string) => Promise<string | null>,
): Promise<File | null>;
/**
* Retrieves a file from the media storage by its filename.
* @param filename - The name of the file to retrieve.
* @returns A promise that resolves to the file or null if not found.
*/
getFile(filename: string): Promise<File | null>;
/**
* Deletes a file from the media storage by its URL.
* @param url - The URL of the file to delete.
* @returns A promise that resolves when the file is deleted.
*/
deleteFileByUrl(url: string): Promise<void>;
}

126
classes/media/drivers/s3.test.ts
View file

@ -1,126 +0,0 @@
/**
* @packageDocumentation
* @module Tests/S3MediaDriver
*/
import { type Mock, beforeEach, describe, expect, it, mock } from "bun:test";
import type { S3Client } from "@bradenmacdonald/s3-lite-client";
import type { Config } from "~/packages/config-manager/config.type";
import type { getMediaHash } from "../media-hasher.ts";
import { S3MediaDriver } from "./s3.ts";
describe("S3MediaDriver", () => {
let s3Driver: S3MediaDriver;
let mockConfig: Config;
let mockS3Client: S3Client;
let mockMediaHasher: Mock<typeof getMediaHash>;
beforeEach(() => {
mockConfig = {
s3: {
endpoint: "s3.amazonaws.com",
region: "us-west-2",
bucket_name: "test-bucket",
access_key: "test-key",
secret_access_key: "test-secret",
},
} as Config;
mockS3Client = mock(() => ({
putObject: mock(() => Promise.resolve()),
getObject: mock(() =>
Promise.resolve({
arrayBuffer: (): Promise<ArrayBuffer> =>
Promise.resolve(new ArrayBuffer(8)),
headers: new Headers({ "Content-Type": "image/webp" }),
}),
),
statObject: mock(() => Promise.resolve()),
deleteObject: mock(() => Promise.resolve()),
}))() as unknown as S3Client;
mockMediaHasher = mock(() => Promise.resolve("testhash"));
mock.module("../media-hasher", () => ({
getMediaHash: mockMediaHasher,
}));
s3Driver = new S3MediaDriver(mockConfig);
// @ts-expect-error: Replacing private property for testing
s3Driver.s3Client = mockS3Client;
// @ts-expect-error: Replacing private property for testing
s3Driver.mediaHasher = mockMediaHasher;
});
it("should add a file", async () => {
const file = new File(["test"], "test.webp", { type: "image/webp" });
const result = await s3Driver.addFile(file);
expect(mockMediaHasher).toHaveBeenCalledWith(file);
expect(mockS3Client.putObject).toHaveBeenCalledWith(
"testhash/test.webp",
expect.any(ReadableStream),
{ size: file.size, metadata: { "Content-Type": file.type } },
);
expect(result).toEqual({
uploadedFile: file,
path: "testhash/test.webp",
hash: "testhash",
});
});
it("should handle a Blob instead of a File", async () => {
const file = new Blob(["test"], { type: "image/webp" });
const result = await s3Driver.addFile(file as File);
expect(mockMediaHasher).toHaveBeenCalledWith(file);
expect(mockS3Client.putObject).toHaveBeenCalledWith(
expect.stringContaining("testhash"),
expect.any(ReadableStream),
{
size: file.size,
metadata: {
"Content-Type": file.type,
},
},
);
expect(result).toEqual({
uploadedFile: expect.any(Blob),
path: expect.stringContaining("testhash"),
hash: "testhash",
});
});
it("should get a file by hash", async () => {
const hash = "testhash";
const databaseHashFetcher = mock(() => Promise.resolve("test.webp"));
const result = await s3Driver.getFileByHash(hash, databaseHashFetcher);
expect(databaseHashFetcher).toHaveBeenCalledWith(hash);
expect(mockS3Client.statObject).toHaveBeenCalledWith("test.webp");
expect(mockS3Client.getObject).toHaveBeenCalledWith("test.webp");
expect(result).toBeInstanceOf(File);
expect(result?.name).toBe("test.webp");
expect(result?.type).toBe("image/webp");
});
it("should get a file by filename", async () => {
const filename = "test.webp";
const result = await s3Driver.getFile(filename);
expect(mockS3Client.statObject).toHaveBeenCalledWith(filename);
expect(mockS3Client.getObject).toHaveBeenCalledWith(filename);
expect(result).toBeInstanceOf(File);
expect(result?.name).toBe(filename);
expect(result?.type).toBe("image/webp");
});
it("should delete a file by URL", async () => {
const url = "https://test-bucket.s3.amazonaws.com/test/test.webp";
await s3Driver.deleteFileByUrl(url);
expect(mockS3Client.deleteObject).toHaveBeenCalledWith(
"test/test.webp",
);
});
});

97
classes/media/drivers/s3.ts
View file

@ -1,97 +0,0 @@
/**
* @packageDocumentation
* @module MediaManager/Drivers
*/
import { S3Client } from "@bradenmacdonald/s3-lite-client";
import type { Config } from "~/packages/config-manager/config.type";
import { getMediaHash } from "../media-hasher.ts";
import type { UploadedFileMetadata } from "../media-manager.ts";
import type { MediaDriver } from "./media-driver.ts";
/**
* Implements the MediaDriver interface for S3 storage.
*/
export class S3MediaDriver implements MediaDriver {
private s3Client: S3Client;
/**
* Creates a new S3MediaDriver instance.
* @param config - The configuration object.
*/
public constructor(config: Config) {
this.s3Client = new S3Client({
endPoint: config.s3.endpoint,
useSSL: true,
region: config.s3.region || "auto",
bucket: config.s3.bucket_name,
accessKey: config.s3.access_key,
secretKey: config.s3.secret_access_key,
});
}
/**
* @inheritdoc
*/
public async addFile(
file: File,
): Promise<Omit<UploadedFileMetadata, "blurhash">> {
// Sometimes the file name is not available, so we generate a random name
const fileName = file.name ?? crypto.randomUUID();
const hash = await getMediaHash(file);
const path = `${hash}/${fileName}`;
await this.s3Client.putObject(path, file.stream(), {
size: file.size,
metadata: {
"Content-Type": file.type,
},
});
return {
uploadedFile: file,
path,
hash,
};
}
/**
* @inheritdoc
*/
public async getFileByHash(
hash: string,
databaseHashFetcher: (sha256: string) => Promise<string | null>,
): Promise<File | null> {
const filename = await databaseHashFetcher(hash);
if (!filename) {
return null;
}
return this.getFile(filename);
}
/**
* @inheritdoc
*/
public async getFile(filename: string): Promise<File | null> {
try {
await this.s3Client.statObject(filename);
const file = await this.s3Client.getObject(filename);
const arrayBuffer = await file.arrayBuffer();
return new File([arrayBuffer], filename, {
type: file.headers.get("Content-Type") || undefined,
});
} catch {
return null;
}
}
/**
* @inheritdoc
*/
public async deleteFileByUrl(url: string): Promise<void> {
const urlObj = new URL(url);
const path = urlObj.pathname.slice(1); // Remove leading slash
await this.s3Client.deleteObject(path);
}
}

123
classes/media/media-manager.test.ts
View file

@ -1,123 +0,0 @@
/**
* @packageDocumentation
* @module Tests/MediaManager
*/
import { beforeEach, describe, expect, it, mock } from "bun:test";
import type { Config } from "~/packages/config-manager/config.type";
import { MediaBackendType } from "~/packages/config-manager/config.type";
import { DiskMediaDriver } from "./drivers/disk.ts";
import { S3MediaDriver } from "./drivers/s3.ts";
import { MediaManager } from "./media-manager.ts";
import type { ImageConversionPreprocessor } from "./preprocessors/image-conversion.ts";
describe("MediaManager", () => {
let mediaManager: MediaManager;
let mockConfig: Config;
let mockS3Driver: S3MediaDriver;
let mockImagePreprocessor: ImageConversionPreprocessor;
beforeEach(() => {
mockConfig = {
media: {
backend: "s3",
conversion: {
convert_images: true,
convert_to: "image/webp",
},
},
s3: {
endpoint: "s3.amazonaws.com",
region: "us-west-2",
bucket_name: "test-bucket",
access_key: "test-key",
secret_access_key: "test-secret",
},
} as Config;
mockS3Driver = mock(() => ({
addFile: mock(() =>
Promise.resolve({
uploadedFile: new File(["hey"], "test.webp"),
path: "test/test.webp",
hash: "testhash",
}),
),
getFileByHash: mock(() => {
return Promise.resolve(new File(["hey"], "test.webp"));
}),
getFile: mock(() =>
Promise.resolve(new File(["hey"], "test.webp")),
),
deleteFileByUrl: mock(() => Promise.resolve()),
}))() as unknown as S3MediaDriver;
mockImagePreprocessor = mock(() => ({
process: mock((_: File) =>
Promise.resolve(new File(["hey"], "test.webp")),
),
}))() as unknown as ImageConversionPreprocessor;
mediaManager = new MediaManager(mockConfig);
// @ts-expect-error: Accessing private property for testing
mediaManager.driver = mockS3Driver;
// @ts-expect-error: Accessing private property for testing
mediaManager.preprocessors = [mockImagePreprocessor];
});
it("should initialize with the correct driver based on config", () => {
const s3Manager = new MediaManager(mockConfig);
// @ts-expect-error: Accessing private property for testing
expect(s3Manager.driver).toBeInstanceOf(S3MediaDriver);
mockConfig.media.backend = MediaBackendType.Local;
const diskManager = new MediaManager(mockConfig);
// @ts-expect-error: Accessing private property for testing
expect(diskManager.driver).toBeInstanceOf(DiskMediaDriver);
});
it("should add a file with preprocessing", async () => {
const file = new File(["test"], "test.jpg", { type: "image/jpeg" });
const result = await mediaManager.addFile(file);
expect(mockImagePreprocessor.process).toHaveBeenCalledWith(file);
expect(mockS3Driver.addFile).toHaveBeenCalled();
expect(result).toEqual({
uploadedFile: new File(["hey"], "test.webp"),
path: "test/test.webp",
hash: "testhash",
});
});
it("should get a file by hash", async () => {
const hash = "testhash";
const databaseHashFetcher = mock(() => Promise.resolve("test.webp"));
const result = await mediaManager.getFileByHash(
hash,
databaseHashFetcher,
);
expect(mockS3Driver.getFileByHash).toHaveBeenCalledWith(
hash,
databaseHashFetcher,
);
expect(result).toBeInstanceOf(File);
expect(result?.name).toBe("test.webp");
});
it("should get a file by filename", async () => {
const filename = "test.webp";
const result = await mediaManager.getFile(filename);
expect(mockS3Driver.getFile).toHaveBeenCalledWith(filename);
expect(result).toBeInstanceOf(File);
expect(result?.name).toBe("test.webp");
});
it("should delete a file by URL", async () => {
const url = "https://test-bucket.s3.amazonaws.com/test/test.webp";
await mediaManager.deleteFileByUrl(url);
expect(mockS3Driver.deleteFileByUrl).toHaveBeenCalledWith(url);
});
});

111
classes/media/media-manager.ts
View file

@ -1,111 +0,0 @@
/**
* @packageDocumentation
* @module MediaManager
*/
import type { Config } from "~/packages/config-manager/config.type";
import { DiskMediaDriver } from "./drivers/disk.ts";
import type { MediaDriver } from "./drivers/media-driver.ts";
import { S3MediaDriver } from "./drivers/s3.ts";
import type { MediaPreprocessor } from "./preprocessors/media-preprocessor.ts";
/**
* Manages media operations with support for different storage drivers and preprocessing plugins.
* @example
* const mediaManager = new MediaManager(config);
*
* const file = new File(["hello"], "hello.txt");
*
* const { path, hash, blurhash } = await mediaManager.addFile(file);
*
* const retrievedFile = await mediaManager.getFileByHash(hash, fetchHashFromDatabase);
*
* await mediaManager.deleteFileByUrl(path);
*/
export class MediaManager {
private driver: MediaDriver;
private preprocessors: MediaPreprocessor[] = [];
/**
* Creates a new MediaManager instance.
* @param config - The configuration object.
*/
public constructor(private config: Config) {
this.driver = this.initializeDriver();
}
/**
* Initializes the appropriate media driver based on the configuration.
* @returns An instance of MediaDriver.
*/
private initializeDriver(): MediaDriver {
switch (this.config.media.backend) {
case "s3":
return new S3MediaDriver(this.config);
case "local":
return new DiskMediaDriver(this.config);
default:
throw new Error(
`Unsupported media backend: ${this.config.media.backend}`,
);
}
}
/**
* Adds a file to the media storage.
* @param file - The file to add.
* @returns A promise that resolves to the metadata of the uploaded file.
*/
public async addFile(file: File): Promise<UploadedFileMetadata> {
let processedFile = file;
for (const preprocessor of this.preprocessors) {
const result = await preprocessor.process(processedFile);
processedFile = result.file;
}
const uploadResult = await this.driver.addFile(processedFile);
return uploadResult;
}
/**
* Retrieves a file from the media storage by its hash.
* @param hash - The hash of the file to retrieve.
* @param databaseHashFetcher - A function to fetch the filename from the database.
* @returns A promise that resolves to the file or null if not found.
*/
public getFileByHash(
hash: string,
databaseHashFetcher: (sha256: string) => Promise<string | null>,
): Promise<File | null> {
return this.driver.getFileByHash(hash, databaseHashFetcher);
}
/**
* Retrieves a file from the media storage by its filename.
* @param filename - The name of the file to retrieve.
* @returns A promise that resolves to the file or null if not found.
*/
public getFile(filename: string): Promise<File | null> {
return this.driver.getFile(filename);
}
/**
* Deletes a file from the media storage by its URL.
* @param url - The URL of the file to delete.
* @returns A promise that resolves when the file is deleted.
*/
public deleteFileByUrl(url: string): Promise<void> {
return this.driver.deleteFileByUrl(url);
}
}
/**
* Represents the metadata of an uploaded file.
*/
export interface UploadedFileMetadata {
uploadedFile: File;
path: string;
hash: string;
}

27
classes/media/preprocessors/blurhash.test.ts
View file

@ -1,14 +1,8 @@
import { beforeEach, describe, expect, it, mock } from "bun:test";
import { describe, expect, it, mock } from "bun:test";
import sharp from "sharp";
import { BlurhashPreprocessor } from "./blurhash.ts";
import { calculateBlurhash } from "./blurhash.ts";
describe("BlurhashPreprocessor", () => {
let preprocessor: BlurhashPreprocessor;
beforeEach(() => {
preprocessor = new BlurhashPreprocessor();
});
it("should calculate blurhash for a valid image", async () => {
const inputBuffer = await sharp({
create: {
@ -24,21 +18,19 @@ describe("BlurhashPreprocessor", () => {
const inputFile = new File([inputBuffer], "test.png", {
type: "image/png",
});
const result = await preprocessor.process(inputFile);
const result = await calculateBlurhash(inputFile);
expect(result.file).toBe(inputFile);
expect(result.blurhash).toBeTypeOf("string");
expect(result.blurhash).not.toBe("");
expect(result).toBeTypeOf("string");
expect(result).not.toBe("");
});
it("should return null blurhash for an invalid image", async () => {
const invalidFile = new File(["invalid image data"], "invalid.png", {
type: "image/png",
});
const result = await preprocessor.process(invalidFile);
const result = await calculateBlurhash(invalidFile);
expect(result.file).toBe(invalidFile);
expect(result.blurhash).toBeNull();
expect(result).toBeNull();
});
it("should handle errors during blurhash calculation", async () => {
@ -63,9 +55,8 @@ describe("BlurhashPreprocessor", () => {
},
}));
const result = await preprocessor.process(inputFile);
const result = await calculateBlurhash(inputFile);
expect(result.file).toBe(inputFile);
expect(result.blurhash).toBeNull();
expect(result).toBeNull();
});
});

15
classes/media/preprocessors/blurhash.ts
View file

@ -1,16 +1,12 @@
import { encode } from "blurhash";
import sharp from "sharp";
import type { MediaPreprocessor } from "./media-preprocessor.ts";
export class BlurhashPreprocessor implements MediaPreprocessor {
public async process(
file: File,
): Promise<{ file: File; blurhash: string | null }> {
export const calculateBlurhash = async (file: File): Promise<string | null> => {
try {
const arrayBuffer = await file.arrayBuffer();
const metadata = await sharp(arrayBuffer).metadata();
const blurhash = await new Promise<string | null>((resolve) => {
return new Promise<string | null>((resolve) => {
sharp(arrayBuffer)
.raw()
.ensureAlpha()
@ -35,10 +31,7 @@ export class BlurhashPreprocessor implements MediaPreprocessor {
}
});
});
return { file, blurhash };
} catch {
return { file, blurhash: null };
return null;
}
}
}
};

49
classes/media/preprocessors/image-conversion.test.ts
View file

@ -1,10 +1,9 @@
import { beforeEach, describe, expect, it } from "bun:test";
import { beforeEach, describe, expect, it, mock } from "bun:test";
import sharp from "sharp";
import type { Config } from "~/packages/config-manager/config.type";
import { ImageConversionPreprocessor } from "./image-conversion.ts";
import { convertImage } from "./image-conversion.ts";
describe("ImageConversionPreprocessor", () => {
let preprocessor: ImageConversionPreprocessor;
let mockConfig: Config;
beforeEach(() => {
@ -18,7 +17,9 @@ describe("ImageConversionPreprocessor", () => {
},
} as Config;
preprocessor = new ImageConversionPreprocessor(mockConfig);
mock.module("~/packages/config-manager/index.ts", () => ({
config: mockConfig,
}));
});
it("should convert a JPEG image to WebP", async () => {
@ -36,12 +37,12 @@ describe("ImageConversionPreprocessor", () => {
const inputFile = new File([inputBuffer], "test.jpg", {
type: "image/jpeg",
});
const result = await preprocessor.process(inputFile);
const result = await convertImage(inputFile);
expect(result.file.type).toBe("image/webp");
expect(result.file.name).toBe("test.webp");
expect(result.type).toBe("image/webp");
expect(result.name).toBe("test.webp");
const resultBuffer = await result.file.arrayBuffer();
const resultBuffer = await result.arrayBuffer();
const metadata = await sharp(resultBuffer).metadata();
expect(metadata.format).toBe("webp");
});
@ -52,38 +53,36 @@ describe("ImageConversionPreprocessor", () => {
const inputFile = new File([svgContent], "test.svg", {
type: "image/svg+xml",
});
const result = await preprocessor.process(inputFile);
const result = await convertImage(inputFile);
expect(result.file).toBe(inputFile);
expect(result).toBe(inputFile);
});
it("should convert SVG when convert_vector is true", async () => {
mockConfig.media.conversion.convert_vector = true;
preprocessor = new ImageConversionPreprocessor(mockConfig);
const svgContent =
'<svg xmlns="http://www.w3.org/2000/svg"><rect width="100" height="100" fill="red"/></svg>';
const inputFile = new File([svgContent], "test.svg", {
type: "image/svg+xml",
});
const result = await preprocessor.process(inputFile);
const result = await convertImage(inputFile);
expect(result.file.type).toBe("image/webp");
expect(result.file.name).toBe("test.webp");
expect(result.type).toBe("image/webp");
expect(result.name).toBe("test.webp");
});
it("should not convert unsupported file types", async () => {
const inputFile = new File(["test content"], "test.txt", {
type: "text/plain",
});
const result = await preprocessor.process(inputFile);
const result = await convertImage(inputFile);
expect(result.file).toBe(inputFile);
expect(result).toBe(inputFile);
});
it("should throw an error for unsupported output format", async () => {
mockConfig.media.conversion.convert_to = "image/bmp";
preprocessor = new ImageConversionPreprocessor(mockConfig);
const inputBuffer = await sharp({
create: {
@ -100,7 +99,7 @@ describe("ImageConversionPreprocessor", () => {
type: "image/png",
});
await expect(preprocessor.process(inputFile)).rejects.toThrow(
await expect(convertImage(inputFile)).rejects.toThrow(
"Unsupported output format: image/bmp",
);
});
@ -121,12 +120,12 @@ describe("ImageConversionPreprocessor", () => {
const inputFile = new File([inputBuffer], "animated.gif", {
type: "image/gif",
});
const result = await preprocessor.process(inputFile);
const result = await convertImage(inputFile);
expect(result.file.type).toBe("image/webp");
expect(result.file.name).toBe("animated.webp");
expect(result.type).toBe("image/webp");
expect(result.name).toBe("animated.webp");
const resultBuffer = await result.file.arrayBuffer();
const resultBuffer = await result.arrayBuffer();
const metadata = await sharp(resultBuffer).metadata();
expect(metadata.format).toBe("webp");
});
@ -148,9 +147,9 @@ describe("ImageConversionPreprocessor", () => {
"test image with spaces.png",
{ type: "image/png" },
);
const result = await preprocessor.process(inputFile);
const result = await convertImage(inputFile);
expect(result.file.type).toBe("image/webp");
expect(result.file.name).toBe("test image with spaces.webp");
expect(result.type).toBe("image/webp");
expect(result.name).toBe("test image with spaces.webp");
});
});

110
classes/media/preprocessors/image-conversion.ts
View file

@ -4,8 +4,7 @@
*/
import sharp from "sharp";
import type { Config } from "~/packages/config-manager/config.type";
import type { MediaPreprocessor } from "./media-preprocessor.ts";
import { config } from "~/packages/config-manager/index.ts";
/**
* Supported input media formats.
@ -33,24 +32,51 @@ const supportedOutputFormats = [
];
/**
* Implements the MediaPreprocessor interface for image conversion.
* Checks if a file is convertible.
* @param file - The file to check.
* @returns True if the file is convertible, false otherwise.
*/
export class ImageConversionPreprocessor implements MediaPreprocessor {
/**
* Creates a new ImageConversionPreprocessor instance.
* @param config - The configuration object.
*/
public constructor(private config: Config) {}
const isConvertible = (file: File): boolean => {
if (
file.type === "image/svg+xml" &&
!config.media.conversion.convert_vector
) {
return false;
}
return supportedInputFormats.includes(file.type);
};
/**
* @inheritdoc
/**
* Extracts the filename from a path.
* @param path - The path to extract the filename from.
* @returns The extracted filename.
*/
public async process(file: File): Promise<{ file: File }> {
if (!this.isConvertible(file)) {
return { file };
const extractFilenameFromPath = (path: string): string => {
const pathParts = path.split(/(?<!\\)\//);
return pathParts[pathParts.length - 1];
};
/**
* Replaces the file extension in the filename.
* @param fileName - The original filename.
* @param newExtension - The new extension.
* @returns The filename with the new extension.
*/
const getReplacedFileName = (fileName: string, newExtension: string): string =>
extractFilenameFromPath(fileName).replace(/\.[^/.]+$/, `.${newExtension}`);
/**
* Converts an image file to the format specified in the configuration.
*
* @param file - The image file to convert.
* @returns The converted image file.
*/
export const convertImage = async (file: File): Promise<File> => {
if (!isConvertible(file)) {
return file;
}
const targetFormat = this.config.media.conversion.convert_to;
const targetFormat = config.media.conversion.convert_to;
if (!supportedOutputFormats.includes(targetFormat)) {
throw new Error(`Unsupported output format: ${targetFormat}`);
}
@ -67,58 +93,12 @@ export class ImageConversionPreprocessor implements MediaPreprocessor {
| "tiff";
const convertedBuffer = await sharpCommand[commandName]().toBuffer();
return {
file: new File(
return new File(
[convertedBuffer],
ImageConversionPreprocessor.getReplacedFileName(
file.name,
commandName,
),
getReplacedFileName(file.name, commandName),
{
type: targetFormat,
lastModified: Date.now(),
},
),
};
}
/**
* Checks if a file is convertible.
* @param file - The file to check.
* @returns True if the file is convertible, false otherwise.
*/
private isConvertible(file: File): boolean {
if (
file.type === "image/svg+xml" &&
!this.config.media.conversion.convert_vector
) {
return false;
}
return supportedInputFormats.includes(file.type);
}
/**
* Replaces the file extension in the filename.
* @param fileName - The original filename.
* @param newExtension - The new extension.
* @returns The filename with the new extension.
*/
private static getReplacedFileName(
fileName: string,
newExtension: string,
): string {
return ImageConversionPreprocessor.extractFilenameFromPath(
fileName,
).replace(/\.[^/.]+$/, `.${newExtension}`);
}
/**
* Extracts the filename from a path.
* @param path - The path to extract the filename from.
* @returns The extracted filename.
*/
private static extractFilenameFromPath(path: string): string {
const pathParts = path.split(/(?<!\\)\//);
return pathParts[pathParts.length - 1];
}
}
);
};

16
classes/media/preprocessors/media-preprocessor.ts
View file

@ -1,16 +0,0 @@
/**
* @packageDocumentation
* @module MediaManager/Preprocessors
*/
/**
* Represents a media preprocessor.
*/
export interface MediaPreprocessor {
/**
* Processes a file before it's stored.
* @param file - The file to process.
* @returns A promise that resolves to the processed file.
*/
process(file: File): Promise<{ file: File } & Record<string, unknown>>;
}

33
classes/workers/media.ts
View file

@ -2,9 +2,8 @@ import { Media } from "@versia/kit/db";
import { Worker } from "bullmq";
import { config } from "~/packages/config-manager";
import { connection } from "~/utils/redis.ts";
import { MediaManager } from "../media/media-manager.ts";
import { BlurhashPreprocessor } from "../media/preprocessors/blurhash.ts";
import { ImageConversionPreprocessor } from "../media/preprocessors/image-conversion.ts";
import { calculateBlurhash } from "../media/preprocessors/blurhash.ts";
import { convertImage } from "../media/preprocessors/image-conversion.ts";
import {
type MediaJobData,
MediaJobType,
@ -29,8 +28,6 @@ export const getMediaWorker = (): Worker<MediaJobData, void, MediaJobType> =>
);
}
const processor = new ImageConversionPreprocessor(config);
await job.log(`Processing attachment [${attachmentId}]`);
await job.log(
`Fetching file from [${attachment.getUrl()}]`,
@ -45,29 +42,11 @@ export const getMediaWorker = (): Worker<MediaJobData, void, MediaJobType> =>
await job.log(`Converting attachment [${attachmentId}]`);
const { file: processedFile } =
await processor.process(file);
const mediaManager = new MediaManager(config);
const processedFile = await convertImage(file);
await job.log(`Uploading attachment [${attachmentId}]`);
const { path, uploadedFile } =
await mediaManager.addFile(processedFile);
const url = Media.getUrl(path);
await attachment.update({
content: await Media.fileToContentFormat(
uploadedFile,
url,
{
description:
attachment.data.content[0].description ||
undefined,
},
),
});
await attachment.updateFromFile(processedFile);
await job.log(
`✔ Finished processing attachment [${attachmentId}]`,
@ -89,8 +68,6 @@ export const getMediaWorker = (): Worker<MediaJobData, void, MediaJobType> =>
);
}
const blurhashProcessor = new BlurhashPreprocessor();
await job.log(`Processing attachment [${attachmentId}]`);
await job.log(
`Fetching file from [${attachment.getUrl()}]`,
@ -106,7 +83,7 @@ export const getMediaWorker = (): Worker<MediaJobData, void, MediaJobType> =>
await job.log(`Generating blurhash for [${attachmentId}]`);
const { blurhash } = await blurhashProcessor.process(file);
const blurhash = await calculateBlurhash(file);
await attachment.update({
blurhash,

11
cli/commands/emoji/delete.ts
View file

@ -1,14 +1,9 @@
import confirm from "@inquirer/confirm";
import { Flags } from "@oclif/core";
import { db } from "@versia/kit/db";
import { Emojis } from "@versia/kit/tables";
import chalk from "chalk";
import { eq } from "drizzle-orm";
import ora from "ora";
import { MediaManager } from "~/classes/media/media-manager";
import { EmojiFinderCommand } from "~/cli/classes";
import { formatArray } from "~/cli/utils/format";
import { config } from "~/packages/config-manager";
export default class EmojiDelete extends EmojiFinderCommand<
typeof EmojiDelete
@ -81,11 +76,7 @@ export default class EmojiDelete extends EmojiFinderCommand<
emojis.findIndex((e) => e.id === emoji.id) + 1
}/${emojis.length})`;
const mediaManager = new MediaManager(config);
await mediaManager.deleteFileByUrl(emoji.media.getUrl());
await db.delete(Emojis).where(eq(Emojis.id, emoji.id));
await emoji.delete();
}
spinner.succeed("Emoji(s) deleted");

7
config/config.example.toml
View file

@ -125,9 +125,8 @@ enabled = false
[media]
# Can be "s3" or "local", where "local" uploads the file to the local filesystem
# If you need to change this value after setting up your instance, you must move all the files
# from one backend to the other manually (the CLI will have an option to do this later)
# TODO: Add CLI command to move files
# Changing this value will not retroactively apply to existing data
# Don't forget to fill in the s3 config :3
backend = "s3"
# Whether to check the hash of media when uploading to avoid duplication
deduplicate_media = true
@ -145,7 +144,7 @@ convert_to = "image/webp"
convert_vector = false
# [s3]
# Can be left blank if you don't use the S3 media backend
# Can be left commented if you don't use the S3 media backend
# endpoint = ""
# access_key = "XXXXX"
# secret_access_key = "XXX"

9
config/config.schema.json
View file

@ -486,14 +486,7 @@
"secret_access_key",
"public_url"
],
"additionalProperties": false,
"default": {
"endpoint": "",
"access_key": "",
"secret_access_key": "",
"bucket_name": "versia",
"public_url": "https://cdn.example.com"
}
"additionalProperties": false
},
"validation": {
"type": "object",

16
packages/config-manager/config.type.ts
View file

@ -276,14 +276,7 @@ export const configValidator = z
public_url: zUrl,
})
.strict()
.default({
endpoint: "",
access_key: "",
secret_access_key: "",
region: undefined,
bucket_name: "versia",
public_url: "https://cdn.example.com",
}),
.optional(),
validation: z
.object({
max_displayname_size: z.number().int().default(50),
@ -854,6 +847,11 @@ export const configValidator = z
.strict()
.optional(),
})
.strict();
.strict()
.refine(
// If media backend is S3, s3 config must be set
(arg) => arg.media.backend === MediaBackendType.Local || !!arg.s3,
"S3 config must be set when using S3 media backend",
);
export type Config = z.infer<typeof configValidator>;