From 9ba6237f13c0d4ea83805595d6f111fd8c0039a0 Mon Sep 17 00:00:00 2001 From: Jesse Wierzbinski Date: Wed, 29 Jan 2025 17:21:40 +0100 Subject: [PATCH] refactor(media): :recycle: Massively simplify media pipeline with Bun.S3 --- .vscode/settings.json | 3 +- api/api/v1/emojis/:id/index.ts | 12 +- classes/database/media.ts | 62 ++++++-- classes/database/notification.ts | 12 -- classes/media/drivers/disk.test.ts | 136 ----------------- classes/media/drivers/disk.ts | 96 ------------ classes/media/drivers/media-driver.ts | 43 ------ classes/media/drivers/s3.test.ts | 126 --------------- classes/media/drivers/s3.ts | 97 ------------ classes/media/media-manager.test.ts | 123 --------------- classes/media/media-manager.ts | 111 -------------- classes/media/preprocessors/blurhash.test.ts | 27 ++-- classes/media/preprocessors/blurhash.ts | 69 ++++----- .../preprocessors/image-conversion.test.ts | 49 +++--- .../media/preprocessors/image-conversion.ts | 144 ++++++++---------- .../media/preprocessors/media-preprocessor.ts | 16 -- classes/workers/media.ts | 33 +--- cli/commands/emoji/delete.ts | 11 +- config/config.example.toml | 7 +- config/config.schema.json | 9 +- packages/config-manager/config.type.ts | 16 +- 21 files changed, 197 insertions(+), 1005 deletions(-) delete mode 100644 classes/media/drivers/disk.test.ts delete mode 100644 classes/media/drivers/disk.ts delete mode 100644 classes/media/drivers/media-driver.ts delete mode 100644 classes/media/drivers/s3.test.ts delete mode 100644 classes/media/drivers/s3.ts delete mode 100644 classes/media/media-manager.test.ts delete mode 100644 classes/media/media-manager.ts delete mode 100644 classes/media/preprocessors/media-preprocessor.ts diff --git a/.vscode/settings.json b/.vscode/settings.json index c69283b4..b85d64bd 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -7,7 +7,8 @@ "federation", "config", "plugin", - "worker" + "worker", + "media" ], "languageToolLinter.languageTool.ignoredWordsInWorkspace": ["versia"] } diff --git a/api/api/v1/emojis/:id/index.ts b/api/api/v1/emojis/:id/index.ts index 9930336a..a1a80a15 100644 --- a/api/api/v1/emojis/:id/index.ts +++ b/api/api/v1/emojis/:id/index.ts @@ -1,12 +1,10 @@ import { apiRoute, auth, emojiValidator, jsonOrForm } from "@/api"; import { mimeLookup } from "@/content_types"; import { createRoute } from "@hono/zod-openapi"; -import { Emoji, db } from "@versia/kit/db"; -import { Emojis, RolePermissions } from "@versia/kit/tables"; -import { eq } from "drizzle-orm"; +import { Emoji } from "@versia/kit/db"; +import { RolePermissions } from "@versia/kit/tables"; import { z } from "zod"; import { ApiError } from "~/classes/errors/api-error"; -import { MediaManager } from "~/classes/media/media-manager"; import { config } from "~/packages/config-manager"; import { ErrorSchema } from "~/types/api"; @@ -306,11 +304,7 @@ export default apiRoute((app) => { ); } - const mediaManager = new MediaManager(config); - - await mediaManager.deleteFileByUrl(emoji.media.getUrl()); - - await db.delete(Emojis).where(eq(Emojis.id, id)); + await emoji.delete(); return context.body(null, 204); }); diff --git a/classes/database/media.ts b/classes/database/media.ts index 8f734af1..0fe0477e 100644 --- a/classes/database/media.ts +++ b/classes/database/media.ts @@ -1,10 +1,11 @@ +import { join } from "node:path"; import { mimeLookup } from "@/content_types.ts"; import { proxyUrl } from "@/response"; import type { Attachment as ApiAttachment } from "@versia/client/types"; import type { ContentFormat } from "@versia/federation/types"; import { db } from "@versia/kit/db"; import { Medias } from "@versia/kit/tables"; -import { SHA256 } from "bun"; +import { S3Client, SHA256, randomUUIDv7, write } from "bun"; import { type InferInsertModel, type InferSelectModel, @@ -18,7 +19,7 @@ import { z } from "zod"; import { MediaBackendType } from "~/packages/config-manager/config.type"; import { config } from "~/packages/config-manager/index.ts"; import { ApiError } from "../errors/api-error.ts"; -import { MediaManager } from "../media/media-manager.ts"; +import { getMediaHash } from "../media/media-hasher.ts"; import { MediaJobType, mediaQueue } from "../queues/media.ts"; import { BaseInterface } from "./base.ts"; @@ -154,6 +155,47 @@ export class Media extends BaseInterface { return attachment; } + private static async upload(file: File): Promise<{ + path: string; + }> { + const fileName = file.name ?? randomUUIDv7(); + const hash = await getMediaHash(file); + + switch (config.media.backend) { + case MediaBackendType.Local: { + const path = join( + config.media.local_uploads_folder, + hash, + fileName, + ); + + await write(path, file); + + return { path: join(hash, fileName) }; + } + + case MediaBackendType.S3: { + const path = join(hash, fileName); + + if (!config.s3) { + throw new ApiError(500, "S3 configuration missing"); + } + + const client = new S3Client({ + endpoint: config.s3.endpoint, + region: config.s3.region, + bucket: config.s3.bucket_name, + accessKeyId: config.s3.access_key, + secretAccessKey: config.s3.secret_access_key, + }); + + await client.write(path, file); + + return { path }; + } + } + } + public static async fromFile( file: File, options?: { @@ -163,16 +205,14 @@ export class Media extends BaseInterface { ): Promise { Media.checkFile(file); - const mediaManager = new MediaManager(config); - - const { path } = await mediaManager.addFile(file); + const { path } = await Media.upload(file); const url = Media.getUrl(path); let thumbnailUrl = ""; if (options?.thumbnail) { - const { path } = await mediaManager.addFile(options.thumbnail); + const { path } = await Media.upload(options.thumbnail); thumbnailUrl = Media.getUrl(path); } @@ -259,9 +299,7 @@ export class Media extends BaseInterface { public async updateFromFile(file: File): Promise { Media.checkFile(file); - const mediaManager = new MediaManager(config); - - const { path } = await mediaManager.addFile(file); + const { path } = await Media.upload(file); const url = Media.getUrl(path); @@ -307,9 +345,7 @@ export class Media extends BaseInterface { public async updateThumbnail(file: File): Promise { Media.checkFile(file); - const mediaManager = new MediaManager(config); - - const { path } = await mediaManager.addFile(file); + const { path } = await Media.upload(file); const url = Media.getUrl(path); @@ -346,7 +382,7 @@ export class Media extends BaseInterface { return new URL(`/media/${name}`, config.http.base_url).toString(); } if (config.media.backend === MediaBackendType.S3) { - return new URL(`/${name}`, config.s3.public_url).toString(); + return new URL(`/${name}`, config.s3?.public_url).toString(); } return ""; } diff --git a/classes/database/notification.ts b/classes/database/notification.ts index 79374e27..ebe3fae9 100644 --- a/classes/database/notification.ts +++ b/classes/database/notification.ts @@ -10,8 +10,6 @@ import { inArray, } from "drizzle-orm"; import { z } from "zod"; -import { MediaBackendType } from "~/packages/config-manager/config.type"; -import { config } from "~/packages/config-manager/index.ts"; import { transformOutputToUserWithRelations, userExtrasTemplate, @@ -215,16 +213,6 @@ export class Notification extends BaseInterface< return this.data.id; } - public static getUrl(name: string): string { - if (config.media.backend === MediaBackendType.Local) { - return new URL(`/media/${name}`, config.http.base_url).toString(); - } - if (config.media.backend === MediaBackendType.S3) { - return new URL(`/${name}`, config.s3.public_url).toString(); - } - return ""; - } - public async toApi(): Promise { const account = new User(this.data.account); diff --git a/classes/media/drivers/disk.test.ts b/classes/media/drivers/disk.test.ts deleted file mode 100644 index 60e15dab..00000000 --- a/classes/media/drivers/disk.test.ts +++ /dev/null @@ -1,136 +0,0 @@ -/** - * @packageDocumentation - * @module Tests/DiskMediaDriver - */ - -import { - type Mock, - beforeEach, - describe, - expect, - it, - mock, - spyOn, -} from "bun:test"; -import { rm } from "node:fs/promises"; -import { join } from "node:path"; -import type { Config } from "~/packages/config-manager/config.type"; -import type { getMediaHash } from "../media-hasher.ts"; -import { DiskMediaDriver } from "./disk.ts"; - -describe("DiskMediaDriver", () => { - let diskDriver: DiskMediaDriver; - let mockConfig: Config; - let mockMediaHasher: Mock; - let bunWriteSpy: Mock; - - beforeEach(() => { - mockConfig = { - media: { - local_uploads_folder: "/test/uploads", - }, - http: { - base_url: "http://localhost:3000", - }, - } as Config; - - mockMediaHasher = mock(() => Promise.resolve("testhash")); - - mock.module("../media-hasher", () => ({ - getMediaHash: mockMediaHasher, - })); - - diskDriver = new DiskMediaDriver(mockConfig); - // @ts-expect-error: Replacing private property for testing - diskDriver.mediaHasher = mockMediaHasher; - - // Mock fs.promises methods - mock.module("node:fs/promises", () => ({ - writeFile: mock(() => Promise.resolve()), - rm: mock(() => { - return Promise.resolve(); - }), - })); - - spyOn(Bun, "file").mockImplementation( - mock(() => ({ - exists: mock(() => Promise.resolve(true)), - arrayBuffer: mock(() => Promise.resolve(new ArrayBuffer(8))), - type: "image/webp", - lastModified: Date.now(), - })) as unknown as typeof Bun.file, - ); - - bunWriteSpy = spyOn(Bun, "write").mockImplementation( - mock(() => Promise.resolve(0)), - ); - }); - - it("should add a file", async () => { - const file = new File(["test"], "test.webp", { type: "image/webp" }); - const result = await diskDriver.addFile(file); - - expect(mockMediaHasher).toHaveBeenCalledWith(file); - expect(bunWriteSpy).toHaveBeenCalledWith( - join("/test/uploads", "testhash", "test.webp"), - expect.any(ArrayBuffer), - ); - expect(result).toEqual({ - uploadedFile: file, - path: join("testhash", "test.webp"), - hash: "testhash", - }); - }); - - it("should properly handle a Blob instead of a File", async () => { - const file = new Blob(["test"], { type: "image/webp" }); - const result = await diskDriver.addFile(file as File); - - expect(mockMediaHasher).toHaveBeenCalledWith(file); - expect(bunWriteSpy).toHaveBeenCalledWith( - expect.stringContaining("testhash"), - expect.any(ArrayBuffer), - ); - expect(result).toEqual({ - uploadedFile: expect.any(Blob), - path: expect.stringContaining("testhash"), - hash: "testhash", - }); - }); - - it("should get a file by hash", async () => { - const hash = "testhash"; - const databaseHashFetcher = mock(() => Promise.resolve("test.webp")); - const result = await diskDriver.getFileByHash( - hash, - databaseHashFetcher, - ); - - expect(databaseHashFetcher).toHaveBeenCalledWith(hash); - expect(Bun.file).toHaveBeenCalledWith( - join("/test/uploads", "test.webp"), - ); - expect(result).toBeInstanceOf(File); - expect(result?.name).toBe("test.webp"); - expect(result?.type).toBe("image/webp"); - }); - - it("should get a file by filename", async () => { - const filename = "test.webp"; - const result = await diskDriver.getFile(filename); - - expect(Bun.file).toHaveBeenCalledWith(join("/test/uploads", filename)); - expect(result).toBeInstanceOf(File); - expect(result?.name).toBe(filename); - expect(result?.type).toBe("image/webp"); - }); - - it("should delete a file by URL", async () => { - const url = "http://localhost:3000/uploads/testhash/test.webp"; - await diskDriver.deleteFileByUrl(url); - - expect(rm).toHaveBeenCalledWith(join("/test/uploads", "testhash"), { - recursive: true, - }); - }); -}); diff --git a/classes/media/drivers/disk.ts b/classes/media/drivers/disk.ts deleted file mode 100644 index 5e380ed1..00000000 --- a/classes/media/drivers/disk.ts +++ /dev/null @@ -1,96 +0,0 @@ -/** - * @packageDocumentation - * @module MediaManager/Drivers - */ - -import { rm } from "node:fs/promises"; -import { join } from "node:path"; -import type { Config } from "~/packages/config-manager/config.type"; -import { getMediaHash } from "../media-hasher.ts"; -import type { UploadedFileMetadata } from "../media-manager.ts"; -import type { MediaDriver } from "./media-driver.ts"; - -/** - * Implements the MediaDriver interface for disk storage. - */ -export class DiskMediaDriver implements MediaDriver { - /** - * Creates a new DiskMediaDriver instance. - * @param config - The configuration object. - */ - public constructor(private config: Config) {} - - /** - * @inheritdoc - */ - public async addFile( - file: File, - ): Promise> { - // Sometimes the file name is not available, so we generate a random name - const fileName = file.name ?? crypto.randomUUID(); - - const hash = await getMediaHash(file); - const path = join(hash, fileName); - const fullPath = join(this.config.media.local_uploads_folder, path); - - await Bun.write(fullPath, await file.arrayBuffer()); - - return { - uploadedFile: file, - path, - hash, - }; - } - - /** - * @inheritdoc - */ - public async getFileByHash( - hash: string, - databaseHashFetcher: (sha256: string) => Promise, - ): Promise { - const filename = await databaseHashFetcher(hash); - if (!filename) { - return null; - } - return this.getFile(filename); - } - - /** - * @inheritdoc - */ - public async getFile(filename: string): Promise { - const fullPath = join(this.config.media.local_uploads_folder, filename); - try { - const file = Bun.file(fullPath); - if (await file.exists()) { - return new File([await file.arrayBuffer()], filename, { - type: file.type, - lastModified: file.lastModified, - }); - } - } catch { - // File doesn't exist or can't be read - } - return null; - } - - /** - * @inheritdoc - */ - public async deleteFileByUrl(url: string): Promise { - const urlObj = new URL(url); - - // Check if URL is from the local uploads folder - if (urlObj.host !== new URL(this.config.http.base_url).host) { - return Promise.resolve(); - } - - const hash = urlObj.pathname.split("/").at(-2); - if (!hash) { - throw new Error("Invalid URL"); - } - const dirPath = join(this.config.media.local_uploads_folder, hash); - await rm(dirPath, { recursive: true }); - } -} diff --git a/classes/media/drivers/media-driver.ts b/classes/media/drivers/media-driver.ts deleted file mode 100644 index f137de56..00000000 --- a/classes/media/drivers/media-driver.ts +++ /dev/null @@ -1,43 +0,0 @@ -/** - * @packageDocumentation - * @module MediaManager/Drivers - */ - -import type { UploadedFileMetadata } from "../media-manager.ts"; - -/** - * Represents a media storage driver. - */ -export interface MediaDriver { - /** - * Adds a file to the media storage. - * @param file - The file to add. - * @returns A promise that resolves to the metadata of the uploaded file. - */ - addFile(file: File): Promise>; - - /** - * Retrieves a file from the media storage by its hash. - * @param hash - The hash of the file to retrieve. - * @param databaseHashFetcher - A function to fetch the filename from the database. - * @returns A promise that resolves to the file or null if not found. - */ - getFileByHash( - hash: string, - databaseHashFetcher: (sha256: string) => Promise, - ): Promise; - - /** - * Retrieves a file from the media storage by its filename. - * @param filename - The name of the file to retrieve. - * @returns A promise that resolves to the file or null if not found. - */ - getFile(filename: string): Promise; - - /** - * Deletes a file from the media storage by its URL. - * @param url - The URL of the file to delete. - * @returns A promise that resolves when the file is deleted. - */ - deleteFileByUrl(url: string): Promise; -} diff --git a/classes/media/drivers/s3.test.ts b/classes/media/drivers/s3.test.ts deleted file mode 100644 index ca02d479..00000000 --- a/classes/media/drivers/s3.test.ts +++ /dev/null @@ -1,126 +0,0 @@ -/** - * @packageDocumentation - * @module Tests/S3MediaDriver - */ - -import { type Mock, beforeEach, describe, expect, it, mock } from "bun:test"; -import type { S3Client } from "@bradenmacdonald/s3-lite-client"; -import type { Config } from "~/packages/config-manager/config.type"; -import type { getMediaHash } from "../media-hasher.ts"; -import { S3MediaDriver } from "./s3.ts"; - -describe("S3MediaDriver", () => { - let s3Driver: S3MediaDriver; - let mockConfig: Config; - let mockS3Client: S3Client; - let mockMediaHasher: Mock; - - beforeEach(() => { - mockConfig = { - s3: { - endpoint: "s3.amazonaws.com", - region: "us-west-2", - bucket_name: "test-bucket", - access_key: "test-key", - secret_access_key: "test-secret", - }, - } as Config; - - mockS3Client = mock(() => ({ - putObject: mock(() => Promise.resolve()), - getObject: mock(() => - Promise.resolve({ - arrayBuffer: (): Promise => - Promise.resolve(new ArrayBuffer(8)), - headers: new Headers({ "Content-Type": "image/webp" }), - }), - ), - statObject: mock(() => Promise.resolve()), - deleteObject: mock(() => Promise.resolve()), - }))() as unknown as S3Client; - - mockMediaHasher = mock(() => Promise.resolve("testhash")); - - mock.module("../media-hasher", () => ({ - getMediaHash: mockMediaHasher, - })); - - s3Driver = new S3MediaDriver(mockConfig); - // @ts-expect-error: Replacing private property for testing - s3Driver.s3Client = mockS3Client; - // @ts-expect-error: Replacing private property for testing - s3Driver.mediaHasher = mockMediaHasher; - }); - - it("should add a file", async () => { - const file = new File(["test"], "test.webp", { type: "image/webp" }); - const result = await s3Driver.addFile(file); - - expect(mockMediaHasher).toHaveBeenCalledWith(file); - expect(mockS3Client.putObject).toHaveBeenCalledWith( - "testhash/test.webp", - expect.any(ReadableStream), - { size: file.size, metadata: { "Content-Type": file.type } }, - ); - expect(result).toEqual({ - uploadedFile: file, - path: "testhash/test.webp", - hash: "testhash", - }); - }); - - it("should handle a Blob instead of a File", async () => { - const file = new Blob(["test"], { type: "image/webp" }); - const result = await s3Driver.addFile(file as File); - - expect(mockMediaHasher).toHaveBeenCalledWith(file); - expect(mockS3Client.putObject).toHaveBeenCalledWith( - expect.stringContaining("testhash"), - expect.any(ReadableStream), - { - size: file.size, - metadata: { - "Content-Type": file.type, - }, - }, - ); - expect(result).toEqual({ - uploadedFile: expect.any(Blob), - path: expect.stringContaining("testhash"), - hash: "testhash", - }); - }); - - it("should get a file by hash", async () => { - const hash = "testhash"; - const databaseHashFetcher = mock(() => Promise.resolve("test.webp")); - const result = await s3Driver.getFileByHash(hash, databaseHashFetcher); - - expect(databaseHashFetcher).toHaveBeenCalledWith(hash); - expect(mockS3Client.statObject).toHaveBeenCalledWith("test.webp"); - expect(mockS3Client.getObject).toHaveBeenCalledWith("test.webp"); - expect(result).toBeInstanceOf(File); - expect(result?.name).toBe("test.webp"); - expect(result?.type).toBe("image/webp"); - }); - - it("should get a file by filename", async () => { - const filename = "test.webp"; - const result = await s3Driver.getFile(filename); - - expect(mockS3Client.statObject).toHaveBeenCalledWith(filename); - expect(mockS3Client.getObject).toHaveBeenCalledWith(filename); - expect(result).toBeInstanceOf(File); - expect(result?.name).toBe(filename); - expect(result?.type).toBe("image/webp"); - }); - - it("should delete a file by URL", async () => { - const url = "https://test-bucket.s3.amazonaws.com/test/test.webp"; - await s3Driver.deleteFileByUrl(url); - - expect(mockS3Client.deleteObject).toHaveBeenCalledWith( - "test/test.webp", - ); - }); -}); diff --git a/classes/media/drivers/s3.ts b/classes/media/drivers/s3.ts deleted file mode 100644 index 21e93d19..00000000 --- a/classes/media/drivers/s3.ts +++ /dev/null @@ -1,97 +0,0 @@ -/** - * @packageDocumentation - * @module MediaManager/Drivers - */ - -import { S3Client } from "@bradenmacdonald/s3-lite-client"; -import type { Config } from "~/packages/config-manager/config.type"; -import { getMediaHash } from "../media-hasher.ts"; -import type { UploadedFileMetadata } from "../media-manager.ts"; -import type { MediaDriver } from "./media-driver.ts"; - -/** - * Implements the MediaDriver interface for S3 storage. - */ -export class S3MediaDriver implements MediaDriver { - private s3Client: S3Client; - - /** - * Creates a new S3MediaDriver instance. - * @param config - The configuration object. - */ - public constructor(config: Config) { - this.s3Client = new S3Client({ - endPoint: config.s3.endpoint, - useSSL: true, - region: config.s3.region || "auto", - bucket: config.s3.bucket_name, - accessKey: config.s3.access_key, - secretKey: config.s3.secret_access_key, - }); - } - - /** - * @inheritdoc - */ - public async addFile( - file: File, - ): Promise> { - // Sometimes the file name is not available, so we generate a random name - const fileName = file.name ?? crypto.randomUUID(); - - const hash = await getMediaHash(file); - const path = `${hash}/${fileName}`; - - await this.s3Client.putObject(path, file.stream(), { - size: file.size, - metadata: { - "Content-Type": file.type, - }, - }); - - return { - uploadedFile: file, - path, - hash, - }; - } - - /** - * @inheritdoc - */ - public async getFileByHash( - hash: string, - databaseHashFetcher: (sha256: string) => Promise, - ): Promise { - const filename = await databaseHashFetcher(hash); - if (!filename) { - return null; - } - return this.getFile(filename); - } - - /** - * @inheritdoc - */ - public async getFile(filename: string): Promise { - try { - await this.s3Client.statObject(filename); - const file = await this.s3Client.getObject(filename); - const arrayBuffer = await file.arrayBuffer(); - return new File([arrayBuffer], filename, { - type: file.headers.get("Content-Type") || undefined, - }); - } catch { - return null; - } - } - - /** - * @inheritdoc - */ - public async deleteFileByUrl(url: string): Promise { - const urlObj = new URL(url); - const path = urlObj.pathname.slice(1); // Remove leading slash - await this.s3Client.deleteObject(path); - } -} diff --git a/classes/media/media-manager.test.ts b/classes/media/media-manager.test.ts deleted file mode 100644 index 11503665..00000000 --- a/classes/media/media-manager.test.ts +++ /dev/null @@ -1,123 +0,0 @@ -/** - * @packageDocumentation - * @module Tests/MediaManager - */ - -import { beforeEach, describe, expect, it, mock } from "bun:test"; -import type { Config } from "~/packages/config-manager/config.type"; -import { MediaBackendType } from "~/packages/config-manager/config.type"; -import { DiskMediaDriver } from "./drivers/disk.ts"; -import { S3MediaDriver } from "./drivers/s3.ts"; -import { MediaManager } from "./media-manager.ts"; -import type { ImageConversionPreprocessor } from "./preprocessors/image-conversion.ts"; - -describe("MediaManager", () => { - let mediaManager: MediaManager; - let mockConfig: Config; - let mockS3Driver: S3MediaDriver; - let mockImagePreprocessor: ImageConversionPreprocessor; - - beforeEach(() => { - mockConfig = { - media: { - backend: "s3", - conversion: { - convert_images: true, - convert_to: "image/webp", - }, - }, - s3: { - endpoint: "s3.amazonaws.com", - region: "us-west-2", - bucket_name: "test-bucket", - access_key: "test-key", - secret_access_key: "test-secret", - }, - } as Config; - - mockS3Driver = mock(() => ({ - addFile: mock(() => - Promise.resolve({ - uploadedFile: new File(["hey"], "test.webp"), - path: "test/test.webp", - hash: "testhash", - }), - ), - getFileByHash: mock(() => { - return Promise.resolve(new File(["hey"], "test.webp")); - }), - getFile: mock(() => - Promise.resolve(new File(["hey"], "test.webp")), - ), - deleteFileByUrl: mock(() => Promise.resolve()), - }))() as unknown as S3MediaDriver; - - mockImagePreprocessor = mock(() => ({ - process: mock((_: File) => - Promise.resolve(new File(["hey"], "test.webp")), - ), - }))() as unknown as ImageConversionPreprocessor; - - mediaManager = new MediaManager(mockConfig); - // @ts-expect-error: Accessing private property for testing - mediaManager.driver = mockS3Driver; - // @ts-expect-error: Accessing private property for testing - mediaManager.preprocessors = [mockImagePreprocessor]; - }); - - it("should initialize with the correct driver based on config", () => { - const s3Manager = new MediaManager(mockConfig); - // @ts-expect-error: Accessing private property for testing - expect(s3Manager.driver).toBeInstanceOf(S3MediaDriver); - - mockConfig.media.backend = MediaBackendType.Local; - const diskManager = new MediaManager(mockConfig); - // @ts-expect-error: Accessing private property for testing - expect(diskManager.driver).toBeInstanceOf(DiskMediaDriver); - }); - - it("should add a file with preprocessing", async () => { - const file = new File(["test"], "test.jpg", { type: "image/jpeg" }); - const result = await mediaManager.addFile(file); - - expect(mockImagePreprocessor.process).toHaveBeenCalledWith(file); - expect(mockS3Driver.addFile).toHaveBeenCalled(); - expect(result).toEqual({ - uploadedFile: new File(["hey"], "test.webp"), - path: "test/test.webp", - hash: "testhash", - }); - }); - - it("should get a file by hash", async () => { - const hash = "testhash"; - const databaseHashFetcher = mock(() => Promise.resolve("test.webp")); - const result = await mediaManager.getFileByHash( - hash, - databaseHashFetcher, - ); - - expect(mockS3Driver.getFileByHash).toHaveBeenCalledWith( - hash, - databaseHashFetcher, - ); - expect(result).toBeInstanceOf(File); - expect(result?.name).toBe("test.webp"); - }); - - it("should get a file by filename", async () => { - const filename = "test.webp"; - const result = await mediaManager.getFile(filename); - - expect(mockS3Driver.getFile).toHaveBeenCalledWith(filename); - expect(result).toBeInstanceOf(File); - expect(result?.name).toBe("test.webp"); - }); - - it("should delete a file by URL", async () => { - const url = "https://test-bucket.s3.amazonaws.com/test/test.webp"; - await mediaManager.deleteFileByUrl(url); - - expect(mockS3Driver.deleteFileByUrl).toHaveBeenCalledWith(url); - }); -}); diff --git a/classes/media/media-manager.ts b/classes/media/media-manager.ts deleted file mode 100644 index 5ffccab0..00000000 --- a/classes/media/media-manager.ts +++ /dev/null @@ -1,111 +0,0 @@ -/** - * @packageDocumentation - * @module MediaManager - */ - -import type { Config } from "~/packages/config-manager/config.type"; -import { DiskMediaDriver } from "./drivers/disk.ts"; -import type { MediaDriver } from "./drivers/media-driver.ts"; -import { S3MediaDriver } from "./drivers/s3.ts"; -import type { MediaPreprocessor } from "./preprocessors/media-preprocessor.ts"; - -/** - * Manages media operations with support for different storage drivers and preprocessing plugins. - * @example - * const mediaManager = new MediaManager(config); - * - * const file = new File(["hello"], "hello.txt"); - * - * const { path, hash, blurhash } = await mediaManager.addFile(file); - * - * const retrievedFile = await mediaManager.getFileByHash(hash, fetchHashFromDatabase); - * - * await mediaManager.deleteFileByUrl(path); - */ -export class MediaManager { - private driver: MediaDriver; - private preprocessors: MediaPreprocessor[] = []; - - /** - * Creates a new MediaManager instance. - * @param config - The configuration object. - */ - public constructor(private config: Config) { - this.driver = this.initializeDriver(); - } - - /** - * Initializes the appropriate media driver based on the configuration. - * @returns An instance of MediaDriver. - */ - private initializeDriver(): MediaDriver { - switch (this.config.media.backend) { - case "s3": - return new S3MediaDriver(this.config); - case "local": - return new DiskMediaDriver(this.config); - default: - throw new Error( - `Unsupported media backend: ${this.config.media.backend}`, - ); - } - } - - /** - * Adds a file to the media storage. - * @param file - The file to add. - * @returns A promise that resolves to the metadata of the uploaded file. - */ - public async addFile(file: File): Promise { - let processedFile = file; - - for (const preprocessor of this.preprocessors) { - const result = await preprocessor.process(processedFile); - - processedFile = result.file; - } - - const uploadResult = await this.driver.addFile(processedFile); - - return uploadResult; - } - /** - * Retrieves a file from the media storage by its hash. - * @param hash - The hash of the file to retrieve. - * @param databaseHashFetcher - A function to fetch the filename from the database. - * @returns A promise that resolves to the file or null if not found. - */ - public getFileByHash( - hash: string, - databaseHashFetcher: (sha256: string) => Promise, - ): Promise { - return this.driver.getFileByHash(hash, databaseHashFetcher); - } - - /** - * Retrieves a file from the media storage by its filename. - * @param filename - The name of the file to retrieve. - * @returns A promise that resolves to the file or null if not found. - */ - public getFile(filename: string): Promise { - return this.driver.getFile(filename); - } - - /** - * Deletes a file from the media storage by its URL. - * @param url - The URL of the file to delete. - * @returns A promise that resolves when the file is deleted. - */ - public deleteFileByUrl(url: string): Promise { - return this.driver.deleteFileByUrl(url); - } -} - -/** - * Represents the metadata of an uploaded file. - */ -export interface UploadedFileMetadata { - uploadedFile: File; - path: string; - hash: string; -} diff --git a/classes/media/preprocessors/blurhash.test.ts b/classes/media/preprocessors/blurhash.test.ts index ce89f4df..c2f5013d 100644 --- a/classes/media/preprocessors/blurhash.test.ts +++ b/classes/media/preprocessors/blurhash.test.ts @@ -1,14 +1,8 @@ -import { beforeEach, describe, expect, it, mock } from "bun:test"; +import { describe, expect, it, mock } from "bun:test"; import sharp from "sharp"; -import { BlurhashPreprocessor } from "./blurhash.ts"; +import { calculateBlurhash } from "./blurhash.ts"; describe("BlurhashPreprocessor", () => { - let preprocessor: BlurhashPreprocessor; - - beforeEach(() => { - preprocessor = new BlurhashPreprocessor(); - }); - it("should calculate blurhash for a valid image", async () => { const inputBuffer = await sharp({ create: { @@ -24,21 +18,19 @@ describe("BlurhashPreprocessor", () => { const inputFile = new File([inputBuffer], "test.png", { type: "image/png", }); - const result = await preprocessor.process(inputFile); + const result = await calculateBlurhash(inputFile); - expect(result.file).toBe(inputFile); - expect(result.blurhash).toBeTypeOf("string"); - expect(result.blurhash).not.toBe(""); + expect(result).toBeTypeOf("string"); + expect(result).not.toBe(""); }); it("should return null blurhash for an invalid image", async () => { const invalidFile = new File(["invalid image data"], "invalid.png", { type: "image/png", }); - const result = await preprocessor.process(invalidFile); + const result = await calculateBlurhash(invalidFile); - expect(result.file).toBe(invalidFile); - expect(result.blurhash).toBeNull(); + expect(result).toBeNull(); }); it("should handle errors during blurhash calculation", async () => { @@ -63,9 +55,8 @@ describe("BlurhashPreprocessor", () => { }, })); - const result = await preprocessor.process(inputFile); + const result = await calculateBlurhash(inputFile); - expect(result.file).toBe(inputFile); - expect(result.blurhash).toBeNull(); + expect(result).toBeNull(); }); }); diff --git a/classes/media/preprocessors/blurhash.ts b/classes/media/preprocessors/blurhash.ts index 527ed7b6..dff23491 100644 --- a/classes/media/preprocessors/blurhash.ts +++ b/classes/media/preprocessors/blurhash.ts @@ -1,44 +1,37 @@ import { encode } from "blurhash"; import sharp from "sharp"; -import type { MediaPreprocessor } from "./media-preprocessor.ts"; -export class BlurhashPreprocessor implements MediaPreprocessor { - public async process( - file: File, - ): Promise<{ file: File; blurhash: string | null }> { - try { - const arrayBuffer = await file.arrayBuffer(); - const metadata = await sharp(arrayBuffer).metadata(); +export const calculateBlurhash = async (file: File): Promise => { + try { + const arrayBuffer = await file.arrayBuffer(); + const metadata = await sharp(arrayBuffer).metadata(); - const blurhash = await new Promise((resolve) => { - sharp(arrayBuffer) - .raw() - .ensureAlpha() - .toBuffer((err, buffer) => { - if (err) { - resolve(null); - return; - } + return new Promise((resolve) => { + sharp(arrayBuffer) + .raw() + .ensureAlpha() + .toBuffer((err, buffer) => { + if (err) { + resolve(null); + return; + } - try { - resolve( - encode( - new Uint8ClampedArray(buffer), - metadata?.width ?? 0, - metadata?.height ?? 0, - 4, - 4, - ) as string, - ); - } catch { - resolve(null); - } - }); - }); - - return { file, blurhash }; - } catch { - return { file, blurhash: null }; - } + try { + resolve( + encode( + new Uint8ClampedArray(buffer), + metadata?.width ?? 0, + metadata?.height ?? 0, + 4, + 4, + ) as string, + ); + } catch { + resolve(null); + } + }); + }); + } catch { + return null; } -} +}; diff --git a/classes/media/preprocessors/image-conversion.test.ts b/classes/media/preprocessors/image-conversion.test.ts index d59c8336..b83061d1 100644 --- a/classes/media/preprocessors/image-conversion.test.ts +++ b/classes/media/preprocessors/image-conversion.test.ts @@ -1,10 +1,9 @@ -import { beforeEach, describe, expect, it } from "bun:test"; +import { beforeEach, describe, expect, it, mock } from "bun:test"; import sharp from "sharp"; import type { Config } from "~/packages/config-manager/config.type"; -import { ImageConversionPreprocessor } from "./image-conversion.ts"; +import { convertImage } from "./image-conversion.ts"; describe("ImageConversionPreprocessor", () => { - let preprocessor: ImageConversionPreprocessor; let mockConfig: Config; beforeEach(() => { @@ -18,7 +17,9 @@ describe("ImageConversionPreprocessor", () => { }, } as Config; - preprocessor = new ImageConversionPreprocessor(mockConfig); + mock.module("~/packages/config-manager/index.ts", () => ({ + config: mockConfig, + })); }); it("should convert a JPEG image to WebP", async () => { @@ -36,12 +37,12 @@ describe("ImageConversionPreprocessor", () => { const inputFile = new File([inputBuffer], "test.jpg", { type: "image/jpeg", }); - const result = await preprocessor.process(inputFile); + const result = await convertImage(inputFile); - expect(result.file.type).toBe("image/webp"); - expect(result.file.name).toBe("test.webp"); + expect(result.type).toBe("image/webp"); + expect(result.name).toBe("test.webp"); - const resultBuffer = await result.file.arrayBuffer(); + const resultBuffer = await result.arrayBuffer(); const metadata = await sharp(resultBuffer).metadata(); expect(metadata.format).toBe("webp"); }); @@ -52,38 +53,36 @@ describe("ImageConversionPreprocessor", () => { const inputFile = new File([svgContent], "test.svg", { type: "image/svg+xml", }); - const result = await preprocessor.process(inputFile); + const result = await convertImage(inputFile); - expect(result.file).toBe(inputFile); + expect(result).toBe(inputFile); }); it("should convert SVG when convert_vector is true", async () => { mockConfig.media.conversion.convert_vector = true; - preprocessor = new ImageConversionPreprocessor(mockConfig); const svgContent = ''; const inputFile = new File([svgContent], "test.svg", { type: "image/svg+xml", }); - const result = await preprocessor.process(inputFile); + const result = await convertImage(inputFile); - expect(result.file.type).toBe("image/webp"); - expect(result.file.name).toBe("test.webp"); + expect(result.type).toBe("image/webp"); + expect(result.name).toBe("test.webp"); }); it("should not convert unsupported file types", async () => { const inputFile = new File(["test content"], "test.txt", { type: "text/plain", }); - const result = await preprocessor.process(inputFile); + const result = await convertImage(inputFile); - expect(result.file).toBe(inputFile); + expect(result).toBe(inputFile); }); it("should throw an error for unsupported output format", async () => { mockConfig.media.conversion.convert_to = "image/bmp"; - preprocessor = new ImageConversionPreprocessor(mockConfig); const inputBuffer = await sharp({ create: { @@ -100,7 +99,7 @@ describe("ImageConversionPreprocessor", () => { type: "image/png", }); - await expect(preprocessor.process(inputFile)).rejects.toThrow( + await expect(convertImage(inputFile)).rejects.toThrow( "Unsupported output format: image/bmp", ); }); @@ -121,12 +120,12 @@ describe("ImageConversionPreprocessor", () => { const inputFile = new File([inputBuffer], "animated.gif", { type: "image/gif", }); - const result = await preprocessor.process(inputFile); + const result = await convertImage(inputFile); - expect(result.file.type).toBe("image/webp"); - expect(result.file.name).toBe("animated.webp"); + expect(result.type).toBe("image/webp"); + expect(result.name).toBe("animated.webp"); - const resultBuffer = await result.file.arrayBuffer(); + const resultBuffer = await result.arrayBuffer(); const metadata = await sharp(resultBuffer).metadata(); expect(metadata.format).toBe("webp"); }); @@ -148,9 +147,9 @@ describe("ImageConversionPreprocessor", () => { "test image with spaces.png", { type: "image/png" }, ); - const result = await preprocessor.process(inputFile); + const result = await convertImage(inputFile); - expect(result.file.type).toBe("image/webp"); - expect(result.file.name).toBe("test image with spaces.webp"); + expect(result.type).toBe("image/webp"); + expect(result.name).toBe("test image with spaces.webp"); }); }); diff --git a/classes/media/preprocessors/image-conversion.ts b/classes/media/preprocessors/image-conversion.ts index 88523cdc..a87c8fc2 100644 --- a/classes/media/preprocessors/image-conversion.ts +++ b/classes/media/preprocessors/image-conversion.ts @@ -4,8 +4,7 @@ */ import sharp from "sharp"; -import type { Config } from "~/packages/config-manager/config.type"; -import type { MediaPreprocessor } from "./media-preprocessor.ts"; +import { config } from "~/packages/config-manager/index.ts"; /** * Supported input media formats. @@ -33,92 +32,73 @@ const supportedOutputFormats = [ ]; /** - * Implements the MediaPreprocessor interface for image conversion. + * Checks if a file is convertible. + * @param file - The file to check. + * @returns True if the file is convertible, false otherwise. */ -export class ImageConversionPreprocessor implements MediaPreprocessor { - /** - * Creates a new ImageConversionPreprocessor instance. - * @param config - The configuration object. - */ - public constructor(private config: Config) {} +const isConvertible = (file: File): boolean => { + if ( + file.type === "image/svg+xml" && + !config.media.conversion.convert_vector + ) { + return false; + } + return supportedInputFormats.includes(file.type); +}; - /** - * @inheritdoc - */ - public async process(file: File): Promise<{ file: File }> { - if (!this.isConvertible(file)) { - return { file }; - } +/** + * Extracts the filename from a path. + * @param path - The path to extract the filename from. + * @returns The extracted filename. + */ +const extractFilenameFromPath = (path: string): string => { + const pathParts = path.split(/(? + extractFilenameFromPath(fileName).replace(/\.[^/.]+$/, `.${newExtension}`); - const sharpCommand = sharp(await file.arrayBuffer(), { - animated: true, - }); - const commandName = targetFormat.split("/")[1] as - | "jpeg" - | "png" - | "webp" - | "avif" - | "gif" - | "tiff"; - const convertedBuffer = await sharpCommand[commandName]().toBuffer(); - - return { - file: new File( - [convertedBuffer], - ImageConversionPreprocessor.getReplacedFileName( - file.name, - commandName, - ), - { - type: targetFormat, - lastModified: Date.now(), - }, - ), - }; +/** + * Converts an image file to the format specified in the configuration. + * + * @param file - The image file to convert. + * @returns The converted image file. + */ +export const convertImage = async (file: File): Promise => { + if (!isConvertible(file)) { + return file; } - /** - * Checks if a file is convertible. - * @param file - The file to check. - * @returns True if the file is convertible, false otherwise. - */ - private isConvertible(file: File): boolean { - if ( - file.type === "image/svg+xml" && - !this.config.media.conversion.convert_vector - ) { - return false; - } - return supportedInputFormats.includes(file.type); + const targetFormat = config.media.conversion.convert_to; + if (!supportedOutputFormats.includes(targetFormat)) { + throw new Error(`Unsupported output format: ${targetFormat}`); } - /** - * Replaces the file extension in the filename. - * @param fileName - The original filename. - * @param newExtension - The new extension. - * @returns The filename with the new extension. - */ - private static getReplacedFileName( - fileName: string, - newExtension: string, - ): string { - return ImageConversionPreprocessor.extractFilenameFromPath( - fileName, - ).replace(/\.[^/.]+$/, `.${newExtension}`); - } + const sharpCommand = sharp(await file.arrayBuffer(), { + animated: true, + }); + const commandName = targetFormat.split("/")[1] as + | "jpeg" + | "png" + | "webp" + | "avif" + | "gif" + | "tiff"; + const convertedBuffer = await sharpCommand[commandName]().toBuffer(); - /** - * Extracts the filename from a path. - * @param path - The path to extract the filename from. - * @returns The extracted filename. - */ - private static extractFilenameFromPath(path: string): string { - const pathParts = path.split(/(?>; -} diff --git a/classes/workers/media.ts b/classes/workers/media.ts index ac116907..cb628616 100644 --- a/classes/workers/media.ts +++ b/classes/workers/media.ts @@ -2,9 +2,8 @@ import { Media } from "@versia/kit/db"; import { Worker } from "bullmq"; import { config } from "~/packages/config-manager"; import { connection } from "~/utils/redis.ts"; -import { MediaManager } from "../media/media-manager.ts"; -import { BlurhashPreprocessor } from "../media/preprocessors/blurhash.ts"; -import { ImageConversionPreprocessor } from "../media/preprocessors/image-conversion.ts"; +import { calculateBlurhash } from "../media/preprocessors/blurhash.ts"; +import { convertImage } from "../media/preprocessors/image-conversion.ts"; import { type MediaJobData, MediaJobType, @@ -29,8 +28,6 @@ export const getMediaWorker = (): Worker => ); } - const processor = new ImageConversionPreprocessor(config); - await job.log(`Processing attachment [${attachmentId}]`); await job.log( `Fetching file from [${attachment.getUrl()}]`, @@ -45,29 +42,11 @@ export const getMediaWorker = (): Worker => await job.log(`Converting attachment [${attachmentId}]`); - const { file: processedFile } = - await processor.process(file); - - const mediaManager = new MediaManager(config); + const processedFile = await convertImage(file); await job.log(`Uploading attachment [${attachmentId}]`); - const { path, uploadedFile } = - await mediaManager.addFile(processedFile); - - const url = Media.getUrl(path); - - await attachment.update({ - content: await Media.fileToContentFormat( - uploadedFile, - url, - { - description: - attachment.data.content[0].description || - undefined, - }, - ), - }); + await attachment.updateFromFile(processedFile); await job.log( `✔ Finished processing attachment [${attachmentId}]`, @@ -89,8 +68,6 @@ export const getMediaWorker = (): Worker => ); } - const blurhashProcessor = new BlurhashPreprocessor(); - await job.log(`Processing attachment [${attachmentId}]`); await job.log( `Fetching file from [${attachment.getUrl()}]`, @@ -106,7 +83,7 @@ export const getMediaWorker = (): Worker => await job.log(`Generating blurhash for [${attachmentId}]`); - const { blurhash } = await blurhashProcessor.process(file); + const blurhash = await calculateBlurhash(file); await attachment.update({ blurhash, diff --git a/cli/commands/emoji/delete.ts b/cli/commands/emoji/delete.ts index eb38f5ff..33b26ba1 100644 --- a/cli/commands/emoji/delete.ts +++ b/cli/commands/emoji/delete.ts @@ -1,14 +1,9 @@ import confirm from "@inquirer/confirm"; import { Flags } from "@oclif/core"; -import { db } from "@versia/kit/db"; -import { Emojis } from "@versia/kit/tables"; import chalk from "chalk"; -import { eq } from "drizzle-orm"; import ora from "ora"; -import { MediaManager } from "~/classes/media/media-manager"; import { EmojiFinderCommand } from "~/cli/classes"; import { formatArray } from "~/cli/utils/format"; -import { config } from "~/packages/config-manager"; export default class EmojiDelete extends EmojiFinderCommand< typeof EmojiDelete @@ -81,11 +76,7 @@ export default class EmojiDelete extends EmojiFinderCommand< emojis.findIndex((e) => e.id === emoji.id) + 1 }/${emojis.length})`; - const mediaManager = new MediaManager(config); - - await mediaManager.deleteFileByUrl(emoji.media.getUrl()); - - await db.delete(Emojis).where(eq(Emojis.id, emoji.id)); + await emoji.delete(); } spinner.succeed("Emoji(s) deleted"); diff --git a/config/config.example.toml b/config/config.example.toml index c28d7823..f4b52460 100644 --- a/config/config.example.toml +++ b/config/config.example.toml @@ -125,9 +125,8 @@ enabled = false [media] # Can be "s3" or "local", where "local" uploads the file to the local filesystem -# If you need to change this value after setting up your instance, you must move all the files -# from one backend to the other manually (the CLI will have an option to do this later) -# TODO: Add CLI command to move files +# Changing this value will not retroactively apply to existing data +# Don't forget to fill in the s3 config :3 backend = "s3" # Whether to check the hash of media when uploading to avoid duplication deduplicate_media = true @@ -145,7 +144,7 @@ convert_to = "image/webp" convert_vector = false # [s3] -# Can be left blank if you don't use the S3 media backend +# Can be left commented if you don't use the S3 media backend # endpoint = "" # access_key = "XXXXX" # secret_access_key = "XXX" diff --git a/config/config.schema.json b/config/config.schema.json index 6229101e..cde891c5 100644 --- a/config/config.schema.json +++ b/config/config.schema.json @@ -486,14 +486,7 @@ "secret_access_key", "public_url" ], - "additionalProperties": false, - "default": { - "endpoint": "", - "access_key": "", - "secret_access_key": "", - "bucket_name": "versia", - "public_url": "https://cdn.example.com" - } + "additionalProperties": false }, "validation": { "type": "object", diff --git a/packages/config-manager/config.type.ts b/packages/config-manager/config.type.ts index 7156be24..91974f6b 100644 --- a/packages/config-manager/config.type.ts +++ b/packages/config-manager/config.type.ts @@ -276,14 +276,7 @@ export const configValidator = z public_url: zUrl, }) .strict() - .default({ - endpoint: "", - access_key: "", - secret_access_key: "", - region: undefined, - bucket_name: "versia", - public_url: "https://cdn.example.com", - }), + .optional(), validation: z .object({ max_displayname_size: z.number().int().default(50), @@ -854,6 +847,11 @@ export const configValidator = z .strict() .optional(), }) - .strict(); + .strict() + .refine( + // If media backend is S3, s3 config must be set + (arg) => arg.media.backend === MediaBackendType.Local || !!arg.s3, + "S3 config must be set when using S3 media backend", + ); export type Config = z.infer;