diff --git a/config/config.example.toml b/config/config.example.toml index e9b95555..fe9bbe28 100644 --- a/config/config.example.toml +++ b/config/config.example.toml @@ -71,6 +71,8 @@ tls = true backend = "s3" # Whether to check the hash of media when uploading to avoid duplication deduplicate_media = true +# If media backend is "local", this is the folder where the files will be stored +local_uploads_folder = "uploads" [media.conversion] convert_images = false diff --git a/packages/config-manager/config-type.type.ts b/packages/config-manager/config-type.type.ts index 940716d5..6364dc39 100644 --- a/packages/config-manager/config-type.type.ts +++ b/packages/config-manager/config-type.type.ts @@ -101,6 +101,7 @@ export interface ConfigType { convert_images: boolean; convert_to: string; }; + local_uploads_folder: string; }; s3: { @@ -234,6 +235,7 @@ export const configDefaults: ConfigType = { convert_images: false, convert_to: "webp", }, + local_uploads_folder: "uploads", }, email: { send_on_report: false, diff --git a/packages/config-manager/index.ts b/packages/config-manager/index.ts index 03d1e4fc..d9cbb52f 100644 --- a/packages/config-manager/index.ts +++ b/packages/config-manager/index.ts @@ -7,6 +7,7 @@ import { parse, stringify, type JsonMap } from "@iarna/toml"; import type { ConfigType } from "./config-type.type"; +import { configDefaults } from "./config-type.type"; import merge from "merge-deep-ts"; export class ConfigManager { @@ -116,3 +117,6 @@ export class ConfigManager { return merge(configs) as T; } } + +export type { ConfigType }; +export const defaultConfig = configDefaults; diff --git a/packages/media-manager/backends/local.ts b/packages/media-manager/backends/local.ts new file mode 100644 index 00000000..5c955b84 --- /dev/null +++ b/packages/media-manager/backends/local.ts @@ -0,0 +1,64 @@ +import type { ConvertableMediaFormats } from "../media-converter"; +import { MediaConverter } from "../media-converter"; +import { MediaBackend, MediaBackendType, MediaHasher } from ".."; +import type { ConfigType } from "config-manager"; + +export class LocalMediaBackend extends MediaBackend { + constructor(private config: ConfigType) { + super(MediaBackendType.LOCAL); + } + + public async addFile(file: File) { + if (this.shouldConvertImages(this.config)) { + const fileExtension = file.name.split(".").pop(); + const mediaConverter = new MediaConverter( + fileExtension as ConvertableMediaFormats, + this.config.media.conversion + .convert_to as ConvertableMediaFormats + ); + file = await mediaConverter.convert(file); + } + + const hash = await new MediaHasher().getMediaHash(file); + + const newFile = Bun.file( + `${this.config.media.local_uploads_folder}/${hash}` + ); + + if (await newFile.exists()) { + throw new Error("File already exists"); + } + + await Bun.write(newFile, file); + + return { + uploadedFile: file, + path: `./uploads/${file.name}`, + hash: hash, + }; + } + + public async getFileByHash( + hash: string, + databaseHashFetcher: (sha256: string) => Promise + ): Promise { + const filename = await databaseHashFetcher(hash); + + if (!filename) return null; + + return this.getFile(filename); + } + + public async getFile(filename: string): Promise { + const file = Bun.file( + `${this.config.media.local_uploads_folder}/${filename}` + ); + + if (!(await file.exists())) return null; + + return new File([await file.arrayBuffer()], filename, { + type: file.type, + lastModified: file.lastModified, + }); + } +} diff --git a/packages/media-manager/backends/s3.ts b/packages/media-manager/backends/s3.ts new file mode 100644 index 00000000..8098e2f2 --- /dev/null +++ b/packages/media-manager/backends/s3.ts @@ -0,0 +1,69 @@ +import { S3Client } from "@bradenmacdonald/s3-lite-client"; +import type { ConvertableMediaFormats } from "../media-converter"; +import { MediaConverter } from "../media-converter"; +import { MediaBackend, MediaBackendType, MediaHasher } from ".."; +import type { ConfigType } from "config-manager"; + +export class S3MediaBackend extends MediaBackend { + constructor( + private config: ConfigType, + private s3Client = new S3Client({ + endPoint: config.s3.endpoint, + useSSL: true, + region: config.s3.region || "auto", + bucket: config.s3.bucket_name, + accessKey: config.s3.access_key, + secretKey: config.s3.secret_access_key, + }) + ) { + super(MediaBackendType.S3); + } + + public async addFile(file: File) { + if (this.shouldConvertImages(this.config)) { + const fileExtension = file.name.split(".").pop(); + const mediaConverter = new MediaConverter( + fileExtension as ConvertableMediaFormats, + this.config.media.conversion + .convert_to as ConvertableMediaFormats + ); + file = await mediaConverter.convert(file); + } + + const hash = await new MediaHasher().getMediaHash(file); + + await this.s3Client.putObject(file.name, file.stream(), { + size: file.size, + }); + + return { + uploadedFile: file, + hash: hash, + }; + } + + public async getFileByHash( + hash: string, + databaseHashFetcher: (sha256: string) => Promise + ): Promise { + const filename = await databaseHashFetcher(hash); + + if (!filename) return null; + + return this.getFile(filename); + } + + public async getFile(filename: string): Promise { + try { + await this.s3Client.statObject(filename); + } catch { + return null; + } + + const file = await this.s3Client.getObject(filename); + + return new File([await file.arrayBuffer()], filename, { + type: file.headers.get("Content-Type") || "undefined", + }); + } +} diff --git a/packages/media-manager/bun.lockb b/packages/media-manager/bun.lockb new file mode 100755 index 00000000..202c862d Binary files /dev/null and b/packages/media-manager/bun.lockb differ diff --git a/packages/media-manager/bunfig.toml b/packages/media-manager/bunfig.toml new file mode 100644 index 00000000..bea1efe1 --- /dev/null +++ b/packages/media-manager/bunfig.toml @@ -0,0 +1,2 @@ +[install.scopes] +"@jsr" = "https://npm.jsr.io" diff --git a/packages/media-manager/index.ts b/packages/media-manager/index.ts new file mode 100644 index 00000000..77c2f581 --- /dev/null +++ b/packages/media-manager/index.ts @@ -0,0 +1,80 @@ +import type { ConfigType } from "config-manager"; + +export enum MediaBackendType { + LOCAL = "local", + S3 = "s3", +} + +interface UploadedFileMetadata { + uploadedFile: File; + path?: string; + hash: string; +} + +export class MediaHasher { + /** + * Returns the SHA-256 hash of a file in hex format + * @param media The file to hash + * @returns The SHA-256 hash of the file in hex format + */ + public async getMediaHash(media: File) { + const hash = new Bun.SHA256() + .update(await media.arrayBuffer()) + .digest("hex"); + + return hash; + } +} + +export class MediaBackend { + constructor(private backend: MediaBackendType) {} + + public getBackendType() { + return this.backend; + } + + public shouldConvertImages(config: ConfigType) { + return config.media.conversion.convert_images; + } + + /** + * Fetches file from backend from SHA-256 hash + * @param file SHA-256 hash of wanted file + * @param databaseHashFetcher Function that takes in a sha256 hash as input and outputs the filename of that file in the database + * @returns The file as a File object + */ + public getFileByHash( + // eslint-disable-next-line @typescript-eslint/no-unused-vars + file: string, + // eslint-disable-next-line @typescript-eslint/no-unused-vars + databaseHashFetcher: (sha256: string) => Promise + ): Promise { + return Promise.reject( + new Error("Do not call MediaBackend directly: use a subclass") + ); + } + + /** + * Fetches file from backend from filename + * @param filename File name + * @returns The file as a File object + */ + // eslint-disable-next-line @typescript-eslint/no-unused-vars + public getFile(filename: string): Promise { + return Promise.reject( + new Error("Do not call MediaBackend directly: use a subclass") + ); + } + + /** + * Adds file to backend + * @param file File to add + * @returns Metadata about the uploaded file + */ + // eslint-disable-next-line @typescript-eslint/no-unused-vars + public addFile(file: File): Promise { + return Promise.reject( + new Error("Do not call MediaBackend directly: use a subclass") + ); + } +} diff --git a/packages/media-manager/media-converter.ts b/packages/media-manager/media-converter.ts new file mode 100644 index 00000000..2602f4cf --- /dev/null +++ b/packages/media-manager/media-converter.ts @@ -0,0 +1,94 @@ +/** + * @packageDocumentation + * @module MediaManager + * @description Handles media conversion between formats + */ +import sharp from "sharp"; + +export enum ConvertableMediaFormats { + PNG = "png", + WEBP = "webp", + JPEG = "jpeg", + JPG = "jpg", + AVIF = "avif", + JXL = "jxl", + HEIF = "heif", +} + +/** + * Handles media conversion between formats + */ +export class MediaConverter { + constructor( + public fromFormat: ConvertableMediaFormats, + public toFormat: ConvertableMediaFormats + ) {} + + /** + * Returns whether the media is convertable + * @returns Whether the media is convertable + */ + public isConvertable() { + return ( + this.fromFormat !== this.toFormat && + Object.values(ConvertableMediaFormats).includes(this.fromFormat) + ); + } + + /** + * Returns the file name with the extension replaced + * @param fileName File name to replace + * @returns File name with extension replaced + */ + private getReplacedFileName(fileName: string) { + return this.extractFilenameFromPath(fileName).replace( + new RegExp(`\\.${this.fromFormat}$`), + `.${this.toFormat}` + ); + } + + /** + * Extracts the filename from a path + * @param path Path to extract filename from + * @returns Extracted filename + */ + private extractFilenameFromPath(path: string) { + // Don't count escaped slashes as path separators + const pathParts = path.split(/(? = { + [P in keyof T]?: DeepPartial; +}; + +describe("MediaBackend", () => { + let mediaBackend: MediaBackend; + let mockConfig: ConfigType; + + beforeEach(() => { + mediaBackend = new MediaBackend(MediaBackendType.S3); + mockConfig = { + media: { + conversion: { + convert_images: true, + }, + }, + } as ConfigType; + }); + + it("should initialize with correct backend type", () => { + expect(mediaBackend.getBackendType()).toEqual(MediaBackendType.S3); + }); + + it("should check if images should be converted", () => { + expect(mediaBackend.shouldConvertImages(mockConfig)).toBe(true); + mockConfig.media.conversion.convert_images = false; + expect(mediaBackend.shouldConvertImages(mockConfig)).toBe(false); + }); + + it("should throw error when calling getFileByHash", () => { + const mockHash = "test-hash"; + const databaseHashFetcher = jest.fn().mockResolvedValue("test.jpg"); + + expect( + mediaBackend.getFileByHash(mockHash, databaseHashFetcher) + ).rejects.toThrow(Error); + }); + + it("should throw error when calling getFile", () => { + const mockFilename = "test.jpg"; + + expect(mediaBackend.getFile(mockFilename)).rejects.toThrow(Error); + }); + + it("should throw error when calling addFile", () => { + const mockFile = new File([""], "test.jpg"); + + expect(mediaBackend.addFile(mockFile)).rejects.toThrow(); + }); +}); + +describe("S3MediaBackend", () => { + let s3MediaBackend: S3MediaBackend; + let mockS3Client: Partial; + let mockConfig: DeepPartial; + let mockFile: File; + let mockMediaHasher: MediaHasher; + + beforeEach(() => { + mockConfig = { + s3: { + endpoint: "http://localhost:4566", + region: "us-east-1", + bucket_name: "test-bucket", + access_key: "test-access-key", + secret_access_key: "test-secret-access-key", + public_url: "test", + }, + media: { + conversion: { + convert_to: ConvertableMediaFormats.PNG, + }, + }, + }; + mockFile = new File([new TextEncoder().encode("test")], "test.jpg"); + mockMediaHasher = new MediaHasher(); + mockS3Client = { + putObject: jest.fn().mockResolvedValue({}), + statObject: jest.fn().mockResolvedValue({}), + getObject: jest.fn().mockResolvedValue({ + blob: jest.fn().mockResolvedValue(new Blob()), + headers: new Headers({ "Content-Type": "image/jpeg" }), + }), + } as Partial; + s3MediaBackend = new S3MediaBackend( + mockConfig as ConfigType, + mockS3Client as S3Client + ); + }); + + it("should initialize with correct type", () => { + expect(s3MediaBackend.getBackendType()).toEqual(MediaBackendType.S3); + }); + + it("should add file", async () => { + const mockHash = "test-hash"; + spyOn(mockMediaHasher, "getMediaHash").mockResolvedValue(mockHash); + + const result = await s3MediaBackend.addFile(mockFile); + + expect(result.uploadedFile).toEqual(mockFile); + expect(result.hash).toHaveLength(64); + expect(mockS3Client.putObject).toHaveBeenCalledWith( + mockFile.name, + expect.any(ReadableStream), + { size: mockFile.size } + ); + }); + + it("should get file by hash", async () => { + const mockHash = "test-hash"; + const mockFilename = "test.jpg"; + const databaseHashFetcher = jest.fn().mockResolvedValue(mockFilename); + mockS3Client.statObject = jest.fn().mockResolvedValue({}); + mockS3Client.getObject = jest.fn().mockResolvedValue({ + arrayBuffer: jest.fn().mockResolvedValue(new ArrayBuffer(10)), + headers: new Headers({ "Content-Type": "image/jpeg" }), + }); + + const file = await s3MediaBackend.getFileByHash( + mockHash, + databaseHashFetcher + ); + + expect(file).not.toBeNull(); + expect(file?.name).toEqual(mockFilename); + expect(file?.type).toEqual("image/jpeg"); + }); + + it("should get file", async () => { + const mockFilename = "test.jpg"; + mockS3Client.statObject = jest.fn().mockResolvedValue({}); + mockS3Client.getObject = jest.fn().mockResolvedValue({ + arrayBuffer: jest.fn().mockResolvedValue(new ArrayBuffer(10)), + headers: new Headers({ "Content-Type": "image/jpeg" }), + }); + + const file = await s3MediaBackend.getFile(mockFilename); + + expect(file).not.toBeNull(); + expect(file?.name).toEqual(mockFilename); + expect(file?.type).toEqual("image/jpeg"); + }); +}); + +describe("LocalMediaBackend", () => { + let localMediaBackend: LocalMediaBackend; + let mockConfig: ConfigType; + let mockFile: File; + let mockMediaHasher: MediaHasher; + + beforeEach(() => { + mockConfig = { + media: { + conversion: { + convert_images: true, + convert_to: ConvertableMediaFormats.PNG, + }, + local_uploads_folder: "./uploads", + }, + } as ConfigType; + mockFile = Bun.file(__dirname + "/megamind.jpg") as unknown as File; + mockMediaHasher = new MediaHasher(); + localMediaBackend = new LocalMediaBackend(mockConfig); + }); + + it("should initialize with correct type", () => { + expect(localMediaBackend.getBackendType()).toEqual( + MediaBackendType.LOCAL + ); + }); + + it("should add file", async () => { + const mockHash = "test-hash"; + spyOn(mockMediaHasher, "getMediaHash").mockResolvedValue(mockHash); + const mockMediaConverter = new MediaConverter( + ConvertableMediaFormats.JPG, + ConvertableMediaFormats.PNG + ); + spyOn(mockMediaConverter, "convert").mockResolvedValue(mockFile); + // @ts-expect-error This is a mock + spyOn(Bun, "file").mockImplementationOnce(() => ({ + exists: () => Promise.resolve(false), + })); + spyOn(Bun, "write").mockImplementationOnce(() => + Promise.resolve(mockFile.size) + ); + + const result = await localMediaBackend.addFile(mockFile); + + expect(result.uploadedFile).toEqual(mockFile); + expect(result.path).toEqual(`./uploads/megamind.png`); + expect(result.hash).toHaveLength(64); + }); + + it("should get file by hash", async () => { + const mockHash = "test-hash"; + const mockFilename = "test.jpg"; + const databaseHashFetcher = jest.fn().mockResolvedValue(mockFilename); + // @ts-expect-error This is a mock + spyOn(Bun, "file").mockImplementationOnce(() => ({ + exists: () => Promise.resolve(true), + arrayBuffer: () => Promise.resolve(new ArrayBuffer(8)), + type: "image/jpeg", + lastModified: 123456789, + })); + + const file = await localMediaBackend.getFileByHash( + mockHash, + databaseHashFetcher + ); + + expect(file).not.toBeNull(); + expect(file?.name).toEqual(mockFilename); + expect(file?.type).toEqual("image/jpeg"); + }); + + it("should get file", async () => { + const mockFilename = "test.jpg"; + // @ts-expect-error This is a mock + spyOn(Bun, "file").mockImplementationOnce(() => ({ + exists: () => Promise.resolve(true), + arrayBuffer: () => Promise.resolve(new ArrayBuffer(8)), + type: "image/jpeg", + lastModified: 123456789, + })); + + const file = await localMediaBackend.getFile(mockFilename); + + expect(file).not.toBeNull(); + expect(file?.name).toEqual(mockFilename); + expect(file?.type).toEqual("image/jpeg"); + }); +}); diff --git a/packages/media-manager/tests/media-manager.test.ts b/packages/media-manager/tests/media-manager.test.ts new file mode 100644 index 00000000..017f3b6a --- /dev/null +++ b/packages/media-manager/tests/media-manager.test.ts @@ -0,0 +1,65 @@ +// FILEPATH: /home/jessew/Dev/lysand/packages/media-manager/media-converter.test.ts +import { describe, it, expect, beforeEach } from "bun:test"; +import { MediaConverter, ConvertableMediaFormats } from "../media-converter"; + +describe("MediaConverter", () => { + let mediaConverter: MediaConverter; + + beforeEach(() => { + mediaConverter = new MediaConverter( + ConvertableMediaFormats.JPG, + ConvertableMediaFormats.PNG + ); + }); + + it("should initialize with correct formats", () => { + expect(mediaConverter.fromFormat).toEqual(ConvertableMediaFormats.JPG); + expect(mediaConverter.toFormat).toEqual(ConvertableMediaFormats.PNG); + }); + + it("should check if media is convertable", () => { + expect(mediaConverter.isConvertable()).toBe(true); + mediaConverter.toFormat = ConvertableMediaFormats.JPG; + expect(mediaConverter.isConvertable()).toBe(false); + }); + + it("should replace file name extension", () => { + const fileName = "test.jpg"; + const expectedFileName = "test.png"; + // Written like this because it's a private function + expect(mediaConverter["getReplacedFileName"](fileName)).toEqual( + expectedFileName + ); + }); + + describe("Filename extractor", () => { + it("should extract filename from path", () => { + const path = "path/to/test.jpg"; + const expectedFileName = "test.jpg"; + expect(mediaConverter["extractFilenameFromPath"](path)).toEqual( + expectedFileName + ); + }); + + it("should handle escaped slashes", () => { + const path = "path/to/test\\/test.jpg"; + const expectedFileName = "test\\/test.jpg"; + expect(mediaConverter["extractFilenameFromPath"](path)).toEqual( + expectedFileName + ); + }); + }); + + it("should convert media", async () => { + const file = Bun.file(__dirname + "/megamind.jpg"); + + const convertedFile = await mediaConverter.convert( + file as unknown as File + ); + + expect(convertedFile.name).toEqual("megamind.png"); + expect(convertedFile.type).toEqual( + `image/${ConvertableMediaFormats.PNG}` + ); + }); +}); diff --git a/packages/media-manager/tests/megamind.jpg b/packages/media-manager/tests/megamind.jpg new file mode 100644 index 00000000..0f8f035a Binary files /dev/null and b/packages/media-manager/tests/megamind.jpg differ