Add new media handling package

config/config.example.toml

@@ -71,6 +71,8 @@ tls = true
 backend = "s3"
 # Whether to check the hash of media when uploading to avoid duplication
 deduplicate_media = true
+# If media backend is "local", this is the folder where the files will be stored
+local_uploads_folder = "uploads"
 
 [media.conversion]
 convert_images = false

packages/config-manager/config-type.type.ts

@@ -101,6 +101,7 @@ export interface ConfigType {
 			convert_images: boolean;
 			convert_to: string;
 		};
+		local_uploads_folder: string;
 	};
 
 	s3: {
@@ -234,6 +235,7 @@ export const configDefaults: ConfigType = {
 			convert_images: false,
 			convert_to: "webp",
 		},
+		local_uploads_folder: "uploads",
 	},
 	email: {
 		send_on_report: false,

packages/config-manager/index.ts

@@ -7,6 +7,7 @@
 
 import { parse, stringify, type JsonMap } from "@iarna/toml";
 import type { ConfigType } from "./config-type.type";
+import { configDefaults } from "./config-type.type";
 import merge from "merge-deep-ts";
 
 export class ConfigManager {
@@ -116,3 +117,6 @@ export class ConfigManager {
 		return merge(configs) as T;
 	}
 }
+
+export type { ConfigType };
+export const defaultConfig = configDefaults;

packages/media-manager/backends/local.ts

@@ -0,0 +1,64 @@
+import type { ConvertableMediaFormats } from "../media-converter";
+import { MediaConverter } from "../media-converter";
+import { MediaBackend, MediaBackendType, MediaHasher } from "..";
+import type { ConfigType } from "config-manager";
+
+export class LocalMediaBackend extends MediaBackend {
+	constructor(private config: ConfigType) {
+		super(MediaBackendType.LOCAL);
+	}
+
+	public async addFile(file: File) {
+		if (this.shouldConvertImages(this.config)) {
+			const fileExtension = file.name.split(".").pop();
+			const mediaConverter = new MediaConverter(
+				fileExtension as ConvertableMediaFormats,
+				this.config.media.conversion
+					.convert_to as ConvertableMediaFormats
+			);
+			file = await mediaConverter.convert(file);
+		}
+
+		const hash = await new MediaHasher().getMediaHash(file);
+
+		const newFile = Bun.file(
+			`${this.config.media.local_uploads_folder}/${hash}`
+		);
+
+		if (await newFile.exists()) {
+			throw new Error("File already exists");
+		}
+
+		await Bun.write(newFile, file);
+
+		return {
+			uploadedFile: file,
+			path: `./uploads/${file.name}`,
+			hash: hash,
+		};
+	}
+
+	public async getFileByHash(
+		hash: string,
+		databaseHashFetcher: (sha256: string) => Promise<string | null>
+	): Promise<File | null> {
+		const filename = await databaseHashFetcher(hash);
+
+		if (!filename) return null;
+
+		return this.getFile(filename);
+	}
+
+	public async getFile(filename: string): Promise<File | null> {
+		const file = Bun.file(
+			`${this.config.media.local_uploads_folder}/${filename}`
+		);
+
+		if (!(await file.exists())) return null;
+
+		return new File([await file.arrayBuffer()], filename, {
+			type: file.type,
+			lastModified: file.lastModified,
+		});
+	}
+}

packages/media-manager/backends/s3.ts

@@ -0,0 +1,69 @@
+import { S3Client } from "@bradenmacdonald/s3-lite-client";
+import type { ConvertableMediaFormats } from "../media-converter";
+import { MediaConverter } from "../media-converter";
+import { MediaBackend, MediaBackendType, MediaHasher } from "..";
+import type { ConfigType } from "config-manager";
+
+export class S3MediaBackend extends MediaBackend {
+	constructor(
+		private config: ConfigType,
+		private s3Client = new S3Client({
+			endPoint: config.s3.endpoint,
+			useSSL: true,
+			region: config.s3.region || "auto",
+			bucket: config.s3.bucket_name,
+			accessKey: config.s3.access_key,
+			secretKey: config.s3.secret_access_key,
+		})
+	) {
+		super(MediaBackendType.S3);
+	}
+
+	public async addFile(file: File) {
+		if (this.shouldConvertImages(this.config)) {
+			const fileExtension = file.name.split(".").pop();
+			const mediaConverter = new MediaConverter(
+				fileExtension as ConvertableMediaFormats,
+				this.config.media.conversion
+					.convert_to as ConvertableMediaFormats
+			);
+			file = await mediaConverter.convert(file);
+		}
+
+		const hash = await new MediaHasher().getMediaHash(file);
+
+		await this.s3Client.putObject(file.name, file.stream(), {
+			size: file.size,
+		});
+
+		return {
+			uploadedFile: file,
+			hash: hash,
+		};
+	}
+
+	public async getFileByHash(
+		hash: string,
+		databaseHashFetcher: (sha256: string) => Promise<string | null>
+	): Promise<File | null> {
+		const filename = await databaseHashFetcher(hash);
+
+		if (!filename) return null;
+
+		return this.getFile(filename);
+	}
+
+	public async getFile(filename: string): Promise<File | null> {
+		try {
+			await this.s3Client.statObject(filename);
+		} catch {
+			return null;
+		}
+
+		const file = await this.s3Client.getObject(filename);
+
+		return new File([await file.arrayBuffer()], filename, {
+			type: file.headers.get("Content-Type") || "undefined",
+		});
+	}
+}

packages/media-manager/bunfig.toml

@@ -0,0 +1,2 @@
+[install.scopes]
+"@jsr" = "https://npm.jsr.io"

packages/media-manager/index.ts

@@ -0,0 +1,80 @@
+import type { ConfigType } from "config-manager";
+
+export enum MediaBackendType {
+	LOCAL = "local",
+	S3 = "s3",
+}
+
+interface UploadedFileMetadata {
+	uploadedFile: File;
+	path?: string;
+	hash: string;
+}
+
+export class MediaHasher {
+	/**
+	 * Returns the SHA-256 hash of a file in hex format
+	 * @param media The file to hash
+	 * @returns The SHA-256 hash of the file in hex format
+	 */
+	public async getMediaHash(media: File) {
+		const hash = new Bun.SHA256()
+			.update(await media.arrayBuffer())
+			.digest("hex");
+
+		return hash;
+	}
+}
+
+export class MediaBackend {
+	constructor(private backend: MediaBackendType) {}
+
+	public getBackendType() {
+		return this.backend;
+	}
+
+	public shouldConvertImages(config: ConfigType) {
+		return config.media.conversion.convert_images;
+	}
+
+	/**
+	 * Fetches file from backend from SHA-256 hash
+	 * @param file SHA-256 hash of wanted file
+	 * @param databaseHashFetcher Function that takes in a sha256 hash as input and outputs the filename of that file in the database
+	 * @returns The file as a File object
+	 */
+	public getFileByHash(
+		// eslint-disable-next-line @typescript-eslint/no-unused-vars
+		file: string,
+		// eslint-disable-next-line @typescript-eslint/no-unused-vars
+		databaseHashFetcher: (sha256: string) => Promise<string>
+	): Promise<File | null> {
+		return Promise.reject(
+			new Error("Do not call MediaBackend directly: use a subclass")
+		);
+	}
+
+	/**
+	 * Fetches file from backend from filename
+	 * @param filename File name
+	 * @returns The file as a File object
+	 */
+	// eslint-disable-next-line @typescript-eslint/no-unused-vars
+	public getFile(filename: string): Promise<File | null> {
+		return Promise.reject(
+			new Error("Do not call MediaBackend directly: use a subclass")
+		);
+	}
+
+	/**
+	 * Adds file to backend
+	 * @param file File to add
+	 * @returns Metadata about the uploaded file
+	 */
+	// eslint-disable-next-line @typescript-eslint/no-unused-vars
+	public addFile(file: File): Promise<UploadedFileMetadata> {
+		return Promise.reject(
+			new Error("Do not call MediaBackend directly: use a subclass")
+		);
+	}
+}

packages/media-manager/media-converter.ts

@@ -0,0 +1,94 @@
+/**
+ * @packageDocumentation
+ * @module MediaManager
+ * @description Handles media conversion between formats
+ */
+import sharp from "sharp";
+
+export enum ConvertableMediaFormats {
+	PNG = "png",
+	WEBP = "webp",
+	JPEG = "jpeg",
+	JPG = "jpg",
+	AVIF = "avif",
+	JXL = "jxl",
+	HEIF = "heif",
+}
+
+/**
+ * Handles media conversion between formats
+ */
+export class MediaConverter {
+	constructor(
+		public fromFormat: ConvertableMediaFormats,
+		public toFormat: ConvertableMediaFormats
+	) {}
+
+	/**
+	 * Returns whether the media is convertable
+	 * @returns Whether the media is convertable
+	 */
+	public isConvertable() {
+		return (
+			this.fromFormat !== this.toFormat &&
+			Object.values(ConvertableMediaFormats).includes(this.fromFormat)
+		);
+	}
+
+	/**
+	 * Returns the file name with the extension replaced
+	 * @param fileName File name to replace
+	 * @returns File name with extension replaced
+	 */
+	private getReplacedFileName(fileName: string) {
+		return this.extractFilenameFromPath(fileName).replace(
+			new RegExp(`\\.${this.fromFormat}$`),
+			`.${this.toFormat}`
+		);
+	}
+
+	/**
+	 * Extracts the filename from a path
+	 * @param path Path to extract filename from
+	 * @returns Extracted filename
+	 */
+	private extractFilenameFromPath(path: string) {
+		// Don't count escaped slashes as path separators
+		const pathParts = path.split(/(?<!\\)\//);
+		return pathParts[pathParts.length - 1];
+	}
+
+	/**
+	 * Converts media to the specified format
+	 * @param media Media to convert
+	 * @returns Converted media
+	 */
+	public async convert(media: File) {
+		if (!this.isConvertable()) {
+			return media;
+		}
+
+		const sharpCommand = sharp(await media.arrayBuffer());
+
+		// Calculate newFilename before changing formats to prevent errors with jpg files
+		const newFilename = this.getReplacedFileName(media.name);
+
+		if (this.fromFormat === ConvertableMediaFormats.JPG) {
+			this.fromFormat = ConvertableMediaFormats.JPEG;
+		}
+
+		if (this.toFormat === ConvertableMediaFormats.JPG) {
+			this.toFormat = ConvertableMediaFormats.JPEG;
+		}
+
+		const convertedBuffer = await sharpCommand[this.toFormat]().toBuffer();
+
+		// Convert the buffer to a BlobPart
+		const buffer = new Blob([convertedBuffer]);
+
+		return new File([buffer], newFilename, {
+			type: `image/${this.toFormat}`,
+			lastModified: Date.now(),
+		});
+	}
+}

packages/media-manager/package.json

@@ -0,0 +1,6 @@
+{
+  "name": "media-manager",
+  "version": "0.0.0",
+  "main": "index.ts",
+  "dependencies": { "@bradenmacdonald/s3-lite-client": "npm:@jsr/bradenmacdonald__s3-lite-client" }
+}

packages/media-manager/tests/media-backends.test.ts

@@ -0,0 +1,243 @@
+// FILEPATH: /home/jessew/Dev/lysand/packages/media-manager/backends/s3.test.ts
+import { MediaBackend, MediaBackendType, MediaHasher } from "..";
+import type { S3Client } from "@bradenmacdonald/s3-lite-client";
+import { beforeEach, describe, jest, it, expect, spyOn } from "bun:test";
+import { S3MediaBackend } from "../backends/s3";
+import type { ConfigType } from "config-manager";
+import { ConvertableMediaFormats, MediaConverter } from "../media-converter";
+import { LocalMediaBackend } from "../backends/local";
+
+type DeepPartial<T> = {
+	[P in keyof T]?: DeepPartial<T[P]>;
+};
+
+describe("MediaBackend", () => {
+	let mediaBackend: MediaBackend;
+	let mockConfig: ConfigType;
+
+	beforeEach(() => {
+		mediaBackend = new MediaBackend(MediaBackendType.S3);
+		mockConfig = {
+			media: {
+				conversion: {
+					convert_images: true,
+				},
+			},
+		} as ConfigType;
+	});
+
+	it("should initialize with correct backend type", () => {
+		expect(mediaBackend.getBackendType()).toEqual(MediaBackendType.S3);
+	});
+
+	it("should check if images should be converted", () => {
+		expect(mediaBackend.shouldConvertImages(mockConfig)).toBe(true);
+		mockConfig.media.conversion.convert_images = false;
+		expect(mediaBackend.shouldConvertImages(mockConfig)).toBe(false);
+	});
+
+	it("should throw error when calling getFileByHash", () => {
+		const mockHash = "test-hash";
+		const databaseHashFetcher = jest.fn().mockResolvedValue("test.jpg");
+
+		expect(
+			mediaBackend.getFileByHash(mockHash, databaseHashFetcher)
+		).rejects.toThrow(Error);
+	});
+
+	it("should throw error when calling getFile", () => {
+		const mockFilename = "test.jpg";
+
+		expect(mediaBackend.getFile(mockFilename)).rejects.toThrow(Error);
+	});
+
+	it("should throw error when calling addFile", () => {
+		const mockFile = new File([""], "test.jpg");
+
+		expect(mediaBackend.addFile(mockFile)).rejects.toThrow();
+	});
+});
+
+describe("S3MediaBackend", () => {
+	let s3MediaBackend: S3MediaBackend;
+	let mockS3Client: Partial<S3Client>;
+	let mockConfig: DeepPartial<ConfigType>;
+	let mockFile: File;
+	let mockMediaHasher: MediaHasher;
+
+	beforeEach(() => {
+		mockConfig = {
+			s3: {
+				endpoint: "http://localhost:4566",
+				region: "us-east-1",
+				bucket_name: "test-bucket",
+				access_key: "test-access-key",
+				secret_access_key: "test-secret-access-key",
+				public_url: "test",
+			},
+			media: {
+				conversion: {
+					convert_to: ConvertableMediaFormats.PNG,
+				},
+			},
+		};
+		mockFile = new File([new TextEncoder().encode("test")], "test.jpg");
+		mockMediaHasher = new MediaHasher();
+		mockS3Client = {
+			putObject: jest.fn().mockResolvedValue({}),
+			statObject: jest.fn().mockResolvedValue({}),
+			getObject: jest.fn().mockResolvedValue({
+				blob: jest.fn().mockResolvedValue(new Blob()),
+				headers: new Headers({ "Content-Type": "image/jpeg" }),
+			}),
+		} as Partial<S3Client>;
+		s3MediaBackend = new S3MediaBackend(
+			mockConfig as ConfigType,
+			mockS3Client as S3Client
+		);
+	});
+
+	it("should initialize with correct type", () => {
+		expect(s3MediaBackend.getBackendType()).toEqual(MediaBackendType.S3);
+	});
+
+	it("should add file", async () => {
+		const mockHash = "test-hash";
+		spyOn(mockMediaHasher, "getMediaHash").mockResolvedValue(mockHash);
+
+		const result = await s3MediaBackend.addFile(mockFile);
+
+		expect(result.uploadedFile).toEqual(mockFile);
+		expect(result.hash).toHaveLength(64);
+		expect(mockS3Client.putObject).toHaveBeenCalledWith(
+			mockFile.name,
+			expect.any(ReadableStream),
+			{ size: mockFile.size }
+		);
+	});
+
+	it("should get file by hash", async () => {
+		const mockHash = "test-hash";
+		const mockFilename = "test.jpg";
+		const databaseHashFetcher = jest.fn().mockResolvedValue(mockFilename);
+		mockS3Client.statObject = jest.fn().mockResolvedValue({});
+		mockS3Client.getObject = jest.fn().mockResolvedValue({
+			arrayBuffer: jest.fn().mockResolvedValue(new ArrayBuffer(10)),
+			headers: new Headers({ "Content-Type": "image/jpeg" }),
+		});
+
+		const file = await s3MediaBackend.getFileByHash(
+			mockHash,
+			databaseHashFetcher
+		);
+
+		expect(file).not.toBeNull();
+		expect(file?.name).toEqual(mockFilename);
+		expect(file?.type).toEqual("image/jpeg");
+	});
+
+	it("should get file", async () => {
+		const mockFilename = "test.jpg";
+		mockS3Client.statObject = jest.fn().mockResolvedValue({});
+		mockS3Client.getObject = jest.fn().mockResolvedValue({
+			arrayBuffer: jest.fn().mockResolvedValue(new ArrayBuffer(10)),
+			headers: new Headers({ "Content-Type": "image/jpeg" }),
+		});
+
+		const file = await s3MediaBackend.getFile(mockFilename);
+
+		expect(file).not.toBeNull();
+		expect(file?.name).toEqual(mockFilename);
+		expect(file?.type).toEqual("image/jpeg");
+	});
+});
+
+describe("LocalMediaBackend", () => {
+	let localMediaBackend: LocalMediaBackend;
+	let mockConfig: ConfigType;
+	let mockFile: File;
+	let mockMediaHasher: MediaHasher;
+
+	beforeEach(() => {
+		mockConfig = {
+			media: {
+				conversion: {
+					convert_images: true,
+					convert_to: ConvertableMediaFormats.PNG,
+				},
+				local_uploads_folder: "./uploads",
+			},
+		} as ConfigType;
+		mockFile = Bun.file(__dirname + "/megamind.jpg") as unknown as File;
+		mockMediaHasher = new MediaHasher();
+		localMediaBackend = new LocalMediaBackend(mockConfig);
+	});
+
+	it("should initialize with correct type", () => {
+		expect(localMediaBackend.getBackendType()).toEqual(
+			MediaBackendType.LOCAL
+		);
+	});
+
+	it("should add file", async () => {
+		const mockHash = "test-hash";
+		spyOn(mockMediaHasher, "getMediaHash").mockResolvedValue(mockHash);
+		const mockMediaConverter = new MediaConverter(
+			ConvertableMediaFormats.JPG,
+			ConvertableMediaFormats.PNG
+		);
+		spyOn(mockMediaConverter, "convert").mockResolvedValue(mockFile);
+		// @ts-expect-error This is a mock
+		spyOn(Bun, "file").mockImplementationOnce(() => ({
+			exists: () => Promise.resolve(false),
+		}));
+		spyOn(Bun, "write").mockImplementationOnce(() =>
+			Promise.resolve(mockFile.size)
+		);
+
+		const result = await localMediaBackend.addFile(mockFile);
+
+		expect(result.uploadedFile).toEqual(mockFile);
+		expect(result.path).toEqual(`./uploads/megamind.png`);
+		expect(result.hash).toHaveLength(64);
+	});
+
+	it("should get file by hash", async () => {
+		const mockHash = "test-hash";
+		const mockFilename = "test.jpg";
+		const databaseHashFetcher = jest.fn().mockResolvedValue(mockFilename);
+		// @ts-expect-error This is a mock
+		spyOn(Bun, "file").mockImplementationOnce(() => ({
+			exists: () => Promise.resolve(true),
+			arrayBuffer: () => Promise.resolve(new ArrayBuffer(8)),
+			type: "image/jpeg",
+			lastModified: 123456789,
+		}));
+
+		const file = await localMediaBackend.getFileByHash(
+			mockHash,
+			databaseHashFetcher
+		);
+
+		expect(file).not.toBeNull();
+		expect(file?.name).toEqual(mockFilename);
+		expect(file?.type).toEqual("image/jpeg");
+	});
+
+	it("should get file", async () => {
+		const mockFilename = "test.jpg";
+		// @ts-expect-error This is a mock
+		spyOn(Bun, "file").mockImplementationOnce(() => ({
+			exists: () => Promise.resolve(true),
+			arrayBuffer: () => Promise.resolve(new ArrayBuffer(8)),
+			type: "image/jpeg",
+			lastModified: 123456789,
+		}));
+
+		const file = await localMediaBackend.getFile(mockFilename);
+
+		expect(file).not.toBeNull();
+		expect(file?.name).toEqual(mockFilename);
+		expect(file?.type).toEqual("image/jpeg");
+	});
+});

packages/media-manager/tests/media-manager.test.ts

@@ -0,0 +1,65 @@
+// FILEPATH: /home/jessew/Dev/lysand/packages/media-manager/media-converter.test.ts
+import { describe, it, expect, beforeEach } from "bun:test";
+import { MediaConverter, ConvertableMediaFormats } from "../media-converter";
+
+describe("MediaConverter", () => {
+	let mediaConverter: MediaConverter;
+
+	beforeEach(() => {
+		mediaConverter = new MediaConverter(
+			ConvertableMediaFormats.JPG,
+			ConvertableMediaFormats.PNG
+		);
+	});
+
+	it("should initialize with correct formats", () => {
+		expect(mediaConverter.fromFormat).toEqual(ConvertableMediaFormats.JPG);
+		expect(mediaConverter.toFormat).toEqual(ConvertableMediaFormats.PNG);
+	});
+
+	it("should check if media is convertable", () => {
+		expect(mediaConverter.isConvertable()).toBe(true);
+		mediaConverter.toFormat = ConvertableMediaFormats.JPG;
+		expect(mediaConverter.isConvertable()).toBe(false);
+	});
+
+	it("should replace file name extension", () => {
+		const fileName = "test.jpg";
+		const expectedFileName = "test.png";
+		// Written like this because it's a private function
+		expect(mediaConverter["getReplacedFileName"](fileName)).toEqual(
+			expectedFileName
+		);
+	});
+
+	describe("Filename extractor", () => {
+		it("should extract filename from path", () => {
+			const path = "path/to/test.jpg";
+			const expectedFileName = "test.jpg";
+			expect(mediaConverter["extractFilenameFromPath"](path)).toEqual(
+				expectedFileName
+			);
+		});
+
+		it("should handle escaped slashes", () => {
+			const path = "path/to/test\\/test.jpg";
+			const expectedFileName = "test\\/test.jpg";
+			expect(mediaConverter["extractFilenameFromPath"](path)).toEqual(
+				expectedFileName
+			);
+		});
+	});
+
+	it("should convert media", async () => {
+		const file = Bun.file(__dirname + "/megamind.jpg");
+
+		const convertedFile = await mediaConverter.convert(
+			file as unknown as File
+		);
+
+		expect(convertedFile.name).toEqual("megamind.png");
+		expect(convertedFile.type).toEqual(
+			`image/${ConvertableMediaFormats.PNG}`
+		);
+	});
+});