versia-pub/server
2
0
Fork 0
mirror of https://github.com/versia-pub/server.git synced 2025-12-06 08:28:19 +01:00
Code Issues Actions Packages Projects Releases Wiki Activity

refactor: ♻️ Rewrite media management code

Browse source
This commit is contained in:
Jesse Wierzbinski 2024-06-28 20:10:02 -10:00
parent d09f74e58a
commit faf829437d
No known key found for this signature in database
34 changed files with 1195 additions and 904 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

BIN
bun.lockb
View file

Binary file not shown.

118
classes/media/drivers/disk.test.ts Normal file
View file

@ -0,0 +1,118 @@
/**
* @packageDocumentation
* @module Tests/DiskMediaDriver
*/
import {
type Mock,
beforeEach,
describe,
expect,
it,
mock,
spyOn,
} from "bun:test";
import * as fs from "node:fs/promises";
import * as path from "node:path";
import type { Config } from "config-manager";
import type { MediaHasher } from "../media-hasher";
import { DiskMediaDriver } from "./disk";
describe("DiskMediaDriver", () => {
let diskDriver: DiskMediaDriver;
let mockConfig: Config;
let mockMediaHasher: MediaHasher;
let bunWriteSpy: Mock<typeof Bun.write>;
beforeEach(() => {
mockConfig = {
media: {
local_uploads_folder: "/test/uploads",
},
} as Config;
mockMediaHasher = mock(() => ({
getMediaHash: mock(() => Promise.resolve("testhash")),
}))();
diskDriver = new DiskMediaDriver(mockConfig);
// @ts-ignore: Replacing private property for testing
diskDriver.mediaHasher = mockMediaHasher;
// Mock fs.promises methods
mock.module("node:fs/promises", () => ({
writeFile: mock(() => Promise.resolve()),
rm: mock(() => {
return Promise.resolve();
}),
}));
spyOn(Bun, "file").mockImplementation(
mock(() => ({
exists: mock(() => Promise.resolve(true)),
arrayBuffer: mock(() => Promise.resolve(new ArrayBuffer(8))),
type: "image/webp",
lastModified: Date.now(),
})) as unknown as typeof Bun.file,
);
bunWriteSpy = spyOn(Bun, "write").mockImplementation(
mock(() => Promise.resolve(0)),
);
});
it("should add a file", async () => {
const file = new File(["test"], "test.webp", { type: "image/webp" });
const result = await diskDriver.addFile(file);
expect(mockMediaHasher.getMediaHash).toHaveBeenCalledWith(file);
expect(bunWriteSpy).toHaveBeenCalledWith(
path.join("/test/uploads", "testhash", "test.webp"),
expect.any(ArrayBuffer),
);
expect(result).toEqual({
uploadedFile: file,
path: path.join("testhash", "test.webp"),
hash: "testhash",
});
});
it("should get a file by hash", async () => {
const hash = "testhash";
const databaseHashFetcher = mock(() => Promise.resolve("test.webp"));
const result = await diskDriver.getFileByHash(
hash,
databaseHashFetcher,
);
expect(databaseHashFetcher).toHaveBeenCalledWith(hash);
expect(Bun.file).toHaveBeenCalledWith(
path.join("/test/uploads", "test.webp"),
);
expect(result).toBeInstanceOf(File);
expect(result?.name).toBe("test.webp");
expect(result?.type).toBe("image/webp");
});
it("should get a file by filename", async () => {
const filename = "test.webp";
const result = await diskDriver.getFile(filename);
expect(Bun.file).toHaveBeenCalledWith(
path.join("/test/uploads", filename),
);
expect(result).toBeInstanceOf(File);
expect(result?.name).toBe(filename);
expect(result?.type).toBe("image/webp");
});
it("should delete a file by URL", async () => {
const url = "http://localhost:3000/uploads/testhash/test.webp";
await diskDriver.deleteFileByUrl(url);
expect(fs.rm).toHaveBeenCalledWith(
path.join("/test/uploads", "testhash"),
{ recursive: true },
);
});
});

91
classes/media/drivers/disk.ts Normal file
View file

@ -0,0 +1,91 @@
/**
* @packageDocumentation
* @module MediaManager/Drivers
*/
import { rm } from "node:fs/promises";
import { join } from "node:path";
import type { Config } from "config-manager";
import { MediaHasher } from "../media-hasher";
import type { UploadedFileMetadata } from "../media-manager";
import type { MediaDriver } from "./media-driver";
/**
* Implements the MediaDriver interface for disk storage.
*/
export class DiskMediaDriver implements MediaDriver {
private mediaHasher: MediaHasher;
/**
* Creates a new DiskMediaDriver instance.
* @param config - The configuration object.
*/
constructor(private config: Config) {
this.mediaHasher = new MediaHasher();
}
/**
* @inheritdoc
*/
public async addFile(
file: File,
): Promise<Omit<UploadedFileMetadata, "blurhash">> {
const hash = await this.mediaHasher.getMediaHash(file);
const path = join(hash, file.name);
const fullPath = join(this.config.media.local_uploads_folder, path);
await Bun.write(fullPath, await file.arrayBuffer());
return {
uploadedFile: file,
path,
hash,
};
}
/**
* @inheritdoc
*/
public async getFileByHash(
hash: string,
databaseHashFetcher: (sha256: string) => Promise<string | null>,
): Promise<File | null> {
const filename = await databaseHashFetcher(hash);
if (!filename) {
return null;
}
return this.getFile(filename);
}
/**
* @inheritdoc
*/
public async getFile(filename: string): Promise<File | null> {
const fullPath = join(this.config.media.local_uploads_folder, filename);
try {
const file = Bun.file(fullPath);
if (await file.exists()) {
return new File([await file.arrayBuffer()], filename, {
type: file.type,
lastModified: file.lastModified,
});
}
} catch {
// File doesn't exist or can't be read
}
return null;
}
/**
* @inheritdoc
*/
public async deleteFileByUrl(url: string): Promise<void> {
const urlObj = new URL(url);
const hash = urlObj.pathname.split("/").at(-2);
if (!hash) {
throw new Error("Invalid URL");
}
const dirPath = join(this.config.media.local_uploads_folder, hash);
await rm(dirPath, { recursive: true });
}
}

43
classes/media/drivers/media-driver.ts Normal file
View file

@ -0,0 +1,43 @@
/**
* @packageDocumentation
* @module MediaManager/Drivers
*/
import type { UploadedFileMetadata } from "../media-manager";
/**
* Represents a media storage driver.
*/
export interface MediaDriver {
/**
* Adds a file to the media storage.
* @param file - The file to add.
* @returns A promise that resolves to the metadata of the uploaded file.
*/
addFile(file: File): Promise<Omit<UploadedFileMetadata, "blurhash">>;
/**
* Retrieves a file from the media storage by its hash.
* @param hash - The hash of the file to retrieve.
* @param databaseHashFetcher - A function to fetch the filename from the database.
* @returns A promise that resolves to the file or null if not found.
*/
getFileByHash(
hash: string,
databaseHashFetcher: (sha256: string) => Promise<string | null>,
): Promise<File | null>;
/**
* Retrieves a file from the media storage by its filename.
* @param filename - The name of the file to retrieve.
* @returns A promise that resolves to the file or null if not found.
*/
getFile(filename: string): Promise<File | null>;
/**
* Deletes a file from the media storage by its URL.
* @param url - The URL of the file to delete.
* @returns A promise that resolves when the file is deleted.
*/
deleteFileByUrl(url: string): Promise<void>;
}

101
classes/media/drivers/s3.test.ts Normal file
View file

@ -0,0 +1,101 @@
/**
* @packageDocumentation
* @module Tests/S3MediaDriver
*/
import { beforeEach, describe, expect, it, mock } from "bun:test";
import type { S3Client } from "@jsr/bradenmacdonald__s3-lite-client";
import type { Config } from "config-manager";
import type { MediaHasher } from "../media-hasher";
import { S3MediaDriver } from "./s3";
describe("S3MediaDriver", () => {
let s3Driver: S3MediaDriver;
let mockConfig: Config;
let mockS3Client: S3Client;
let mockMediaHasher: MediaHasher;
beforeEach(() => {
mockConfig = {
s3: {
endpoint: "s3.amazonaws.com",
region: "us-west-2",
bucket_name: "test-bucket",
access_key: "test-key",
secret_access_key: "test-secret",
},
} as Config;
mockS3Client = mock(() => ({
putObject: mock(() => Promise.resolve()),
getObject: mock(() =>
Promise.resolve({
arrayBuffer: () => Promise.resolve(new ArrayBuffer(8)),
headers: new Headers({ "Content-Type": "image/webp" }),
}),
),
statObject: mock(() => Promise.resolve()),
deleteObject: mock(() => Promise.resolve()),
}))() as unknown as S3Client;
mockMediaHasher = mock(() => ({
getMediaHash: mock(() => Promise.resolve("testhash")),
}))();
s3Driver = new S3MediaDriver(mockConfig);
// @ts-ignore: Replacing private property for testing
s3Driver.s3Client = mockS3Client;
// @ts-ignore: Replacing private property for testing
s3Driver.mediaHasher = mockMediaHasher;
});
it("should add a file", async () => {
const file = new File(["test"], "test.webp", { type: "image/webp" });
const result = await s3Driver.addFile(file);
expect(mockMediaHasher.getMediaHash).toHaveBeenCalledWith(file);
expect(mockS3Client.putObject).toHaveBeenCalledWith(
"testhash/test.webp",
expect.any(ReadableStream),
{ size: file.size },
);
expect(result).toEqual({
uploadedFile: file,
path: "testhash/test.webp",
hash: "testhash",
});
});
it("should get a file by hash", async () => {
const hash = "testhash";
const databaseHashFetcher = mock(() => Promise.resolve("test.webp"));
const result = await s3Driver.getFileByHash(hash, databaseHashFetcher);
expect(databaseHashFetcher).toHaveBeenCalledWith(hash);
expect(mockS3Client.statObject).toHaveBeenCalledWith("test.webp");
expect(mockS3Client.getObject).toHaveBeenCalledWith("test.webp");
expect(result).toBeInstanceOf(File);
expect(result?.name).toBe("test.webp");
expect(result?.type).toBe("image/webp");
});
it("should get a file by filename", async () => {
const filename = "test.webp";
const result = await s3Driver.getFile(filename);
expect(mockS3Client.statObject).toHaveBeenCalledWith(filename);
expect(mockS3Client.getObject).toHaveBeenCalledWith(filename);
expect(result).toBeInstanceOf(File);
expect(result?.name).toBe(filename);
expect(result?.type).toBe("image/webp");
});
it("should delete a file by URL", async () => {
const url = "https://test-bucket.s3.amazonaws.com/test/test.webp";
await s3Driver.deleteFileByUrl(url);
expect(mockS3Client.deleteObject).toHaveBeenCalledWith(
"test/test.webp",
);
});
});

93
classes/media/drivers/s3.ts Normal file
View file

@ -0,0 +1,93 @@
/**
* @packageDocumentation
* @module MediaManager/Drivers
*/
import { S3Client } from "@jsr/bradenmacdonald__s3-lite-client";
import type { Config } from "config-manager";
import { MediaHasher } from "../media-hasher";
import type { UploadedFileMetadata } from "../media-manager";
import type { MediaDriver } from "./media-driver";
/**
* Implements the MediaDriver interface for S3 storage.
*/
export class S3MediaDriver implements MediaDriver {
private s3Client: S3Client;
private mediaHasher: MediaHasher;
/**
* Creates a new S3MediaDriver instance.
* @param config - The configuration object.
*/
constructor(config: Config) {
this.s3Client = new S3Client({
endPoint: config.s3.endpoint,
useSSL: true,
region: config.s3.region || "auto",
bucket: config.s3.bucket_name,
accessKey: config.s3.access_key,
secretKey: config.s3.secret_access_key,
});
this.mediaHasher = new MediaHasher();
}
/**
* @inheritdoc
*/
public async addFile(
file: File,
): Promise<Omit<UploadedFileMetadata, "blurhash">> {
const hash = await this.mediaHasher.getMediaHash(file);
const path = `${hash}/${file.name}`;
await this.s3Client.putObject(path, file.stream(), {
size: file.size,
});
return {
uploadedFile: file,
path,
hash,
};
}
/**
* @inheritdoc
*/
public async getFileByHash(
hash: string,
databaseHashFetcher: (sha256: string) => Promise<string | null>,
): Promise<File | null> {
const filename = await databaseHashFetcher(hash);
if (!filename) {
return null;
}
return this.getFile(filename);
}
/**
* @inheritdoc
*/
public async getFile(filename: string): Promise<File | null> {
try {
await this.s3Client.statObject(filename);
const file = await this.s3Client.getObject(filename);
const arrayBuffer = await file.arrayBuffer();
return new File([arrayBuffer], filename, {
type: file.headers.get("Content-Type") || undefined,
});
} catch {
return null;
}
}
/**
* @inheritdoc
*/
public async deleteFileByUrl(url: string): Promise<void> {
const urlObj = new URL(url);
const path = urlObj.pathname.slice(1); // Remove leading slash
await this.s3Client.deleteObject(path);
}
}

20
classes/media/media-hasher.ts Normal file
View file

@ -0,0 +1,20 @@
/**
* @packageDocumentation
* @module MediaManager/Utils
*/
/**
* Utility class for hashing media files.
*/
export class MediaHasher {
/**
* Generates a SHA-256 hash for a given file.
* @param file - The file to hash.
* @returns A promise that resolves to the SHA-256 hash of the file in hex format.
*/
public async getMediaHash(file: File): Promise<string> {
const arrayBuffer = await file.arrayBuffer();
const hash = new Bun.SHA256().update(arrayBuffer).digest("hex");
return hash;
}
}

124
classes/media/media-manager.test.ts Normal file
View file

@ -0,0 +1,124 @@
/**
* @packageDocumentation
* @module Tests/MediaManager
*/
import { beforeEach, describe, expect, it, mock } from "bun:test";
import type { Config } from "config-manager";
import { MediaBackendType } from "~/packages/config-manager/config.type";
import { DiskMediaDriver } from "./drivers/disk";
import { S3MediaDriver } from "./drivers/s3";
import { MediaManager } from "./media-manager";
import type { ImageConversionPreprocessor } from "./preprocessors/image-conversion";
describe("MediaManager", () => {
let mediaManager: MediaManager;
let mockConfig: Config;
let mockS3Driver: S3MediaDriver;
let mockImagePreprocessor: ImageConversionPreprocessor;
beforeEach(() => {
mockConfig = {
media: {
backend: "s3",
conversion: {
convert_images: true,
convert_to: "image/webp",
},
},
s3: {
endpoint: "s3.amazonaws.com",
region: "us-west-2",
bucket_name: "test-bucket",
access_key: "test-key",
secret_access_key: "test-secret",
},
} as Config;
mockS3Driver = mock(() => ({
addFile: mock(() =>
Promise.resolve({
uploadedFile: new File(["hey"], "test.webp"),
path: "test/test.webp",
hash: "testhash",
}),
),
getFileByHash: mock(() => {
return Promise.resolve(new File(["hey"], "test.webp"));
}),
getFile: mock(() =>
Promise.resolve(new File(["hey"], "test.webp")),
),
deleteFileByUrl: mock(() => Promise.resolve()),
}))() as unknown as S3MediaDriver;
mockImagePreprocessor = mock(() => ({
process: mock((_: File) =>
Promise.resolve(new File(["hey"], "test.webp")),
),
}))() as unknown as ImageConversionPreprocessor;
mediaManager = new MediaManager(mockConfig);
// @ts-expect-error: Accessing private property for testing
mediaManager.driver = mockS3Driver;
// @ts-expect-error: Accessing private property for testing
mediaManager.preprocessors = [mockImagePreprocessor];
});
it("should initialize with the correct driver based on config", () => {
const s3Manager = new MediaManager(mockConfig);
// @ts-expect-error: Accessing private property for testing
expect(s3Manager.driver).toBeInstanceOf(S3MediaDriver);
mockConfig.media.backend = MediaBackendType.Local;
const diskManager = new MediaManager(mockConfig);
// @ts-expect-error: Accessing private property for testing
expect(diskManager.driver).toBeInstanceOf(DiskMediaDriver);
});
it("should add a file with preprocessing", async () => {
const file = new File(["test"], "test.jpg", { type: "image/jpeg" });
const result = await mediaManager.addFile(file);
expect(mockImagePreprocessor.process).toHaveBeenCalledWith(file);
expect(mockS3Driver.addFile).toHaveBeenCalled();
expect(result).toEqual({
uploadedFile: new File(["hey"], "test.webp"),
path: "test/test.webp",
hash: "testhash",
blurhash: null,
});
});
it("should get a file by hash", async () => {
const hash = "testhash";
const databaseHashFetcher = mock(() => Promise.resolve("test.webp"));
const result = await mediaManager.getFileByHash(
hash,
databaseHashFetcher,
);
expect(mockS3Driver.getFileByHash).toHaveBeenCalledWith(
hash,
databaseHashFetcher,
);
expect(result).toBeInstanceOf(File);
expect(result?.name).toBe("test.webp");
});
it("should get a file by filename", async () => {
const filename = "test.webp";
const result = await mediaManager.getFile(filename);
expect(mockS3Driver.getFile).toHaveBeenCalledWith(filename);
expect(result).toBeInstanceOf(File);
expect(result?.name).toBe("test.webp");
});
it("should delete a file by URL", async () => {
const url = "https://test-bucket.s3.amazonaws.com/test/test.webp";
await mediaManager.deleteFileByUrl(url);
expect(mockS3Driver.deleteFileByUrl).toHaveBeenCalledWith(url);
});
});

132
classes/media/media-manager.ts Normal file
View file

@ -0,0 +1,132 @@
/**
* @packageDocumentation
* @module MediaManager
*/
import type { Config } from "config-manager";
import { DiskMediaDriver } from "./drivers/disk";
import type { MediaDriver } from "./drivers/media-driver";
import { S3MediaDriver } from "./drivers/s3";
import { BlurhashPreprocessor } from "./preprocessors/blurhash";
import { ImageConversionPreprocessor } from "./preprocessors/image-conversion";
import type { MediaPreprocessor } from "./preprocessors/media-preprocessor";
/**
* Manages media operations with support for different storage drivers and preprocessing plugins.
* @example
* const mediaManager = new MediaManager(config);
*
* const file = new File(["hello"], "hello.txt");
*
* const { path, hash, blurhash } = await mediaManager.addFile(file);
*
* const retrievedFile = await mediaManager.getFileByHash(hash, fetchHashFromDatabase);
*
* await mediaManager.deleteFileByUrl(path);
*/
export class MediaManager {
private driver: MediaDriver;
private preprocessors: MediaPreprocessor[] = [];
/**
* Creates a new MediaManager instance.
* @param config - The configuration object.
*/
constructor(private config: Config) {
this.driver = this.initializeDriver();
this.initializePreprocessors();
}
/**
* Initializes the appropriate media driver based on the configuration.
* @returns An instance of MediaDriver.
*/
private initializeDriver(): MediaDriver {
switch (this.config.media.backend) {
case "s3":
return new S3MediaDriver(this.config);
case "local":
return new DiskMediaDriver(this.config);
default:
throw new Error(
`Unsupported media backend: ${this.config.media.backend}`,
);
}
}
/**
* Initializes the preprocessors based on the configuration.
*/
private initializePreprocessors(): void {
if (this.config.media.conversion.convert_images) {
this.preprocessors.push(
new ImageConversionPreprocessor(this.config),
);
}
this.preprocessors.push(new BlurhashPreprocessor());
// Add other preprocessors here as needed
}
/**
* Adds a file to the media storage.
* @param file - The file to add.
* @returns A promise that resolves to the metadata of the uploaded file.
*/
public async addFile(file: File): Promise<UploadedFileMetadata> {
let processedFile = file;
let blurhash: string | null = null;
for (const preprocessor of this.preprocessors) {
const result = await preprocessor.process(processedFile);
if ("blurhash" in result) {
blurhash = result.blurhash as string;
processedFile = result.file;
} else {
processedFile = result.file;
}
}
const uploadResult = await this.driver.addFile(processedFile);
return { ...uploadResult, blurhash };
}
/**
* Retrieves a file from the media storage by its hash.
* @param hash - The hash of the file to retrieve.
* @param databaseHashFetcher - A function to fetch the filename from the database.
* @returns A promise that resolves to the file or null if not found.
*/
public getFileByHash(
hash: string,
databaseHashFetcher: (sha256: string) => Promise<string | null>,
): Promise<File | null> {
return this.driver.getFileByHash(hash, databaseHashFetcher);
}
/**
* Retrieves a file from the media storage by its filename.
* @param filename - The name of the file to retrieve.
* @returns A promise that resolves to the file or null if not found.
*/
public getFile(filename: string): Promise<File | null> {
return this.driver.getFile(filename);
}
/**
* Deletes a file from the media storage by its URL.
* @param url - The URL of the file to delete.
* @returns A promise that resolves when the file is deleted.
*/
public deleteFileByUrl(url: string): Promise<void> {
return this.driver.deleteFileByUrl(url);
}
}
/**
* Represents the metadata of an uploaded file.
*/
export interface UploadedFileMetadata {
uploadedFile: File;
path: string;
hash: string;
blurhash: string | null;
}

71
classes/media/preprocessors/blurhash.test.ts Normal file
View file

@ -0,0 +1,71 @@
import { beforeEach, describe, expect, it, mock } from "bun:test";
import sharp from "sharp";
import { BlurhashPreprocessor } from "./blurhash";
describe("BlurhashPreprocessor", () => {
let preprocessor: BlurhashPreprocessor;
beforeEach(() => {
preprocessor = new BlurhashPreprocessor();
});
it("should calculate blurhash for a valid image", async () => {
const inputBuffer = await sharp({
create: {
width: 100,
height: 100,
channels: 3,
background: { r: 255, g: 0, b: 0 },
},
})
.png()
.toBuffer();
const inputFile = new File([inputBuffer], "test.png", {
type: "image/png",
});
const result = await preprocessor.process(inputFile);
expect(result.file).toBe(inputFile);
expect(result.blurhash).toBeTypeOf("string");
expect(result.blurhash).not.toBe("");
});
it("should return null blurhash for an invalid image", async () => {
const invalidFile = new File(["invalid image data"], "invalid.png", {
type: "image/png",
});
const result = await preprocessor.process(invalidFile);
expect(result.file).toBe(invalidFile);
expect(result.blurhash).toBeNull();
});
it("should handle errors during blurhash calculation", async () => {
const inputBuffer = await sharp({
create: {
width: 100,
height: 100,
channels: 3,
background: { r: 255, g: 0, b: 0 },
},
})
.png()
.toBuffer();
const inputFile = new File([inputBuffer], "test.png", {
type: "image/png",
});
mock.module("blurhash", () => ({
encode: () => {
throw new Error("Test error");
},
}));
const result = await preprocessor.process(inputFile);
expect(result.file).toBe(inputFile);
expect(result.blurhash).toBeNull();
});
});

45
classes/media/preprocessors/blurhash.ts Normal file
View file

@ -0,0 +1,45 @@
import { encode } from "blurhash";
import sharp from "sharp";
import type { MediaPreprocessor } from "./media-preprocessor";
export class BlurhashPreprocessor implements MediaPreprocessor {
public async process(
file: File,
): Promise<{ file: File; blurhash: string | null }> {
try {
const arrayBuffer = await file.arrayBuffer();
const metadata = await sharp(arrayBuffer).metadata();
const blurhash = await new Promise<string | null>((resolve) => {
(async () =>
sharp(arrayBuffer)
.raw()
.ensureAlpha()
.toBuffer((err, buffer) => {
if (err) {
resolve(null);
return;
}
try {
resolve(
encode(
new Uint8ClampedArray(buffer),
metadata?.width ?? 0,
metadata?.height ?? 0,
4,
4,
) as string,
);
} catch {
resolve(null);
}
}))();
});
return { file, blurhash };
} catch {
return { file, blurhash: null };
}
}
}

156
classes/media/preprocessors/image-conversion.test.ts Normal file
View file

@ -0,0 +1,156 @@
import { beforeEach, describe, expect, it } from "bun:test";
import type { Config } from "config-manager";
import sharp from "sharp";
import { ImageConversionPreprocessor } from "./image-conversion";
describe("ImageConversionPreprocessor", () => {
let preprocessor: ImageConversionPreprocessor;
let mockConfig: Config;
beforeEach(() => {
mockConfig = {
media: {
conversion: {
convert_images: true,
convert_to: "image/webp",
convert_vector: false,
},
},
} as Config;
preprocessor = new ImageConversionPreprocessor(mockConfig);
});
it("should convert a JPEG image to WebP", async () => {
const inputBuffer = await sharp({
create: {
width: 100,
height: 100,
channels: 3,
background: { r: 255, g: 0, b: 0 },
},
})
.jpeg()
.toBuffer();
const inputFile = new File([inputBuffer], "test.jpg", {
type: "image/jpeg",
});
const result = await preprocessor.process(inputFile);
expect(result.file.type).toBe("image/webp");
expect(result.file.name).toBe("test.webp");
const resultBuffer = await result.file.arrayBuffer();
const metadata = await sharp(resultBuffer).metadata();
expect(metadata.format).toBe("webp");
});
it("should not convert SVG when convert_vector is false", async () => {
const svgContent =
'<svg xmlns="http://www.w3.org/2000/svg"><rect width="100" height="100" fill="red"/></svg>';
const inputFile = new File([svgContent], "test.svg", {
type: "image/svg+xml",
});
const result = await preprocessor.process(inputFile);
expect(result.file).toBe(inputFile);
});
it("should convert SVG when convert_vector is true", async () => {
mockConfig.media.conversion.convert_vector = true;
preprocessor = new ImageConversionPreprocessor(mockConfig);
const svgContent =
'<svg xmlns="http://www.w3.org/2000/svg"><rect width="100" height="100" fill="red"/></svg>';
const inputFile = new File([svgContent], "test.svg", {
type: "image/svg+xml",
});
const result = await preprocessor.process(inputFile);
expect(result.file.type).toBe("image/webp");
expect(result.file.name).toBe("test.webp");
});
it("should not convert unsupported file types", async () => {
const inputFile = new File(["test content"], "test.txt", {
type: "text/plain",
});
const result = await preprocessor.process(inputFile);
expect(result.file).toBe(inputFile);
});
it("should throw an error for unsupported output format", async () => {
mockConfig.media.conversion.convert_to = "image/bmp";
preprocessor = new ImageConversionPreprocessor(mockConfig);
const inputBuffer = await sharp({
create: {
width: 100,
height: 100,
channels: 3,
background: { r: 255, g: 0, b: 0 },
},
})
.png()
.toBuffer();
const inputFile = new File([inputBuffer], "test.png", {
type: "image/png",
});
await expect(preprocessor.process(inputFile)).rejects.toThrow(
"Unsupported output format: image/bmp",
);
});
it("should convert animated GIF to WebP while preserving animation", async () => {
// Create a simple animated GIF
const inputBuffer = await sharp({
create: {
width: 100,
height: 100,
channels: 4,
background: { r: 255, g: 0, b: 0, alpha: 1 },
},
})
.gif()
.toBuffer();
const inputFile = new File([inputBuffer], "animated.gif", {
type: "image/gif",
});
const result = await preprocessor.process(inputFile);
expect(result.file.type).toBe("image/webp");
expect(result.file.name).toBe("animated.webp");
const resultBuffer = await result.file.arrayBuffer();
const metadata = await sharp(resultBuffer).metadata();
expect(metadata.format).toBe("webp");
});
it("should handle files with spaces in the name", async () => {
const inputBuffer = await sharp({
create: {
width: 100,
height: 100,
channels: 3,
background: { r: 255, g: 0, b: 0 },
},
})
.png()
.toBuffer();
const inputFile = new File(
[inputBuffer],
"test image with spaces.png",
{ type: "image/png" },
);
const result = await preprocessor.process(inputFile);
expect(result.file.type).toBe("image/webp");
expect(result.file.name).toBe("test image with spaces.webp");
});
});

122
classes/media/preprocessors/image-conversion.ts Normal file
View file

@ -0,0 +1,122 @@
/**
* @packageDocumentation
* @module MediaManager/Preprocessors
*/
import type { Config } from "config-manager";
import sharp from "sharp";
import type { MediaPreprocessor } from "./media-preprocessor";
/**
* Supported input media formats.
*/
const supportedInputFormats = [
"image/png",
"image/jpeg",
"image/webp",
"image/avif",
"image/svg+xml",
"image/gif",
"image/tiff",
];
/**
* Supported output media formats.
*/
const supportedOutputFormats = [
"image/jpeg",
"image/png",
"image/webp",
"image/avif",
"image/gif",
"image/tiff",
];
/**
* Implements the MediaPreprocessor interface for image conversion.
*/
export class ImageConversionPreprocessor implements MediaPreprocessor {
/**
* Creates a new ImageConversionPreprocessor instance.
* @param config - The configuration object.
*/
constructor(private config: Config) {}
/**
* @inheritdoc
*/
public async process(file: File): Promise<{ file: File }> {
if (!this.isConvertible(file)) {
return { file };
}
const targetFormat = this.config.media.conversion.convert_to;
if (!supportedOutputFormats.includes(targetFormat)) {
throw new Error(`Unsupported output format: ${targetFormat}`);
}
const sharpCommand = sharp(await file.arrayBuffer(), {
animated: true,
});
const commandName = targetFormat.split("/")[1] as
| "jpeg"
| "png"
| "webp"
| "avif"
| "gif"
| "tiff";
const convertedBuffer = await sharpCommand[commandName]().toBuffer();
return {
file: new File(
[convertedBuffer],
this.getReplacedFileName(file.name, commandName),
{
type: targetFormat,
lastModified: Date.now(),
},
),
};
}
/**
* Checks if a file is convertible.
* @param file - The file to check.
* @returns True if the file is convertible, false otherwise.
*/
private isConvertible(file: File): boolean {
if (
file.type === "image/svg+xml" &&
!this.config.media.conversion.convert_vector
) {
return false;
}
return supportedInputFormats.includes(file.type);
}
/**
* Replaces the file extension in the filename.
* @param fileName - The original filename.
* @param newExtension - The new extension.
* @returns The filename with the new extension.
*/
private getReplacedFileName(
fileName: string,
newExtension: string,
): string {
return this.extractFilenameFromPath(fileName).replace(
/\.[^/.]+$/,
`.${newExtension}`,
);
}
/**
* Extracts the filename from a path.
* @param path - The path to extract the filename from.
* @returns The extracted filename.
*/
private extractFilenameFromPath(path: string): string {
const pathParts = path.split(/(?<!\\)\//);
return pathParts[pathParts.length - 1];
}
}

16
classes/media/preprocessors/media-preprocessor.ts Normal file
View file

@ -0,0 +1,16 @@
/**
* @packageDocumentation
* @module MediaManager/Preprocessors
*/
/**
* Represents a media preprocessor.
*/
export interface MediaPreprocessor {
/**
* Processes a file before it's stored.
* @param file - The file to process.
* @returns A promise that resolves to the processed file.
*/
process(file: File): Promise<{ file: File } & Record<string, unknown>>;
}

9
cli/commands/emoji/add.ts
View file

@ -2,12 +2,12 @@ import { Args } from "@oclif/core";
import chalk from "chalk";
import { and, eq, isNull } from "drizzle-orm";
import ora from "ora";
import { MediaManager } from "~/classes/media/media-manager";
import { BaseCommand } from "~/cli/base";
import { Emojis } from "~/drizzle/schema";
import { config } from "~/packages/config-manager";
import { Attachment } from "~/packages/database-interface/attachment";
import { Emoji } from "~/packages/database-interface/emoji";
import { MediaBackend } from "~/packages/media-manager";
export default class EmojiAdd extends BaseCommand<typeof EmojiAdd> {
static override args = {
@ -97,14 +97,11 @@ export default class EmojiAdd extends BaseCommand<typeof EmojiAdd> {
);
}
const media = await MediaBackend.fromBackendType(
config.media.backend,
config,
);
const mediaManager = new MediaManager(config);
const spinner = ora("Uploading emoji").start();
const uploaded = await media.addFile(file).catch((e: Error) => {
const uploaded = await mediaManager.addFile(file).catch((e: Error) => {
spinner.fail();
this.log(`${chalk.red("✗")} Error: ${chalk.red(e.message)}`);
return null;

9
cli/commands/emoji/delete.ts
View file

@ -3,12 +3,12 @@ import { Flags } from "@oclif/core";
import chalk from "chalk";
import { eq } from "drizzle-orm";
import ora from "ora";
import { MediaManager } from "~/classes/media/media-manager";
import { EmojiFinderCommand } from "~/cli/classes";
import { formatArray } from "~/cli/utils/format";
import { db } from "~/drizzle/db";
import { Emojis } from "~/drizzle/schema";
import { config } from "~/packages/config-manager";
import { MediaBackend } from "~/packages/media-manager";
export default class EmojiDelete extends EmojiFinderCommand<
typeof EmojiDelete
@ -84,12 +84,9 @@ export default class EmojiDelete extends EmojiFinderCommand<
emojis.findIndex((e) => e.id === emoji.id) + 1
}/${emojis.length})`;
const mediaBackend = await MediaBackend.fromBackendType(
config.media.backend,
config,
);
const mediaManager = new MediaManager(config);
await mediaBackend.deleteFileByUrl(emoji.url);
await mediaManager.deleteFileByUrl(emoji.url);
await db.delete(Emojis).where(eq(Emojis.id, emoji.id));
}

25
cli/commands/emoji/import.ts
View file

@ -4,12 +4,12 @@ import { and, inArray, isNull } from "drizzle-orm";
import { lookup } from "mime-types";
import ora from "ora";
import { unzip } from "unzipit";
import { MediaManager } from "~/classes/media/media-manager";
import { BaseCommand } from "~/cli/base";
import { Emojis } from "~/drizzle/schema";
import { config } from "~/packages/config-manager";
import { Attachment } from "~/packages/database-interface/attachment";
import { Emoji } from "~/packages/database-interface/emoji";
import { MediaBackend } from "~/packages/media-manager";
type MetaType = {
emojis: {
@ -169,10 +169,7 @@ export default class EmojiImport extends BaseCommand<typeof EmojiImport> {
const importSpinner = ora("Importing emojis").start();
const media = await MediaBackend.fromBackendType(
config.media.backend,
config,
);
const mediaManager = new MediaManager(config);
const successfullyImported: MetaType["emojis"] = [];
@ -200,14 +197,16 @@ export default class EmojiImport extends BaseCommand<typeof EmojiImport> {
type: contentType,
});
const uploaded = await media.addFile(newFile).catch((e: Error) => {
this.log(
`${chalk.red("✗")} Error uploading ${chalk.red(
emoji.emoji.name,
)}: ${chalk.red(e.message)}`,
);
return null;
});
const uploaded = await mediaManager
.addFile(newFile)
.catch((e: Error) => {
this.log(
`${chalk.red("✗")} Error uploading ${chalk.red(
emoji.emoji.name,
)}: ${chalk.red(e.message)}`,
);
return null;
});
if (!uploaded) {
continue;

1
package.json
View file

@ -128,7 +128,6 @@
"markdown-it-anchor": "^9.0.1",
"markdown-it-container": "^4.0.0",
"markdown-it-toc-done-right": "^4.2.0",
"media-manager": "workspace:*",
"meilisearch": "^0.40.0",
"mime-types": "^2.1.35",
"oauth4webapi": "^2.11.1",

2
packages/database-interface/attachment.ts
View file

@ -1,6 +1,7 @@
import { proxyUrl } from "@/response";
import type { ContentFormat } from "@lysand-org/federation/types";
import { config } from "config-manager";
import { MediaBackendType } from "config-manager/config.type";
import {
type InferInsertModel,
type InferSelectModel,
@ -9,7 +10,6 @@ import {
eq,
inArray,
} from "drizzle-orm";
import { MediaBackendType } from "media-manager";
import { db } from "~/drizzle/db";
import { Attachments } from "~/drizzle/schema";
import type { AsyncAttachment as APIAsyncAttachment } from "~/types/mastodon/async_attachment";

BIN
packages/media-manager/bun.lockb
View file

Binary file not shown.

2
packages/media-manager/bunfig.toml
View file

@ -1,2 +0,0 @@
[install.scopes]
"@jsr" = "https://npm.jsr.io"

272
packages/media-manager/index.ts
View file

@ -1,272 +0,0 @@
import { rm } from "node:fs/promises";
import { S3Client } from "@jsr/bradenmacdonald__s3-lite-client";
import type { Config } from "config-manager";
import { MediaConverter } from "./media-converter";
export enum MediaBackendType {
Local = "local",
S3 = "s3",
}
interface UploadedFileMetadata {
uploadedFile: File;
path: string;
hash: string;
}
export class MediaHasher {
/**
* Returns the SHA-256 hash of a file in hex format
* @param media The file to hash
* @returns The SHA-256 hash of the file in hex format
*/
public async getMediaHash(media: File) {
const hash = new Bun.SHA256()
.update(await media.arrayBuffer())
.digest("hex");
return hash;
}
}
export class MediaBackend {
constructor(
public config: Config,
public backend: MediaBackendType,
) {}
public static fromBackendType(
backend: MediaBackendType,
config: Config,
): MediaBackend {
switch (backend) {
case MediaBackendType.Local:
return new LocalMediaBackend(config);
case MediaBackendType.S3:
return new S3MediaBackend(config);
default:
throw new Error(`Unknown backend type: ${backend as string}`);
}
}
public getBackendType() {
return this.backend;
}
public shouldConvertImages(config: Config) {
return config.media.conversion.convert_images;
}
/**
* Fetches file from backend from SHA-256 hash
* @param file SHA-256 hash of wanted file
* @param databaseHashFetcher Function that takes in a sha256 hash as input and outputs the filename of that file in the database
* @returns The file as a File object
*/
public getFileByHash(
_file: string,
_databaseHashFetcher: (sha256: string) => Promise<string>,
): Promise<File | null> {
return Promise.reject(
new Error("Do not call MediaBackend directly: use a subclass"),
);
}
public deleteFileByUrl(_url: string): Promise<void> {
return Promise.reject(
new Error("Do not call MediaBackend directly: use a subclass"),
);
}
/**
* Fetches file from backend from filename
* @param filename File name
* @returns The file as a File object
*/
public getFile(_filename: string): Promise<File | null> {
return Promise.reject(
new Error("Do not call MediaBackend directly: use a subclass"),
);
}
/**
* Adds file to backend
* @param file File to add
* @returns Metadata about the uploaded file
*/
public addFile(_file: File): Promise<UploadedFileMetadata> {
return Promise.reject(
new Error("Do not call MediaBackend directly: use a subclass"),
);
}
}
export class LocalMediaBackend extends MediaBackend {
constructor(config: Config) {
super(config, MediaBackendType.Local);
}
public async addFile(file: File) {
let convertedFile = file;
if (this.shouldConvertImages(this.config)) {
const mediaConverter = new MediaConverter();
convertedFile = await mediaConverter.convert(
file,
this.config.media.conversion.convert_to,
);
}
const hash = await new MediaHasher().getMediaHash(convertedFile);
const newFile = Bun.file(
`${this.config.media.local_uploads_folder}/${hash}/${convertedFile.name}`,
);
if (await newFile.exists()) {
// Already exists, we don't need to upload it again
return {
uploadedFile: convertedFile,
path: `${hash}/${convertedFile.name}`,
hash: hash,
};
}
await Bun.write(newFile, convertedFile);
return {
uploadedFile: convertedFile,
path: `${hash}/${convertedFile.name}`,
hash: hash,
};
}
public async deleteFileByUrl(url: string) {
// url is of format https://base-url/media/SHA256HASH/FILENAME
const urlO = new URL(url);
const hash = urlO.pathname.split("/")[1];
const dirPath = `${this.config.media.local_uploads_folder}/${hash}`;
try {
await rm(dirPath, { recursive: true });
} catch (e) {
console.error(`Failed to delete directory at ${dirPath}`);
console.error(e);
}
return;
}
public async getFileByHash(
hash: string,
databaseHashFetcher: (sha256: string) => Promise<string | null>,
): Promise<File | null> {
const filename = await databaseHashFetcher(hash);
if (!filename) {
return null;
}
return this.getFile(filename);
}
public async getFile(filename: string): Promise<File | null> {
const file = Bun.file(
`${this.config.media.local_uploads_folder}/${filename}`,
);
if (!(await file.exists())) {
return null;
}
return new File([await file.arrayBuffer()], filename, {
type: file.type,
lastModified: file.lastModified,
});
}
}
export class S3MediaBackend extends MediaBackend {
constructor(
config: Config,
private s3Client = new S3Client({
endPoint: config.s3.endpoint,
useSSL: true,
region: config.s3.region || "auto",
bucket: config.s3.bucket_name,
accessKey: config.s3.access_key,
secretKey: config.s3.secret_access_key,
}),
) {
super(config, MediaBackendType.S3);
}
public async deleteFileByUrl(url: string) {
// url is of format https://s3-base-url/SHA256HASH/FILENAME
const urlO = new URL(url);
const hash = urlO.pathname.split("/")[1];
const filename = urlO.pathname.split("/")[2];
await this.s3Client.deleteObject(`${hash}/${filename}`);
return;
}
public async addFile(file: File) {
let convertedFile = file;
if (this.shouldConvertImages(this.config)) {
const mediaConverter = new MediaConverter();
convertedFile = await mediaConverter.convert(
file,
this.config.media.conversion.convert_to,
);
}
const hash = await new MediaHasher().getMediaHash(convertedFile);
await this.s3Client.putObject(
`${hash}/${convertedFile.name}`,
convertedFile.stream(),
{
size: convertedFile.size,
},
);
return {
uploadedFile: convertedFile,
path: `${hash}/${convertedFile.name}`,
hash: hash,
};
}
public async getFileByHash(
hash: string,
databaseHashFetcher: (sha256: string) => Promise<string | null>,
): Promise<File | null> {
const filename = await databaseHashFetcher(hash);
if (!filename) {
return null;
}
return this.getFile(filename);
}
public async getFile(filename: string): Promise<File | null> {
try {
await this.s3Client.statObject(filename);
} catch {
return null;
}
const file = await this.s3Client.getObject(filename);
return new File([await file.arrayBuffer()], filename, {
type: file.headers.get("Content-Type") || "undefined",
});
}
}
export { MediaConverter };

117
packages/media-manager/media-converter.ts
View file

@ -1,117 +0,0 @@
/**
* @packageDocumentation
* @module MediaManager
* @description Handles media conversion between formats
*/
import { config } from "config-manager";
import sharp from "sharp";
export const supportedMediaFormats = [
"image/png",
"image/jpeg",
"image/webp",
"image/avif",
"image/svg+xml",
"image/gif",
"image/tiff",
];
export const supportedOutputFormats = [
"image/jpeg",
"image/png",
"image/webp",
"image/avif",
"image/gif",
"image/tiff",
];
/**
* Handles media conversion between formats
*/
export class MediaConverter {
/**
* Returns whether the media is convertable
* @returns Whether the media is convertable
*/
public isConvertable(file: File) {
if (
file.type === "image/svg+xml" &&
!config.media.conversion.convert_vector
) {
return false;
}
return supportedMediaFormats.includes(file.type);
}
/**
* Returns the file name with the extension replaced
* @param fileName File name to replace
* @returns File name with extension replaced
*/
private getReplacedFileName(fileName: string, newExtension: string) {
return this.extractFilenameFromPath(fileName).replace(
/\.[^/.]+$/,
`.${newExtension}`,
);
}
/**
* Extracts the filename from a path
* @param path Path to extract filename from
* @returns Extracted filename
*/
private extractFilenameFromPath(path: string) {
// Don't count escaped slashes as path separators
const pathParts = path.split(/(?<!\\)\//);
return pathParts[pathParts.length - 1];
}
/**
* Converts media to the specified format
* @param media Media to convert
* @returns Converted media
*/
public async convert(
media: File,
toMime: (typeof supportedMediaFormats)[number],
) {
if (!this.isConvertable(media)) {
return media;
}
if (!supportedOutputFormats.includes(toMime)) {
throw new Error(
`Unsupported image output format: ${toMime}. Supported formats: ${supportedOutputFormats.join(
", ",
)}`,
);
}
const sharpCommand = sharp(await media.arrayBuffer(), {
animated: true,
});
const commandName = toMime.split("/")[1] as
| "jpeg"
| "png"
| "webp"
| "avif"
| "gif"
| "tiff";
const convertedBuffer = await sharpCommand[commandName]().toBuffer();
// Convert the buffer to a BlobPart
const buffer = new Blob([convertedBuffer]);
return new File(
[buffer],
this.getReplacedFileName(media.name || "image", commandName),
{
type: toMime,
lastModified: Date.now(),
},
);
}
}

9
packages/media-manager/package.json
View file

@ -1,9 +0,0 @@
{
"name": "media-manager",
"version": "0.0.0",
"main": "index.ts",
"dependencies": {
"@jsr/bradenmacdonald__s3-lite-client": "npm:@jsr/bradenmacdonald__s3-lite-client",
"config-manager": "workspace:*"
}
}

317
packages/media-manager/tests/media-backends.test.ts
View file

@ -1,317 +0,0 @@
import { beforeEach, describe, expect, it, jest, mock, spyOn } from "bun:test";
import type { S3Client } from "@jsr/bradenmacdonald__s3-lite-client";
import type { Config } from "config-manager";
import {
LocalMediaBackend,
MediaBackend,
MediaBackendType,
MediaHasher,
S3MediaBackend,
} from "../index";
import { MediaConverter } from "../media-converter";
type DeepPartial<T> = {
[P in keyof T]?: DeepPartial<T[P]>;
};
describe("MediaBackend", () => {
let mediaBackend: MediaBackend;
let mockConfig: Config;
beforeEach(() => {
mockConfig = {
media: {
conversion: {
convert_images: true,
},
},
} as Config;
mediaBackend = new MediaBackend(mockConfig, MediaBackendType.S3);
});
it("should initialize with correct backend type", () => {
expect(mediaBackend.getBackendType()).toEqual(MediaBackendType.S3);
});
describe("fromBackendType", () => {
it("should return a LocalMediaBackend instance for LOCAL backend type", async () => {
const backend = await MediaBackend.fromBackendType(
MediaBackendType.Local,
mockConfig,
);
expect(backend).toBeInstanceOf(LocalMediaBackend);
});
it("should return a S3MediaBackend instance for S3 backend type", async () => {
const backend = await MediaBackend.fromBackendType(
MediaBackendType.S3,
{
s3: {
endpoint: "localhost:4566",
region: "us-east-1",
bucket_name: "test-bucket",
access_key: "test-access",
public_url: "test",
secret_access_key: "test-secret",
},
} as Config,
);
expect(backend).toBeInstanceOf(S3MediaBackend);
});
it("should throw an error for unknown backend type", () => {
expect(
// @ts-expect-error This is a test
() => MediaBackend.fromBackendType("unknown", mockConfig),
).toThrow("Unknown backend type: unknown");
});
});
it("should check if images should be converted", () => {
expect(mediaBackend.shouldConvertImages(mockConfig)).toBe(true);
mockConfig.media.conversion.convert_images = false;
expect(mediaBackend.shouldConvertImages(mockConfig)).toBe(false);
});
it("should throw error when calling getFileByHash", () => {
const mockHash = "test-hash";
const databaseHashFetcher = jest.fn().mockResolvedValue("test.jpg");
expect(
mediaBackend.getFileByHash(mockHash, databaseHashFetcher),
).rejects.toThrow(Error);
});
it("should throw error when calling getFile", () => {
const mockFilename = "test.jpg";
expect(mediaBackend.getFile(mockFilename)).rejects.toThrow(Error);
});
it("should throw error when calling addFile", () => {
const mockFile = new File([""], "test.jpg");
expect(mediaBackend.addFile(mockFile)).rejects.toThrow();
});
});
describe("S3MediaBackend", () => {
let s3MediaBackend: S3MediaBackend;
let mockS3Client: Partial<S3Client>;
let mockConfig: DeepPartial<Config>;
let mockFile: File;
let mockMediaHasher: MediaHasher;
beforeEach(() => {
mockConfig = {
s3: {
endpoint: "http://localhost:4566",
region: "us-east-1",
bucket_name: "test-bucket",
access_key: "test-access-key",
secret_access_key: "test-secret-access-key",
public_url: "test",
},
media: {
conversion: {
convert_to: "image/png",
},
},
};
mockFile = new File([new TextEncoder().encode("test")], "test.jpg");
mockMediaHasher = new MediaHasher();
mockS3Client = {
putObject: jest.fn().mockResolvedValue({}),
statObject: jest.fn().mockResolvedValue({}),
getObject: jest.fn().mockResolvedValue({
blob: jest.fn().mockResolvedValue(new Blob()),
headers: new Headers({ "Content-Type": "image/jpeg" }),
}),
deleteObject: jest.fn().mockResolvedValue({}),
} as Partial<S3Client>;
s3MediaBackend = new S3MediaBackend(
mockConfig as Config,
mockS3Client as S3Client,
);
});
it("should initialize with correct type", () => {
expect(s3MediaBackend.getBackendType()).toEqual(MediaBackendType.S3);
});
it("should add file", async () => {
const mockHash = "test-hash";
spyOn(mockMediaHasher, "getMediaHash").mockResolvedValue(mockHash);
const result = await s3MediaBackend.addFile(mockFile);
expect(result.uploadedFile).toEqual(mockFile);
expect(result.hash).toHaveLength(64);
expect(mockS3Client.putObject).toHaveBeenCalledWith(
expect.stringContaining(mockFile.name),
expect.any(ReadableStream),
{ size: mockFile.size },
);
});
it("should get file by hash", async () => {
const mockHash = "test-hash";
const mockFilename = "test.jpg";
const databaseHashFetcher = jest.fn().mockResolvedValue(mockFilename);
mockS3Client.statObject = jest.fn().mockResolvedValue({});
mockS3Client.getObject = jest.fn().mockResolvedValue({
arrayBuffer: jest.fn().mockResolvedValue(new ArrayBuffer(10)),
headers: new Headers({ "Content-Type": "image/jpeg" }),
});
const file = await s3MediaBackend.getFileByHash(
mockHash,
databaseHashFetcher,
);
expect(file).not.toBeNull();
expect(file?.name).toEqual(mockFilename);
expect(file?.type).toEqual("image/jpeg");
});
it("should get file", async () => {
const mockFilename = "test.jpg";
mockS3Client.statObject = jest.fn().mockResolvedValue({});
mockS3Client.getObject = jest.fn().mockResolvedValue({
arrayBuffer: jest.fn().mockResolvedValue(new ArrayBuffer(10)),
headers: new Headers({ "Content-Type": "image/jpeg" }),
});
const file = await s3MediaBackend.getFile(mockFilename);
expect(file).not.toBeNull();
expect(file?.name).toEqual(mockFilename);
expect(file?.type).toEqual("image/jpeg");
});
it("should delete file", async () => {
// deleteFileByUrl
// Upload file first
const mockHash = "test-hash";
spyOn(mockMediaHasher, "getMediaHash").mockResolvedValue(mockHash);
const result = await s3MediaBackend.addFile(mockFile);
const url = result.path;
await s3MediaBackend.deleteFileByUrl(`http://localhost:4566/${url}`);
expect(mockS3Client.deleteObject).toHaveBeenCalledWith(
expect.stringContaining(url),
);
});
});
describe("LocalMediaBackend", () => {
let localMediaBackend: LocalMediaBackend;
let mockConfig: Config;
let mockFile: File;
let mockMediaHasher: MediaHasher;
beforeEach(() => {
mockConfig = {
media: {
conversion: {
convert_images: true,
convert_to: "image/png",
},
local_uploads_folder: "./uploads",
},
} as Config;
mockFile = Bun.file(`${__dirname}/megamind.jpg`) as unknown as File;
mockMediaHasher = new MediaHasher();
localMediaBackend = new LocalMediaBackend(mockConfig);
});
it("should initialize with correct type", () => {
expect(localMediaBackend.getBackendType()).toEqual(
MediaBackendType.Local,
);
});
it("should add file", async () => {
const mockHash = "test-hash";
spyOn(mockMediaHasher, "getMediaHash").mockResolvedValue(mockHash);
const mockMediaConverter = new MediaConverter();
spyOn(mockMediaConverter, "convert").mockResolvedValue(mockFile);
// @ts-expect-error This is a mock
spyOn(Bun, "file").mockImplementationOnce(() => ({
exists: () => Promise.resolve(false),
}));
spyOn(Bun, "write").mockImplementationOnce(() =>
Promise.resolve(mockFile.size),
);
const result = await localMediaBackend.addFile(mockFile);
expect(result.uploadedFile).toEqual(mockFile);
expect(result.path).toEqual(expect.stringContaining("megamind.png"));
expect(result.hash).toHaveLength(64);
});
it("should get file by hash", async () => {
const mockHash = "test-hash";
const mockFilename = "test.jpg";
const databaseHashFetcher = jest.fn().mockResolvedValue(mockFilename);
// @ts-expect-error This is a mock
spyOn(Bun, "file").mockImplementationOnce(() => ({
exists: () => Promise.resolve(true),
arrayBuffer: () => Promise.resolve(new ArrayBuffer(8)),
type: "image/jpeg",
lastModified: 123456789,
}));
const file = await localMediaBackend.getFileByHash(
mockHash,
databaseHashFetcher,
);
expect(file).not.toBeNull();
expect(file?.name).toEqual(mockFilename);
expect(file?.type).toEqual("image/jpeg");
});
it("should get file", async () => {
const mockFilename = "test.jpg";
// @ts-expect-error This is a mock
spyOn(Bun, "file").mockImplementationOnce(() => ({
exists: () => Promise.resolve(true),
arrayBuffer: () => Promise.resolve(new ArrayBuffer(8)),
type: "image/jpeg",
lastModified: 123456789,
}));
const file = await localMediaBackend.getFile(mockFilename);
expect(file).not.toBeNull();
expect(file?.name).toEqual(mockFilename);
expect(file?.type).toEqual("image/jpeg");
});
it("should delete file", async () => {
// deleteByUrl
const mockHash = "test-hash";
spyOn(mockMediaHasher, "getMediaHash").mockResolvedValue(mockHash);
await localMediaBackend.addFile(mockFile);
const rmMock = jest.fn().mockResolvedValue(Promise.resolve());
// Spy on fs/promises rm
mock.module("fs/promises", () => {
return {
rm: rmMock,
};
});
await localMediaBackend.deleteFileByUrl(
"http://localhost:4566/test-hash",
);
expect(rmMock).toHaveBeenCalledWith(
`${mockConfig.media.local_uploads_folder}/${mockHash}`,
{ recursive: true },
);
});
});

53
packages/media-manager/tests/media-manager.test.ts
View file

@ -1,53 +0,0 @@
// FILEPATH: /home/jessew/Dev/lysand/packages/media-manager/media-converter.test.ts
import { beforeEach, describe, expect, it } from "bun:test";
import { MediaConverter } from "../media-converter";
describe("MediaConverter", () => {
let mediaConverter: MediaConverter;
beforeEach(() => {
mediaConverter = new MediaConverter();
});
it("should replace file name extension", () => {
const fileName = "test.jpg";
const expectedFileName = "test.png";
// Written like this because it's a private function
// @ts-ignore
expect(mediaConverter.getReplacedFileName(fileName, "png")).toEqual(
expectedFileName,
);
});
describe("Filename extractor", () => {
it("should extract filename from path", () => {
const path = "path/to/test.jpg";
const expectedFileName = "test.jpg";
// @ts-ignore
expect(mediaConverter.extractFilenameFromPath(path)).toEqual(
expectedFileName,
);
});
it("should handle escaped slashes", () => {
const path = "path/to/test\\/test.jpg";
const expectedFileName = "test\\/test.jpg";
// @ts-ignore
expect(mediaConverter.extractFilenameFromPath(path)).toEqual(
expectedFileName,
);
});
});
it("should convert media", async () => {
const file = Bun.file(`${__dirname}/megamind.jpg`);
const convertedFile = await mediaConverter.convert(
file as unknown as File,
"image/png",
);
expect(convertedFile.name).toEqual("megamind.png");
expect(convertedFile.type).toEqual("image/png");
});
});

BIN
packages/media-manager/tests/megamind.jpg
View file

Binary file not shown.
Side by side

Before

Width:  |  Height:  |  Size: 5.1 KiB

7
server/api/api/v1/accounts/update_credentials/index.ts
View file

@ -6,10 +6,10 @@ import { config } from "config-manager";
import { and, eq, isNull } from "drizzle-orm";
import type { Hono } from "hono";
import ISO6391 from "iso-639-1";
import { MediaBackend } from "media-manager";
import { z } from "zod";
import { parseEmojis } from "~/classes/functions/emoji";
import { contentToHtml } from "~/classes/functions/status";
import { MediaManager } from "~/classes/media/media-manager";
import { db } from "~/drizzle/db";
import { EmojiToUser, RolePermissions, Users } from "~/drizzle/schema";
import { Attachment } from "~/packages/database-interface/attachment";
@ -160,10 +160,7 @@ export default (app: Hono) =>
display_name ?? "",
);
const mediaManager = await MediaBackend.fromBackendType(
config.media.backend,
config,
);
const mediaManager = new MediaManager(config);
if (display_name) {
self.displayName = sanitizedDisplayName;

18
server/api/api/v1/emojis/:id/index.ts
View file

@ -11,12 +11,12 @@ import { zValidator } from "@hono/zod-validator";
import { eq } from "drizzle-orm";
import type { Hono } from "hono";
import { z } from "zod";
import { MediaManager } from "~/classes/media/media-manager";
import { db } from "~/drizzle/db";
import { Emojis, RolePermissions } from "~/drizzle/schema";
import { config } from "~/packages/config-manager";
import { Attachment } from "~/packages/database-interface/attachment";
import { Emoji } from "~/packages/database-interface/emoji";
import { MediaBackend } from "~/packages/media-manager";
export const meta = applyConfig({
allowedMethods: ["DELETE", "GET", "PATCH"],
@ -102,14 +102,11 @@ export default (app: Hono) =>
);
}
const mediaManager = new MediaManager(config);
switch (context.req.method) {
case "DELETE": {
const mediaBackend = await MediaBackend.fromBackendType(
config.media.backend,
config,
);
await mediaBackend.deleteFileByUrl(emoji.data.url);
await mediaManager.deleteFileByUrl(emoji.data.url);
await db.delete(Emojis).where(eq(Emojis.id, id));
@ -172,13 +169,10 @@ export default (app: Hono) =>
let url = "";
if (form.element instanceof File) {
const media = await MediaBackend.fromBackendType(
config.media.backend,
config,
const uploaded = await mediaManager.addFile(
form.element,
);
const uploaded = await media.addFile(form.element);
url = uploaded.path;
contentType = uploaded.uploadedFile.type;
} else {

28
server/api/api/v1/emojis/index.test.ts
View file

@ -1,6 +1,7 @@
import { afterAll, beforeAll, describe, expect, test } from "bun:test";
import { config } from "config-manager";
import { inArray } from "drizzle-orm";
import sharp from "sharp";
import { db } from "~/drizzle/db";
import { Emojis } from "~/drizzle/schema";
import { getTestUsers, sendTestRequest } from "~/tests/utils";
@ -21,6 +22,23 @@ afterAll(async () => {
.where(inArray(Emojis.shortcode, ["test1", "test2", "test3", "test4"]));
});
const createImage = async (name: string): Promise<File> => {
const inputBuffer = await sharp({
create: {
width: 100,
height: 100,
channels: 3,
background: { r: 255, g: 0, b: 0 },
},
})
.png()
.toBuffer();
return new File([inputBuffer], name, {
type: "image/png",
});
};
describe(meta.route, () => {
test("should return 401 if not authenticated", async () => {
const response = await sendTestRequest(
@ -43,7 +61,7 @@ describe(meta.route, () => {
test("should upload a file and create an emoji", async () => {
const formData = new FormData();
formData.append("shortcode", "test1");
formData.append("element", Bun.file("tests/test-image.webp"));
formData.append("element", await createImage("test.png"));
formData.append("global", "true");
const response = await sendTestRequest(
@ -104,7 +122,7 @@ describe(meta.route, () => {
test("should fail when uploading an already existing emoji", async () => {
const formData = new FormData();
formData.append("shortcode", "test1");
formData.append("element", Bun.file("tests/test-image.webp"));
formData.append("element", await createImage("test-image.png"));
const response = await sendTestRequest(
new Request(new URL(meta.route, config.http.base_url), {
@ -124,7 +142,7 @@ describe(meta.route, () => {
test("should upload a file and create an emoji", async () => {
const formData = new FormData();
formData.append("shortcode", "test4");
formData.append("element", Bun.file("tests/test-image.webp"));
formData.append("element", await createImage("test-image.png"));
const response = await sendTestRequest(
new Request(new URL(meta.route, config.http.base_url), {
@ -145,7 +163,7 @@ describe(meta.route, () => {
test("should fail when uploading an already existing global emoji", async () => {
const formData = new FormData();
formData.append("shortcode", "test1");
formData.append("element", Bun.file("tests/test-image.webp"));
formData.append("element", await createImage("test-image.png"));
const response = await sendTestRequest(
new Request(new URL(meta.route, config.http.base_url), {
@ -163,7 +181,7 @@ describe(meta.route, () => {
test("should create an emoji as another user with the same shortcode", async () => {
const formData = new FormData();
formData.append("shortcode", "test4");
formData.append("element", Bun.file("tests/test-image.webp"));
formData.append("element", await createImage("test-image.png"));
const response = await sendTestRequest(
new Request(new URL(meta.route, config.http.base_url), {

9
server/api/api/v1/emojis/index.ts
View file

@ -11,11 +11,11 @@ import { zValidator } from "@hono/zod-validator";
import { and, eq, isNull, or } from "drizzle-orm";
import type { Hono } from "hono";
import { z } from "zod";
import { MediaManager } from "~/classes/media/media-manager";
import { Emojis, RolePermissions } from "~/drizzle/schema";
import { config } from "~/packages/config-manager";
import { Attachment } from "~/packages/database-interface/attachment";
import { Emoji } from "~/packages/database-interface/emoji";
import { MediaBackend } from "~/packages/media-manager";
export const meta = applyConfig({
allowedMethods: ["POST"],
@ -115,12 +115,9 @@ export default (app: Hono) =>
}
if (element instanceof File) {
const media = await MediaBackend.fromBackendType(
config.media.backend,
config,
);
const mediaManager = new MediaManager(config);
const uploaded = await media.addFile(element);
const uploaded = await mediaManager.addFile(element);
url = uploaded.path;
contentType = uploaded.uploadedFile.type;

7
server/api/api/v1/media/:id/index.ts
View file

@ -3,8 +3,8 @@ import { errorResponse, jsonResponse, response } from "@/response";
import { zValidator } from "@hono/zod-validator";
import { config } from "config-manager";
import type { Hono } from "hono";
import { MediaBackend } from "media-manager";
import { z } from "zod";
import { MediaManager } from "~/classes/media/media-manager";
import { RolePermissions } from "~/drizzle/schema";
import { Attachment } from "~/packages/database-interface/attachment";
@ -71,10 +71,7 @@ export default (app: Hono) =>
let thumbnailUrl = attachment.data.thumbnailUrl;
const mediaManager = await MediaBackend.fromBackendType(
config.media.backend,
config,
);
const mediaManager = new MediaManager(config);
if (thumbnail) {
const { path } = await mediaManager.addFile(thumbnail);

41
server/api/api/v1/media/index.ts
View file

@ -1,12 +1,11 @@
import { applyConfig, auth, handleZodError } from "@/api";
import { errorResponse, jsonResponse } from "@/response";
import { zValidator } from "@hono/zod-validator";
import { encode } from "blurhash";
import { config } from "config-manager";
import type { Hono } from "hono";
import { MediaBackend } from "media-manager";
import sharp from "sharp";
import { z } from "zod";
import { MediaManager } from "~/classes/media/media-manager";
import { RolePermissions } from "~/drizzle/schema";
import { Attachment } from "~/packages/database-interface/attachment";
@ -69,43 +68,11 @@ export default (app: Hono) =>
? await sharp(await file.arrayBuffer()).metadata()
: null;
const blurhash = await new Promise<string | null>((resolve) => {
(async () =>
sharp(await file.arrayBuffer())
.raw()
.ensureAlpha()
.toBuffer((err, buffer) => {
if (err) {
resolve(null);
return;
}
const mediaManager = new MediaManager(config);
try {
resolve(
encode(
new Uint8ClampedArray(buffer),
metadata?.width ?? 0,
metadata?.height ?? 0,
4,
4,
) as string,
);
} catch {
resolve(null);
}
}))();
});
const { path, blurhash } = await mediaManager.addFile(file);
let url = "";
const mediaManager = await MediaBackend.fromBackendType(
config.media.backend,
config,
);
const { path } = await mediaManager.addFile(file);
url = Attachment.getUrl(path);
const url = Attachment.getUrl(path);
let thumbnailUrl = "";

41
server/api/api/v2/media/index.ts
View file

@ -1,12 +1,11 @@
import { applyConfig, auth, handleZodError } from "@/api";
import { errorResponse, jsonResponse } from "@/response";
import { zValidator } from "@hono/zod-validator";
import { encode } from "blurhash";
import { config } from "config-manager";
import type { Hono } from "hono";
import { MediaBackend } from "media-manager";
import sharp from "sharp";
import { z } from "zod";
import { MediaManager } from "~/classes/media/media-manager";
import { RolePermissions } from "~/drizzle/schema";
import { Attachment } from "~/packages/database-interface/attachment";
@ -69,43 +68,11 @@ export default (app: Hono) =>
? await sharp(await file.arrayBuffer()).metadata()
: null;
const blurhash = await new Promise<string | null>((resolve) => {
(async () =>
sharp(await file.arrayBuffer())
.raw()
.ensureAlpha()
.toBuffer((err, buffer) => {
if (err) {
resolve(null);
return;
}
const mediaManager = new MediaManager(config);
try {
resolve(
encode(
new Uint8ClampedArray(buffer),
metadata?.width ?? 0,
metadata?.height ?? 0,
4,
4,
) as string,
);
} catch {
resolve(null);
}
}))();
});
const { path, blurhash } = await mediaManager.addFile(file);
let url = "";
const mediaManager = await MediaBackend.fromBackendType(
config.media.backend,
config,
);
const { path } = await mediaManager.addFile(file);
url = Attachment.getUrl(path);
const url = Attachment.getUrl(path);
let thumbnailUrl = "";