Add more cases for media backend

This commit is contained in:
Jesse Wierzbinski 2023-10-17 14:57:47 -10:00
parent 47a53b6990
commit 16cfd5d900
No known key found for this signature in database
GPG key ID: F9A1E418934E40B0
6 changed files with 1042 additions and 711 deletions

1
.gitignore vendored
View file

@ -168,3 +168,4 @@ dist
.yarn/install-state.gz
.pnp.\*
config/config.toml
uploads/

BIN
bun.lockb

Binary file not shown.

177
classes/media.ts Normal file
View file

@ -0,0 +1,177 @@
import {
GetObjectCommand,
GetObjectCommandOutput,
PutObjectCommand,
S3Client,
} from "@aws-sdk/client-s3";
import { ConfigType } from "@config";
class MediaBackend {
backend: string;
constructor(backend: string) {
this.backend = backend;
}
/**
* Adds media to the media backend
* @param media
* @returns The hash of the file in SHA-256 (hex format)
*/
async addMedia(media: File) {
const hash = new Bun.SHA256()
.update(await media.arrayBuffer())
.digest("hex");
return hash;
}
/**
* Retrieves element from media backend by hash
* @param hash The hash of the element in SHA-256 hex format
* @returns The file as a File object
*/
// eslint-disable-next-line @typescript-eslint/require-await, @typescript-eslint/no-unused-vars
async getMediaByHash(
// eslint-disable-next-line @typescript-eslint/no-unused-vars
hash: string,
// eslint-disable-next-line @typescript-eslint/no-unused-vars
extension: string
): Promise<File | null> {
return new File([], "test");
}
}
/**
* S3 Backend, stores files in S3
*/
export class S3Backend extends MediaBackend {
endpoint: string;
bucket: string;
region: string;
accessKey: string;
secretKey: string;
publicUrl: string;
client: S3Client;
constructor(config: ConfigType) {
super("s3");
this.endpoint = config.s3.endpoint;
this.bucket = config.s3.bucket_name;
this.region = config.s3.region;
this.accessKey = config.s3.access_key;
this.secretKey = config.s3.secret_access_key;
this.publicUrl = config.s3.public_url;
this.client = new S3Client({
endpoint: this.endpoint,
region: this.region || "auto",
credentials: {
accessKeyId: this.accessKey,
secretAccessKey: this.secretKey,
},
});
}
async addMedia(media: File): Promise<string> {
const hash = await super.addMedia(media);
if (!hash) {
throw new Error("Failed to hash file");
}
// Check if file is already present
const existingFile = await this.getMediaByHash(
hash,
media.name.split(".").pop() || ""
);
if (existingFile) {
// File already exists, so return the hash without uploading it
return hash;
}
const command = new PutObjectCommand({
Bucket: this.bucket,
Key: hash,
Body: Buffer.from(await media.arrayBuffer()),
ContentType: media.type,
ContentLength: media.size,
Metadata: {
"x-amz-meta-original-name": media.name,
},
});
const response = await this.client.send(command);
if (response.$metadata.httpStatusCode !== 200) {
throw new Error("Failed to upload file");
}
return hash;
}
async getMediaByHash(
hash: string,
extension: string
): Promise<File | null> {
const command = new GetObjectCommand({
Bucket: this.bucket,
Key: hash,
});
let response: GetObjectCommandOutput;
try {
response = await this.client.send(command);
} catch {
return null;
}
if (response.$metadata.httpStatusCode !== 200) {
throw new Error("Failed to get file");
}
const body = await response.Body?.transformToByteArray();
if (!body) {
throw new Error("Failed to get file");
}
return new File([body], `${hash}.${extension}`, {
type: response.ContentType,
});
}
}
/**
* Local backend, stores files on filesystem
*/
export class LocalBackend extends MediaBackend {
constructor() {
super("local");
}
async addMedia(media: File): Promise<string> {
const hash = await super.addMedia(media);
await Bun.write(Bun.file(`${process.cwd()}/uploads/${hash}`), media);
return hash;
}
async getMediaByHash(
hash: string,
extension: string
): Promise<File | null> {
const file = Bun.file(`${process.cwd()}/uploads/${hash}`);
if (!(await file.exists())) {
return null;
}
return new File([await file.arrayBuffer()], `${hash}.${extension}`, {
type: file.type,
});
}
}

View file

@ -56,6 +56,7 @@
"typescript": "^5.0.0"
},
"dependencies": {
"@aws-sdk/client-s3": "^3.429.0",
"ip-matching": "^2.1.2",
"isomorphic-dompurify": "^1.9.0",
"jsonld": "^8.3.1",

View file

@ -21,7 +21,8 @@ let user: User;
let user2: User;
let status: APIStatus | null = null;
beforeAll(async () => {
describe("API Tests", () => {
beforeAll(async () => {
if (!AppDataSource.isInitialized) await AppDataSource.initialize();
// Initialize test user
@ -63,9 +64,9 @@ beforeAll(async () => {
token.user = user;
token = await token.save();
});
});
describe("POST /api/v1/accounts/:id", () => {
describe("POST /api/v1/accounts/:id", () => {
test("should return a 404 error when trying to fetch a non-existent user", async () => {
const response = await fetch(
`${config.http.base_url}/api/v1/accounts/999999`,
@ -79,11 +80,13 @@ describe("POST /api/v1/accounts/:id", () => {
);
expect(response.status).toBe(404);
expect(response.headers.get("content-type")).toBe("application/json");
expect(response.headers.get("content-type")).toBe(
"application/json"
);
});
});
});
describe("POST /api/v1/statuses", () => {
describe("POST /api/v1/statuses", () => {
test("should create a new status and return an APIStatus object", async () => {
const response = await fetch(
`${config.http.base_url}/api/v1/statuses`,
@ -101,7 +104,9 @@ describe("POST /api/v1/statuses", () => {
);
expect(response.status).toBe(200);
expect(response.headers.get("content-type")).toBe("application/json");
expect(response.headers.get("content-type")).toBe(
"application/json"
);
// eslint-disable-next-line @typescript-eslint/no-unnecessary-type-assertion
status = (await response.json()) as APIStatus;
@ -126,9 +131,9 @@ describe("POST /api/v1/statuses", () => {
expect(status.in_reply_to_id).toBeNull();
expect(status.in_reply_to_account_id).toBeNull();
});
});
});
describe("GET /api/v1/timelines/public", () => {
describe("GET /api/v1/timelines/public", () => {
test("should return an array of APIStatus objects that includes the created status", async () => {
const response = await fetch(
`${config.http.base_url}/api/v1/timelines/public`,
@ -141,15 +146,17 @@ describe("GET /api/v1/timelines/public", () => {
);
expect(response.status).toBe(200);
expect(response.headers.get("content-type")).toBe("application/json");
expect(response.headers.get("content-type")).toBe(
"application/json"
);
const statuses: APIStatus[] = await response.json();
expect(statuses.some(s => s.id === status?.id)).toBe(true);
});
});
});
describe("PATCH /api/v1/accounts/update_credentials", () => {
describe("PATCH /api/v1/accounts/update_credentials", () => {
test("should update the authenticated user's display name", async () => {
const response = await fetch(
`${config.http.base_url}/api/v1/accounts/update_credentials`,
@ -166,15 +173,17 @@ describe("PATCH /api/v1/accounts/update_credentials", () => {
);
expect(response.status).toBe(200);
expect(response.headers.get("content-type")).toBe("application/json");
expect(response.headers.get("content-type")).toBe(
"application/json"
);
const user: APIAccount = await response.json();
expect(user.display_name).toBe("New Display Name");
});
});
});
describe("GET /api/v1/accounts/verify_credentials", () => {
describe("GET /api/v1/accounts/verify_credentials", () => {
test("should return the authenticated user's account information", async () => {
const response = await fetch(
`${config.http.base_url}/api/v1/accounts/verify_credentials`,
@ -188,7 +197,9 @@ describe("GET /api/v1/accounts/verify_credentials", () => {
);
expect(response.status).toBe(200);
expect(response.headers.get("content-type")).toBe("application/json");
expect(response.headers.get("content-type")).toBe(
"application/json"
);
const account: APIAccount = await response.json();
@ -215,9 +226,9 @@ describe("GET /api/v1/accounts/verify_credentials", () => {
expect(account.source?.note).toBe("");
expect(account.source?.sensitive).toBe(false);
});
});
});
describe("GET /api/v1/accounts/:id/statuses", () => {
describe("GET /api/v1/accounts/:id/statuses", () => {
test("should return the statuses of the specified user", async () => {
const response = await fetch(
`${config.http.base_url}/api/v1/accounts/${user.id}/statuses`,
@ -231,7 +242,9 @@ describe("GET /api/v1/accounts/:id/statuses", () => {
);
expect(response.status).toBe(200);
expect(response.headers.get("content-type")).toBe("application/json");
expect(response.headers.get("content-type")).toBe(
"application/json"
);
const statuses: APIStatus[] = await response.json();
@ -244,9 +257,9 @@ describe("GET /api/v1/accounts/:id/statuses", () => {
expect(status1.visibility).toBe("public");
expect(status1.account.id).toBe(user.id);
});
});
});
describe("POST /api/v1/accounts/:id/follow", () => {
describe("POST /api/v1/accounts/:id/follow", () => {
test("should follow the specified user and return an APIRelationship object", async () => {
const response = await fetch(
`${config.http.base_url}/api/v1/accounts/${user2.id}/follow`,
@ -261,16 +274,18 @@ describe("POST /api/v1/accounts/:id/follow", () => {
);
expect(response.status).toBe(200);
expect(response.headers.get("content-type")).toBe("application/json");
expect(response.headers.get("content-type")).toBe(
"application/json"
);
const account: APIRelationship = await response.json();
expect(account.id).toBe(user2.id);
expect(account.following).toBe(true);
});
});
});
describe("POST /api/v1/accounts/:id/unfollow", () => {
describe("POST /api/v1/accounts/:id/unfollow", () => {
test("should unfollow the specified user and return an APIRelationship object", async () => {
const response = await fetch(
`${config.http.base_url}/api/v1/accounts/${user2.id}/unfollow`,
@ -285,16 +300,18 @@ describe("POST /api/v1/accounts/:id/unfollow", () => {
);
expect(response.status).toBe(200);
expect(response.headers.get("content-type")).toBe("application/json");
expect(response.headers.get("content-type")).toBe(
"application/json"
);
const account: APIRelationship = await response.json();
expect(account.id).toBe(user2.id);
expect(account.following).toBe(false);
});
});
});
describe("POST /api/v1/accounts/:id/remove_from_followers", () => {
describe("POST /api/v1/accounts/:id/remove_from_followers", () => {
test("should remove the specified user from the authenticated user's followers and return an APIRelationship object", async () => {
const response = await fetch(
`${config.http.base_url}/api/v1/accounts/${user2.id}/remove_from_followers`,
@ -309,16 +326,18 @@ describe("POST /api/v1/accounts/:id/remove_from_followers", () => {
);
expect(response.status).toBe(200);
expect(response.headers.get("content-type")).toBe("application/json");
expect(response.headers.get("content-type")).toBe(
"application/json"
);
const account: APIRelationship = await response.json();
expect(account.id).toBe(user2.id);
expect(account.followed_by).toBe(false);
});
});
});
describe("POST /api/v1/accounts/:id/block", () => {
describe("POST /api/v1/accounts/:id/block", () => {
test("should block the specified user and return an APIRelationship object", async () => {
const response = await fetch(
`${config.http.base_url}/api/v1/accounts/${user2.id}/block`,
@ -333,16 +352,18 @@ describe("POST /api/v1/accounts/:id/block", () => {
);
expect(response.status).toBe(200);
expect(response.headers.get("content-type")).toBe("application/json");
expect(response.headers.get("content-type")).toBe(
"application/json"
);
const account: APIRelationship = await response.json();
expect(account.id).toBe(user2.id);
expect(account.blocking).toBe(true);
});
});
});
describe("POST /api/v1/accounts/:id/unblock", () => {
describe("POST /api/v1/accounts/:id/unblock", () => {
test("should unblock the specified user and return an APIRelationship object", async () => {
const response = await fetch(
`${config.http.base_url}/api/v1/accounts/${user2.id}/unblock`,
@ -357,16 +378,18 @@ describe("POST /api/v1/accounts/:id/unblock", () => {
);
expect(response.status).toBe(200);
expect(response.headers.get("content-type")).toBe("application/json");
expect(response.headers.get("content-type")).toBe(
"application/json"
);
const account: APIRelationship = await response.json();
expect(account.id).toBe(user2.id);
expect(account.blocking).toBe(false);
});
});
});
describe("POST /api/v1/accounts/:id/mute with notifications parameter", () => {
describe("POST /api/v1/accounts/:id/mute with notifications parameter", () => {
test("should mute the specified user and return an APIRelationship object with notifications set to false", async () => {
const response = await fetch(
`${config.http.base_url}/api/v1/accounts/${user2.id}/mute`,
@ -381,7 +404,9 @@ describe("POST /api/v1/accounts/:id/mute with notifications parameter", () => {
);
expect(response.status).toBe(200);
expect(response.headers.get("content-type")).toBe("application/json");
expect(response.headers.get("content-type")).toBe(
"application/json"
);
const account: APIRelationship = await response.json();
@ -404,7 +429,9 @@ describe("POST /api/v1/accounts/:id/mute with notifications parameter", () => {
);
expect(response.status).toBe(200);
expect(response.headers.get("content-type")).toBe("application/json");
expect(response.headers.get("content-type")).toBe(
"application/json"
);
const account: APIRelationship = await response.json();
@ -412,9 +439,9 @@ describe("POST /api/v1/accounts/:id/mute with notifications parameter", () => {
expect(account.muting).toBe(true);
expect(account.muting_notifications).toBe(true);
});
});
});
describe("POST /api/v1/accounts/:id/unmute", () => {
describe("POST /api/v1/accounts/:id/unmute", () => {
test("should unmute the specified user and return an APIRelationship object", async () => {
const response = await fetch(
`${config.http.base_url}/api/v1/accounts/${user2.id}/unmute`,
@ -429,16 +456,18 @@ describe("POST /api/v1/accounts/:id/unmute", () => {
);
expect(response.status).toBe(200);
expect(response.headers.get("content-type")).toBe("application/json");
expect(response.headers.get("content-type")).toBe(
"application/json"
);
const account: APIRelationship = await response.json();
expect(account.id).toBe(user2.id);
expect(account.muting).toBe(false);
});
});
});
describe("POST /api/v1/accounts/:id/pin", () => {
describe("POST /api/v1/accounts/:id/pin", () => {
test("should pin the specified user and return an APIRelationship object", async () => {
const response = await fetch(
`${config.http.base_url}/api/v1/accounts/${user2.id}/pin`,
@ -453,16 +482,18 @@ describe("POST /api/v1/accounts/:id/pin", () => {
);
expect(response.status).toBe(200);
expect(response.headers.get("content-type")).toBe("application/json");
expect(response.headers.get("content-type")).toBe(
"application/json"
);
const account: APIRelationship = await response.json();
expect(account.id).toBe(user2.id);
expect(account.endorsed).toBe(true);
});
});
});
describe("POST /api/v1/accounts/:id/unpin", () => {
describe("POST /api/v1/accounts/:id/unpin", () => {
test("should unpin the specified user and return an APIRelationship object", async () => {
const response = await fetch(
`${config.http.base_url}/api/v1/accounts/${user2.id}/unpin`,
@ -477,16 +508,18 @@ describe("POST /api/v1/accounts/:id/unpin", () => {
);
expect(response.status).toBe(200);
expect(response.headers.get("content-type")).toBe("application/json");
expect(response.headers.get("content-type")).toBe(
"application/json"
);
const account: APIRelationship = await response.json();
expect(account.id).toBe(user2.id);
expect(account.endorsed).toBe(false);
});
});
});
describe("POST /api/v1/accounts/:id/note", () => {
describe("POST /api/v1/accounts/:id/note", () => {
test("should update the specified account's note and return the updated account object", async () => {
const response = await fetch(
`${config.http.base_url}/api/v1/accounts/${user2.id}/note`,
@ -501,16 +534,18 @@ describe("POST /api/v1/accounts/:id/note", () => {
);
expect(response.status).toBe(200);
expect(response.headers.get("content-type")).toBe("application/json");
expect(response.headers.get("content-type")).toBe(
"application/json"
);
const account: APIAccount = await response.json();
expect(account.id).toBe(user2.id);
expect(account.note).toBe("This is a new note");
});
});
});
describe("GET /api/v1/accounts/relationships", () => {
describe("GET /api/v1/accounts/relationships", () => {
test("should return an array of APIRelationship objects for the authenticated user's relationships", async () => {
const response = await fetch(
`${config.http.base_url}/api/v1/accounts/relationships?id[]=${user2.id}`,
@ -523,7 +558,9 @@ describe("GET /api/v1/accounts/relationships", () => {
);
expect(response.status).toBe(200);
expect(response.headers.get("content-type")).toBe("application/json");
expect(response.headers.get("content-type")).toBe(
"application/json"
);
const relationships: APIRelationship[] = await response.json();
@ -539,9 +576,9 @@ describe("GET /api/v1/accounts/relationships", () => {
expect(relationships[0].domain_blocking).toBeDefined();
expect(relationships[0].notifying).toBeDefined();
});
});
});
describe("GET /api/v1/accounts/familiar_followers", () => {
describe("GET /api/v1/accounts/familiar_followers", () => {
test("should return an array of objects with id and accounts properties, where id is a string and accounts is an array of APIAccount objects", async () => {
const response = await fetch(
`${config.http.base_url}/api/v1/accounts/familiar_followers?id[]=${user2.id}`,
@ -554,7 +591,9 @@ describe("GET /api/v1/accounts/familiar_followers", () => {
);
expect(response.status).toBe(200);
expect(response.headers.get("content-type")).toBe("application/json");
expect(response.headers.get("content-type")).toBe(
"application/json"
);
const familiarFollowers: { id: string; accounts: APIAccount[] }[] =
await response.json();
@ -563,7 +602,9 @@ describe("GET /api/v1/accounts/familiar_followers", () => {
expect(familiarFollowers.length).toBeGreaterThan(0);
expect(typeof familiarFollowers[0].id).toBe("string");
expect(Array.isArray(familiarFollowers[0].accounts)).toBe(true);
expect(familiarFollowers[0].accounts.length).toBeGreaterThanOrEqual(0);
expect(familiarFollowers[0].accounts.length).toBeGreaterThanOrEqual(
0
);
if (familiarFollowers[0].accounts.length === 0) return;
expect(familiarFollowers[0].accounts[0].id).toBeDefined();
@ -578,18 +619,28 @@ describe("GET /api/v1/accounts/familiar_followers", () => {
expect(familiarFollowers[0].accounts[0].note).toBeDefined();
expect(familiarFollowers[0].accounts[0].url).toBeDefined();
expect(familiarFollowers[0].accounts[0].avatar).toBeDefined();
expect(familiarFollowers[0].accounts[0].avatar_static).toBeDefined();
expect(
familiarFollowers[0].accounts[0].avatar_static
).toBeDefined();
expect(familiarFollowers[0].accounts[0].header).toBeDefined();
expect(familiarFollowers[0].accounts[0].header_static).toBeDefined();
expect(familiarFollowers[0].accounts[0].followers_count).toBeDefined();
expect(familiarFollowers[0].accounts[0].following_count).toBeDefined();
expect(familiarFollowers[0].accounts[0].statuses_count).toBeDefined();
expect(
familiarFollowers[0].accounts[0].header_static
).toBeDefined();
expect(
familiarFollowers[0].accounts[0].followers_count
).toBeDefined();
expect(
familiarFollowers[0].accounts[0].following_count
).toBeDefined();
expect(
familiarFollowers[0].accounts[0].statuses_count
).toBeDefined();
expect(familiarFollowers[0].accounts[0].emojis).toBeDefined();
expect(familiarFollowers[0].accounts[0].fields).toBeDefined();
});
});
});
describe("GET /api/v1/statuses/:id", () => {
describe("GET /api/v1/statuses/:id", () => {
test("should return the specified status object", async () => {
const response = await fetch(
`${config.http.base_url}/api/v1/statuses/${status?.id}`,
@ -602,7 +653,9 @@ describe("GET /api/v1/statuses/:id", () => {
);
expect(response.status).toBe(200);
expect(response.headers.get("content-type")).toBe("application/json");
expect(response.headers.get("content-type")).toBe(
"application/json"
);
const statusJson = await response.json();
@ -630,9 +683,9 @@ describe("GET /api/v1/statuses/:id", () => {
expect(statusJson.bookmarked).toBeDefined();
expect(statusJson.pinned).toBeDefined();
});
});
});
describe("DELETE /api/v1/statuses/:id", () => {
describe("DELETE /api/v1/statuses/:id", () => {
test("should delete the specified status object", async () => {
const response = await fetch(
`${config.http.base_url}/api/v1/statuses/${status?.id}`,
@ -646,9 +699,9 @@ describe("DELETE /api/v1/statuses/:id", () => {
expect(response.status).toBe(200);
});
});
});
describe("GET /api/v1/instance", () => {
describe("GET /api/v1/instance", () => {
test("should return an APIInstance object", async () => {
const response = await fetch(
`${config.http.base_url}/api/v1/instance`,
@ -661,7 +714,9 @@ describe("GET /api/v1/instance", () => {
);
expect(response.status).toBe(200);
expect(response.headers.get("content-type")).toBe("application/json");
expect(response.headers.get("content-type")).toBe(
"application/json"
);
const instance: APIInstance = await response.json();
@ -680,9 +735,9 @@ describe("GET /api/v1/instance", () => {
expect(instance.approval_required).toBeDefined();
expect(instance.max_toot_chars).toBeDefined();
});
});
});
describe("GET /api/v1/custom_emojis", () => {
describe("GET /api/v1/custom_emojis", () => {
beforeAll(async () => {
const emoji = new Emoji();
@ -705,7 +760,9 @@ describe("GET /api/v1/custom_emojis", () => {
);
expect(response.status).toBe(200);
expect(response.headers.get("content-type")).toBe("application/json");
expect(response.headers.get("content-type")).toBe(
"application/json"
);
const emojis: APIEmoji[] = await response.json();
@ -716,9 +773,9 @@ describe("GET /api/v1/custom_emojis", () => {
afterAll(async () => {
await Emoji.delete({ shortcode: "test" });
});
});
});
afterAll(async () => {
afterAll(async () => {
const activities = await RawActivity.createQueryBuilder("activity")
.where("activity.data->>'actor' = :actor", {
actor: `${config.http.base_url}/users/test`,
@ -738,4 +795,5 @@ afterAll(async () => {
await user.remove();
await user2.remove();
});
});

View file

@ -0,0 +1,94 @@
import { getConfig } from "@config";
import { afterAll, beforeAll, describe, expect, it } from "bun:test";
import { LocalBackend, S3Backend } from "~classes/media";
import { unlink } from "fs/promises";
import { DeleteObjectCommand } from "@aws-sdk/client-s3";
const config = getConfig();
describe("LocalBackend", () => {
let localBackend: LocalBackend;
let fileName: string;
beforeAll(() => {
localBackend = new LocalBackend();
});
afterAll(async () => {
await unlink(`${process.cwd()}/uploads/${fileName}`);
});
describe("addMedia", () => {
it("should write the file to the local filesystem and return the hash", async () => {
const media = new File(["test"], "test.txt", {
type: "text/plain",
});
const hash = await localBackend.addMedia(media);
fileName = `${hash}`;
expect(hash).toBeDefined();
});
});
describe("getMediaByHash", () => {
it("should retrieve the file from the local filesystem and return it as a File object", async () => {
const media = await localBackend.getMediaByHash(fileName, "txt");
expect(media).toBeInstanceOf(File);
});
it("should return null if the file does not exist", async () => {
const media = await localBackend.getMediaByHash(
"does-not-exist",
"txt"
);
expect(media).toBeNull();
});
});
});
describe("S3Backend", () => {
const s3Backend = new S3Backend(config);
let fileName: string;
afterAll(async () => {
const command = new DeleteObjectCommand({
Bucket: config.s3.bucket_name,
Key: fileName,
});
await s3Backend.client.send(command);
});
describe("addMedia", () => {
it("should write the file to the S3 bucket and return the hash", async () => {
const media = new File(["test"], "test.txt", {
type: "text/plain",
});
const hash = await s3Backend.addMedia(media);
fileName = `${hash}`;
expect(hash).toBeDefined();
});
});
describe("getMediaByHash", () => {
it("should retrieve the file from the S3 bucket and return it as a File object", async () => {
const media = await s3Backend.getMediaByHash(fileName, "txt");
expect(media).toBeInstanceOf(File);
});
it("should return null if the file does not exist", async () => {
const media = await s3Backend.getMediaByHash(
"does-not-exist",
"txt"
);
expect(media).toBeNull();
});
});
});