diff --git a/bun.lockb b/bun.lockb index 04d28da7..633b6622 100755 Binary files a/bun.lockb and b/bun.lockb differ diff --git a/bunfig.toml b/bunfig.toml new file mode 100644 index 00000000..bea1efe1 --- /dev/null +++ b/bunfig.toml @@ -0,0 +1,2 @@ +[install.scopes] +"@jsr" = "https://npm.jsr.io" diff --git a/classes/configmanager.ts b/classes/configmanager.ts deleted file mode 100644 index 2007bbed..00000000 --- a/classes/configmanager.ts +++ /dev/null @@ -1,446 +0,0 @@ -/** - * @file configmanager.ts - * @summary ConfigManager system to retrieve and modify system configuration - * @description Can read from a hand-written file, config.toml, or from a machine-saved file, config.internal.toml - * @deprecated Use the new ConfigManager class instead - * Fuses both and provides a way to retrieve individual values - */ - -/* import { parse, stringify } from "@iarna/toml"; -import chalk from "chalk"; -import merge from "merge-deep-ts"; - -const scanConfig = async () => { - const config = Bun.file(process.cwd() + "/config/config.toml"); - - if (!(await config.exists())) { - console.error( - `${chalk.red(`✗`)} ${chalk.bold( - "Error while reading config: " - )} Config file not found` - ); - process.exit(1); - } - - return parse(await config.text()) as ConfigType; -}; */ - -// Creates the internal config with nothing in it if it doesnt exist -/* const scanInternalConfig = async () => { - const config = Bun.file(process.cwd() + "/config/config.internal.toml"); - - if (!(await config.exists())) { - await Bun.write(config, ""); - } - - return parse(await config.text()) as ConfigType; -}; - -let config = await scanConfig(); -const internalConfig = await scanInternalConfig(); - -export interface ConfigType { - database: { - host: string; - port: number; - username: string; - password: string; - database: string; - }; - - redis: { - queue: { - host: string; - port: number; - password: string; - database: number | null; - }; - cache: { - host: string; - port: number; - password: string; - database: number | null; - enabled: boolean; - }; - }; - - meilisearch: { - host: string; - port: number; - api_key: string; - enabled: boolean; - }; - - signups: { - tos_url: string; - rules: string[]; - registration: boolean; - }; - - oidc: { - providers: { - name: string; - id: string; - url: string; - client_id: string; - client_secret: string; - icon: string; - }[]; - }; - - http: { - base_url: string; - bind: string; - bind_port: string; - banned_ips: string[]; - banned_user_agents: string[]; - }; - - instance: { - name: string; - description: string; - banner: string; - logo: string; - }; - - smtp: { - server: string; - port: number; - username: string; - password: string; - tls: boolean; - }; - - validation: { - max_displayname_size: number; - max_bio_size: number; - max_username_size: number; - max_note_size: number; - max_avatar_size: number; - max_header_size: number; - max_media_size: number; - max_media_attachments: number; - max_media_description_size: number; - max_poll_options: number; - max_poll_option_size: number; - min_poll_duration: number; - max_poll_duration: number; - - username_blacklist: string[]; - blacklist_tempmail: boolean; - email_blacklist: string[]; - url_scheme_whitelist: string[]; - - enforce_mime_types: boolean; - allowed_mime_types: string[]; - }; - - media: { - backend: string; - deduplicate_media: boolean; - conversion: { - convert_images: boolean; - convert_to: string; - }; - }; - - s3: { - endpoint: string; - access_key: string; - secret_access_key: string; - region: string; - bucket_name: string; - public_url: string; - }; - - defaults: { - visibility: string; - language: string; - avatar: string; - header: string; - }; - - email: { - send_on_report: boolean; - send_on_suspend: boolean; - send_on_unsuspend: boolean; - }; - - activitypub: { - use_tombstones: boolean; - reject_activities: string[]; - force_followers_only: string[]; - discard_reports: string[]; - discard_deletes: string[]; - discard_banners: string[]; - discard_avatars: string[]; - discard_updates: string[]; - discard_follows: string[]; - force_sensitive: string[]; - remove_media: string[]; - fetch_all_collection_members: boolean; - authorized_fetch: boolean; - }; - - filters: { - note_filters: string[]; - username_filters: string[]; - displayname_filters: string[]; - bio_filters: string[]; - emoji_filters: string[]; - }; - - logging: { - log_requests: boolean; - log_requests_verbose: boolean; - log_filters: boolean; - }; - - ratelimits: { - duration_coeff: number; - max_coeff: number; - }; - - custom_ratelimits: Record< - string, - { - duration: number; - max: number; - } - >; - [key: string]: unknown; -} - -export const configDefaults: ConfigType = { - http: { - bind: "http://0.0.0.0", - bind_port: "8000", - base_url: "http://lysand.localhost:8000", - banned_ips: [], - banned_user_agents: [], - }, - database: { - host: "localhost", - port: 5432, - username: "postgres", - password: "postgres", - database: "lysand", - }, - redis: { - queue: { - host: "localhost", - port: 6379, - password: "", - database: 0, - }, - cache: { - host: "localhost", - port: 6379, - password: "", - database: 1, - enabled: false, - }, - }, - meilisearch: { - host: "localhost", - port: 1491, - api_key: "", - enabled: false, - }, - signups: { - tos_url: "", - rules: [], - registration: false, - }, - oidc: { - providers: [], - }, - instance: { - banner: "", - description: "", - logo: "", - name: "", - }, - smtp: { - password: "", - port: 465, - server: "", - tls: true, - username: "", - }, - media: { - backend: "local", - deduplicate_media: true, - conversion: { - convert_images: false, - convert_to: "webp", - }, - }, - email: { - send_on_report: false, - send_on_suspend: false, - send_on_unsuspend: false, - }, - s3: { - access_key: "", - bucket_name: "", - endpoint: "", - public_url: "", - region: "", - secret_access_key: "", - }, - validation: { - max_displayname_size: 50, - max_bio_size: 6000, - max_note_size: 5000, - max_avatar_size: 5_000_000, - max_header_size: 5_000_000, - max_media_size: 40_000_000, - max_media_attachments: 10, - max_media_description_size: 1000, - max_poll_options: 20, - max_poll_option_size: 500, - min_poll_duration: 60, - max_poll_duration: 1893456000, - max_username_size: 30, - - username_blacklist: [ - ".well-known", - "~", - "about", - "activities", - "api", - "auth", - "dev", - "inbox", - "internal", - "main", - "media", - "nodeinfo", - "notice", - "oauth", - "objects", - "proxy", - "push", - "registration", - "relay", - "settings", - "status", - "tag", - "users", - "web", - "search", - "mfa", - ], - - blacklist_tempmail: false, - - email_blacklist: [], - - url_scheme_whitelist: [ - "http", - "https", - "ftp", - "dat", - "dweb", - "gopher", - "hyper", - "ipfs", - "ipns", - "irc", - "xmpp", - "ircs", - "magnet", - "mailto", - "mumble", - "ssb", - ], - - enforce_mime_types: false, - allowed_mime_types: [], - }, - defaults: { - visibility: "public", - language: "en", - avatar: "", - header: "", - }, - activitypub: { - use_tombstones: true, - reject_activities: [], - force_followers_only: [], - discard_reports: [], - discard_deletes: [], - discard_banners: [], - discard_avatars: [], - force_sensitive: [], - discard_updates: [], - discard_follows: [], - remove_media: [], - fetch_all_collection_members: false, - authorized_fetch: false, - }, - filters: { - note_filters: [], - username_filters: [], - displayname_filters: [], - bio_filters: [], - emoji_filters: [], - }, - logging: { - log_requests: false, - log_requests_verbose: false, - log_filters: true, - }, - ratelimits: { - duration_coeff: 1, - max_coeff: 1, - }, - custom_ratelimits: {}, -}; */ - -/* export const getConfig = () => { - // Deeply merge configDefaults, config and internalConfig - return merge([configDefaults, config, internalConfig]) as any as ConfigType; -}; - */ -/** - * Sets the internal config - * @param newConfig Any part of ConfigType - */ -/* export const setConfig = async (newConfig: Partial) => { - const newInternalConfig = merge([ - internalConfig, - newConfig, - ]) as any as ConfigType; - - // Prepend a warning comment and write the new TOML to the file - await Bun.write( - Bun.file(process.cwd() + "/config/config.internal.toml"), - `# This file is automatically generated. Do not modify it manually.\n${stringify( - newInternalConfig as any - )}` - ); -}; */ - -/* export const getHost = () => { - const url = new URL(getConfig().http.base_url); - - return url.host; -}; - */ -// Refresh config every 5 seconds -/* setInterval(() => { - scanConfig() - .then(newConfig => { - if (newConfig !== config) { - config = newConfig; - } - }) - .catch(e => { - console.error(e); - }); -}, 5000); */ - -/* export { config }; - */ diff --git a/classes/media.ts b/classes/media.ts deleted file mode 100644 index 483a173d..00000000 --- a/classes/media.ts +++ /dev/null @@ -1,273 +0,0 @@ -import type { GetObjectCommandOutput } from "@aws-sdk/client-s3"; -import { - GetObjectCommand, - PutObjectCommand, - S3Client, -} from "@aws-sdk/client-s3"; -import type { ConfigType } from "~classes/configmanager"; -import sharp from "sharp"; -import { exists, mkdir } from "fs/promises"; -class MediaBackend { - backend: string; - - constructor(backend: string) { - this.backend = backend; - } - - /** - * Adds media to the media backend - * @param media - * @returns The hash of the file in SHA-256 (hex format) with the file extension added to it - */ - async addMedia(media: File) { - const hash = new Bun.SHA256() - .update(await media.arrayBuffer()) - .digest("hex"); - - return `${hash}.${media.name.split(".").pop()}`; - } - - async convertMedia(media: File, config: ConfigType) { - const sharpCommand = sharp(await media.arrayBuffer()); - - // Rename ".jpg" files to ".jpeg" to avoid sharp errors - let name = media.name; - if (media.name.endsWith(".jpg")) { - name = media.name.replace(".jpg", ".jpeg"); - } - - const fileFormatToConvertTo = config.media.conversion.convert_to; - - switch (fileFormatToConvertTo) { - case "png": - return new File( - [(await sharpCommand.png().toBuffer()).buffer] as any, - // Replace the file extension with PNG - name.replace(/\.[^/.]+$/, ".png"), - { - type: "image/png", - } - ); - case "webp": - return new File( - [(await sharpCommand.webp().toBuffer()).buffer] as any, - // Replace the file extension with WebP - name.replace(/\.[^/.]+$/, ".webp"), - { - type: "image/webp", - } - ); - case "jpeg": - return new File( - [(await sharpCommand.jpeg().toBuffer()).buffer] as any, - // Replace the file extension with JPEG - name.replace(/\.[^/.]+$/, ".jpeg"), - { - type: "image/jpeg", - } - ); - case "avif": - return new File( - [(await sharpCommand.avif().toBuffer()).buffer] as any, - // Replace the file extension with AVIF - name.replace(/\.[^/.]+$/, ".avif"), - { - type: "image/avif", - } - ); - // Needs special build of libvips - case "jxl": - return new File( - [(await sharpCommand.jxl().toBuffer()).buffer] as any, - // Replace the file extension with JXL - name.replace(/\.[^/.]+$/, ".jxl"), - { - type: "image/jxl", - } - ); - case "heif": - return new File( - [(await sharpCommand.heif().toBuffer()).buffer] as any, - // Replace the file extension with HEIF - name.replace(/\.[^/.]+$/, ".heif"), - { - type: "image/heif", - } - ); - default: - return media; - } - } - - /** - * Retrieves element from media backend by hash - * @param hash The hash of the element in SHA-256 hex format - * @param extension The extension of the file - * @returns The file as a File object - */ - // eslint-disable-next-line @typescript-eslint/require-await, @typescript-eslint/no-unused-vars - async getMediaByHash( - // eslint-disable-next-line @typescript-eslint/no-unused-vars - hash: string - ): Promise { - return new File([], "test"); - } -} - -/** - * S3 Backend, stores files in S3 - */ -export class S3Backend extends MediaBackend { - client: S3Client; - config: ConfigType; - - constructor(config: ConfigType) { - super("s3"); - - this.config = config; - - this.client = new S3Client({ - endpoint: this.config.s3.endpoint, - region: this.config.s3.region || "auto", - credentials: { - accessKeyId: this.config.s3.access_key, - secretAccessKey: this.config.s3.secret_access_key, - }, - }); - } - - async addMedia(media: File): Promise { - if (this.config.media.conversion.convert_images) { - media = await this.convertMedia(media, this.config); - } - - const hash = await super.addMedia(media); - - if (!hash) { - throw new Error("Failed to hash file"); - } - - // Check if file is already present - const existingFile = await this.getMediaByHash(hash); - - if (existingFile) { - // File already exists, so return the hash without uploading it - return hash; - } - - const command = new PutObjectCommand({ - Bucket: this.config.s3.bucket_name, - Key: hash, - Body: Buffer.from(await media.arrayBuffer()), - ContentType: media.type, - ContentLength: media.size, - Metadata: { - "x-amz-meta-original-name": media.name, - }, - }); - - const response = await this.client.send(command); - - if (response.$metadata.httpStatusCode !== 200) { - throw new Error("Failed to upload file"); - } - - return hash; - } - - async getMediaByHash(hash: string): Promise { - const command = new GetObjectCommand({ - Bucket: this.config.s3.bucket_name, - Key: hash, - }); - - let response: GetObjectCommandOutput; - - try { - response = await this.client.send(command); - } catch { - return null; - } - - if (response.$metadata.httpStatusCode !== 200) { - throw new Error("Failed to get file"); - } - - const body = await response.Body?.transformToByteArray(); - - if (!body) { - throw new Error("Failed to get file"); - } - - return new File([body], hash, { - type: response.ContentType, - }); - } -} - -/** - * Local backend, stores files on filesystem - */ -export class LocalBackend extends MediaBackend { - config: ConfigType; - - constructor(config: ConfigType) { - super("local"); - - this.config = config; - } - - async addMedia(media: File): Promise { - if (this.config.media.conversion.convert_images) { - media = await this.convertMedia(media, this.config); - } - - const hash = await super.addMedia(media); - - if (!(await exists(`${process.cwd()}/uploads`))) { - await mkdir(`${process.cwd()}/uploads`); - } - - await Bun.write(Bun.file(`${process.cwd()}/uploads/${hash}`), media); - - return hash; - } - - async getMediaByHash(hash: string): Promise { - const file = Bun.file(`${process.cwd()}/uploads/${hash}`); - - if (!(await file.exists())) { - return null; - } - - return new File([await file.arrayBuffer()], `${hash}`, { - type: file.type, - }); - } -} - -export const uploadFile = (file: File, config: ConfigType) => { - const backend = config.media.backend; - - if (backend === "local") { - return new LocalBackend(config).addMedia(file); - } else if (backend === "s3") { - return new S3Backend(config).addMedia(file); - } -}; - -export const getFile = ( - hash: string, - extension: string, - config: ConfigType -) => { - const backend = config.media.backend; - - if (backend === "local") { - return new LocalBackend(config).getMediaByHash(hash); - } else if (backend === "s3") { - return new S3Backend(config).getMediaByHash(hash); - } - - return null; -}; diff --git a/database/entities/Attachment.ts b/database/entities/Attachment.ts index fefa8cd7..abb07c9e 100644 --- a/database/entities/Attachment.ts +++ b/database/entities/Attachment.ts @@ -1,5 +1,6 @@ -import type { ConfigType } from "~classes/configmanager"; import type { Attachment } from "@prisma/client"; +import type { ConfigType } from "config-manager"; +import { MediaBackendType } from "media-manager"; import type { APIAsyncAttachment } from "~types/entities/async_attachment"; import type { APIAttachment } from "~types/entities/attachment"; @@ -56,11 +57,13 @@ export const attachmentToAPI = ( }; }; -export const getUrl = (hash: string, config: ConfigType) => { - if (config.media.backend === "local") { - return `${config.http.base_url}/media/${hash}`; - } else if (config.media.backend === "s3") { - return `${config.s3.public_url}/${hash}`; +export const getUrl = (name: string, config: ConfigType) => { + // eslint-disable-next-line @typescript-eslint/no-unsafe-enum-comparison + if (config.media.backend === MediaBackendType.LOCAL) { + return `${config.http.base_url}/media/${name}`; + // eslint-disable-next-line @typescript-eslint/no-unsafe-enum-comparison + } else if (config.media.backend === MediaBackendType.S3) { + return `${config.s3.public_url}/${name}`; } return ""; }; diff --git a/package.json b/package.json index f1ef982e..ea00043a 100644 --- a/package.json +++ b/package.json @@ -111,6 +111,9 @@ "semver": "^7.5.4", "sharp": "^0.33.0-rc.2", "request-parser": "file:packages/request-parser", - "config-manager": "file:packages/config-manager" + "config-manager": "file:packages/config-manager", + "cli-parser": "file:packages/cli-parser", + "log-manager": "file:packages/log-manager", + "media-manager": "file:packages/media-manager" } } \ No newline at end of file diff --git a/packages/media-manager/backends/local.ts b/packages/media-manager/backends/local.ts index 5c955b84..d5a8fb99 100644 --- a/packages/media-manager/backends/local.ts +++ b/packages/media-manager/backends/local.ts @@ -4,8 +4,8 @@ import { MediaBackend, MediaBackendType, MediaHasher } from ".."; import type { ConfigType } from "config-manager"; export class LocalMediaBackend extends MediaBackend { - constructor(private config: ConfigType) { - super(MediaBackendType.LOCAL); + constructor(config: ConfigType) { + super(config, MediaBackendType.LOCAL); } public async addFile(file: File) { diff --git a/packages/media-manager/backends/s3.ts b/packages/media-manager/backends/s3.ts index 8098e2f2..46c2cb41 100644 --- a/packages/media-manager/backends/s3.ts +++ b/packages/media-manager/backends/s3.ts @@ -6,7 +6,7 @@ import type { ConfigType } from "config-manager"; export class S3MediaBackend extends MediaBackend { constructor( - private config: ConfigType, + config: ConfigType, private s3Client = new S3Client({ endPoint: config.s3.endpoint, useSSL: true, @@ -16,7 +16,7 @@ export class S3MediaBackend extends MediaBackend { secretKey: config.s3.secret_access_key, }) ) { - super(MediaBackendType.S3); + super(config, MediaBackendType.S3); } public async addFile(file: File) { diff --git a/packages/media-manager/index.ts b/packages/media-manager/index.ts index 77c2f581..501495b1 100644 --- a/packages/media-manager/index.ts +++ b/packages/media-manager/index.ts @@ -27,7 +27,10 @@ export class MediaHasher { } export class MediaBackend { - constructor(private backend: MediaBackendType) {} + constructor( + public config: ConfigType, + public backend: MediaBackendType + ) {} public getBackendType() { return this.backend; diff --git a/packages/media-manager/tests/media-backends.test.ts b/packages/media-manager/tests/media-backends.test.ts index bad3ac5e..fc36068f 100644 --- a/packages/media-manager/tests/media-backends.test.ts +++ b/packages/media-manager/tests/media-backends.test.ts @@ -16,7 +16,6 @@ describe("MediaBackend", () => { let mockConfig: ConfigType; beforeEach(() => { - mediaBackend = new MediaBackend(MediaBackendType.S3); mockConfig = { media: { conversion: { @@ -24,6 +23,7 @@ describe("MediaBackend", () => { }, }, } as ConfigType; + mediaBackend = new MediaBackend(mockConfig, MediaBackendType.S3); }); it("should initialize with correct backend type", () => { diff --git a/server/api/api/v1/accounts/update_credentials/index.ts b/server/api/api/v1/accounts/update_credentials/index.ts index fc66d92c..4a48807f 100644 --- a/server/api/api/v1/accounts/update_credentials/index.ts +++ b/server/api/api/v1/accounts/update_credentials/index.ts @@ -3,12 +3,16 @@ import { userRelations, userToAPI } from "~database/entities/User"; import { apiRoute, applyConfig } from "@api"; import { sanitize } from "isomorphic-dompurify"; import { sanitizeHtml } from "@sanitization"; -import { uploadFile } from "~classes/media"; import ISO6391 from "iso-639-1"; import { parseEmojis } from "~database/entities/Emoji"; import { client } from "~database/datasource"; import type { APISource } from "~types/entities/source"; import { convertTextToHtml } from "@formatting"; +import { MediaBackendType } from "media-manager"; +import type { MediaBackend } from "media-manager"; +import { LocalMediaBackend } from "~packages/media-manager/backends/local"; +import { S3MediaBackend } from "~packages/media-manager/backends/s3"; +import { getUrl } from "~database/entities/Attachment"; export const meta = applyConfig({ allowedMethods: ["PATCH"], @@ -69,6 +73,20 @@ export default apiRoute<{ }; } + let mediaManager: MediaBackend; + + switch (config.media.backend as MediaBackendType) { + case MediaBackendType.LOCAL: + mediaManager = new LocalMediaBackend(config); + break; + case MediaBackendType.S3: + mediaManager = new S3MediaBackend(config); + break; + default: + // TODO: Replace with logger + throw new Error("Invalid media backend"); + } + if (display_name) { // Check if within allowed display name lengths if ( @@ -167,9 +185,9 @@ export default apiRoute<{ ); } - const hash = await uploadFile(avatar, config); + const { uploadedFile } = await mediaManager.addFile(avatar); - user.avatar = hash || ""; + user.avatar = getUrl(uploadedFile.name, config); } if (header) { @@ -181,9 +199,9 @@ export default apiRoute<{ ); } - const hash = await uploadFile(header, config); + const { uploadedFile } = await mediaManager.addFile(header); - user.header = hash || ""; + user.header = getUrl(uploadedFile.name, config); } if (locked) { diff --git a/server/api/api/v1/media/[id]/index.ts b/server/api/api/v1/media/[id]/index.ts index bec0e978..8abd1bf0 100644 --- a/server/api/api/v1/media/[id]/index.ts +++ b/server/api/api/v1/media/[id]/index.ts @@ -1,11 +1,13 @@ import { apiRoute, applyConfig } from "@api"; import { errorResponse, jsonResponse } from "@response"; import { client } from "~database/datasource"; -import type { APIRouteMeta } from "~types/api"; -import { uploadFile } from "~classes/media"; import { attachmentToAPI, getUrl } from "~database/entities/Attachment"; +import type { MediaBackend } from "media-manager"; +import { MediaBackendType } from "media-manager"; +import { LocalMediaBackend } from "~packages/media-manager/backends/local"; +import { S3MediaBackend } from "~packages/media-manager/backends/s3"; -export const meta: APIRouteMeta = applyConfig({ +export const meta = applyConfig({ allowedMethods: ["GET", "PUT"], ratelimits: { max: 10, @@ -61,13 +63,23 @@ export default apiRoute<{ let thumbnailUrl = attachment.thumbnail_url; - if (thumbnail) { - const hash = await uploadFile( - thumbnail as unknown as File, - config - ); + let mediaManager: MediaBackend; - thumbnailUrl = hash ? getUrl(hash, config) : ""; + switch (config.media.backend as MediaBackendType) { + case MediaBackendType.LOCAL: + mediaManager = new LocalMediaBackend(config); + break; + case MediaBackendType.S3: + mediaManager = new S3MediaBackend(config); + break; + default: + // TODO: Replace with logger + throw new Error("Invalid media backend"); + } + + if (thumbnail) { + const { uploadedFile } = await mediaManager.addFile(thumbnail); + thumbnailUrl = getUrl(uploadedFile.name, config); } const descriptionText = description || attachment.description; diff --git a/server/api/api/v1/media/index.ts b/server/api/api/v1/media/index.ts index 0119a7d4..202e5dbb 100644 --- a/server/api/api/v1/media/index.ts +++ b/server/api/api/v1/media/index.ts @@ -2,12 +2,14 @@ import { apiRoute, applyConfig } from "@api"; import { errorResponse, jsonResponse } from "@response"; import { client } from "~database/datasource"; import { encode } from "blurhash"; -import type { APIRouteMeta } from "~types/api"; import sharp from "sharp"; -import { uploadFile } from "~classes/media"; import { attachmentToAPI, getUrl } from "~database/entities/Attachment"; +import { MediaBackendType } from "media-manager"; +import type { MediaBackend } from "media-manager"; +import { LocalMediaBackend } from "~packages/media-manager/backends/local"; +import { S3MediaBackend } from "~packages/media-manager/backends/s3"; -export const meta: APIRouteMeta = applyConfig({ +export const meta = applyConfig({ allowedMethods: ["POST"], ratelimits: { max: 10, @@ -88,16 +90,30 @@ export default apiRoute<{ let url = ""; - const hash = await uploadFile(file, config); + let mediaManager: MediaBackend; - url = hash ? getUrl(hash, config) : ""; + switch (config.media.backend as MediaBackendType) { + case MediaBackendType.LOCAL: + mediaManager = new LocalMediaBackend(config); + break; + case MediaBackendType.S3: + mediaManager = new S3MediaBackend(config); + break; + default: + // TODO: Replace with logger + throw new Error("Invalid media backend"); + } + + const { uploadedFile } = await mediaManager.addFile(file); + + url = getUrl(uploadedFile.name, config); let thumbnailUrl = ""; if (thumbnail) { - const hash = await uploadFile(thumbnail as unknown as File, config); + const { uploadedFile } = await mediaManager.addFile(thumbnail); - thumbnailUrl = hash ? getUrl(hash, config) : ""; + thumbnailUrl = getUrl(uploadedFile.name, config); } const newAttachment = await client.attachment.create({ diff --git a/server/api/api/v1/profile/avatar.ts b/server/api/api/v1/profile/avatar.ts index f65ce264..f6a1ea47 100644 --- a/server/api/api/v1/profile/avatar.ts +++ b/server/api/api/v1/profile/avatar.ts @@ -2,9 +2,8 @@ import { apiRoute, applyConfig } from "@api"; import { errorResponse, jsonResponse } from "@response"; import { client } from "~database/datasource"; import { userRelations, userToAPI } from "~database/entities/User"; -import type { APIRouteMeta } from "~types/api"; -export const meta: APIRouteMeta = applyConfig({ +export const meta = applyConfig({ allowedMethods: ["DELETE"], ratelimits: { max: 10, diff --git a/server/api/api/v1/profile/header.ts b/server/api/api/v1/profile/header.ts index 6a53e6d5..b3b52a60 100644 --- a/server/api/api/v1/profile/header.ts +++ b/server/api/api/v1/profile/header.ts @@ -2,9 +2,8 @@ import { apiRoute, applyConfig } from "@api"; import { errorResponse, jsonResponse } from "@response"; import { client } from "~database/datasource"; import { userRelations, userToAPI } from "~database/entities/User"; -import type { APIRouteMeta } from "~types/api"; -export const meta: APIRouteMeta = applyConfig({ +export const meta = applyConfig({ allowedMethods: ["DELETE"], ratelimits: { max: 10, diff --git a/server/api/api/v1/statuses/[id]/context.ts b/server/api/api/v1/statuses/[id]/context.ts index 518b9493..93eed932 100644 --- a/server/api/api/v1/statuses/[id]/context.ts +++ b/server/api/api/v1/statuses/[id]/context.ts @@ -7,9 +7,8 @@ import { statusAndUserRelations, statusToAPI, } from "~database/entities/Status"; -import type { APIRouteMeta } from "~types/api"; -export const meta: APIRouteMeta = applyConfig({ +export const meta = applyConfig({ allowedMethods: ["GET"], ratelimits: { max: 8, diff --git a/server/api/api/v1/statuses/[id]/favourite.ts b/server/api/api/v1/statuses/[id]/favourite.ts index 8557a7b2..c58e959e 100644 --- a/server/api/api/v1/statuses/[id]/favourite.ts +++ b/server/api/api/v1/statuses/[id]/favourite.ts @@ -8,10 +8,9 @@ import { statusAndUserRelations, statusToAPI, } from "~database/entities/Status"; -import type { APIRouteMeta } from "~types/api"; import type { APIStatus } from "~types/entities/status"; -export const meta: APIRouteMeta = applyConfig({ +export const meta = applyConfig({ allowedMethods: ["POST"], ratelimits: { max: 100, diff --git a/server/api/api/v1/statuses/[id]/favourited_by.ts b/server/api/api/v1/statuses/[id]/favourited_by.ts index 3d61969f..066eb681 100644 --- a/server/api/api/v1/statuses/[id]/favourited_by.ts +++ b/server/api/api/v1/statuses/[id]/favourited_by.ts @@ -6,9 +6,8 @@ import { statusAndUserRelations, } from "~database/entities/Status"; import { userRelations, userToAPI } from "~database/entities/User"; -import type { APIRouteMeta } from "~types/api"; -export const meta: APIRouteMeta = applyConfig({ +export const meta = applyConfig({ allowedMethods: ["GET"], ratelimits: { max: 100, diff --git a/server/api/api/v1/statuses/[id]/index.ts b/server/api/api/v1/statuses/[id]/index.ts index d0fae956..814dcca1 100644 --- a/server/api/api/v1/statuses/[id]/index.ts +++ b/server/api/api/v1/statuses/[id]/index.ts @@ -9,9 +9,8 @@ import { statusAndUserRelations, statusToAPI, } from "~database/entities/Status"; -import type { APIRouteMeta } from "~types/api"; -export const meta: APIRouteMeta = applyConfig({ +export const meta = applyConfig({ allowedMethods: ["GET", "DELETE", "PUT"], ratelimits: { max: 100, diff --git a/server/api/api/v1/statuses/[id]/pin.ts b/server/api/api/v1/statuses/[id]/pin.ts index 1d3a2e56..fd215f46 100644 --- a/server/api/api/v1/statuses/[id]/pin.ts +++ b/server/api/api/v1/statuses/[id]/pin.ts @@ -3,9 +3,8 @@ import { apiRoute, applyConfig } from "@api"; import { errorResponse, jsonResponse } from "@response"; import { client } from "~database/datasource"; import { statusAndUserRelations, statusToAPI } from "~database/entities/Status"; -import type { APIRouteMeta } from "~types/api"; -export const meta: APIRouteMeta = applyConfig({ +export const meta = applyConfig({ allowedMethods: ["POST"], ratelimits: { max: 100, diff --git a/server/api/api/v1/statuses/[id]/reblog.ts b/server/api/api/v1/statuses/[id]/reblog.ts index b93846ed..73b0ef73 100644 --- a/server/api/api/v1/statuses/[id]/reblog.ts +++ b/server/api/api/v1/statuses/[id]/reblog.ts @@ -8,9 +8,8 @@ import { statusToAPI, } from "~database/entities/Status"; import { type UserWithRelations } from "~database/entities/User"; -import type { APIRouteMeta } from "~types/api"; -export const meta: APIRouteMeta = applyConfig({ +export const meta = applyConfig({ allowedMethods: ["POST"], ratelimits: { max: 100, diff --git a/server/api/api/v1/statuses/[id]/reblogged_by.ts b/server/api/api/v1/statuses/[id]/reblogged_by.ts index 9af70f36..b02a02a2 100644 --- a/server/api/api/v1/statuses/[id]/reblogged_by.ts +++ b/server/api/api/v1/statuses/[id]/reblogged_by.ts @@ -6,9 +6,8 @@ import { statusAndUserRelations, } from "~database/entities/Status"; import { userRelations, userToAPI } from "~database/entities/User"; -import type { APIRouteMeta } from "~types/api"; -export const meta: APIRouteMeta = applyConfig({ +export const meta = applyConfig({ allowedMethods: ["GET"], ratelimits: { max: 100, diff --git a/server/api/api/v1/statuses/[id]/source.ts b/server/api/api/v1/statuses/[id]/source.ts index 02d3fe27..5bb90600 100644 --- a/server/api/api/v1/statuses/[id]/source.ts +++ b/server/api/api/v1/statuses/[id]/source.ts @@ -5,9 +5,8 @@ import { isViewableByUser, statusAndUserRelations, } from "~database/entities/Status"; -import type { APIRouteMeta } from "~types/api"; -export const meta: APIRouteMeta = applyConfig({ +export const meta = applyConfig({ allowedMethods: ["GET"], ratelimits: { max: 100, diff --git a/server/api/api/v1/statuses/[id]/unfavourite.ts b/server/api/api/v1/statuses/[id]/unfavourite.ts index 7d02a30d..66ebe82b 100644 --- a/server/api/api/v1/statuses/[id]/unfavourite.ts +++ b/server/api/api/v1/statuses/[id]/unfavourite.ts @@ -8,10 +8,9 @@ import { statusAndUserRelations, statusToAPI, } from "~database/entities/Status"; -import type { APIRouteMeta } from "~types/api"; import type { APIStatus } from "~types/entities/status"; -export const meta: APIRouteMeta = applyConfig({ +export const meta = applyConfig({ allowedMethods: ["POST"], ratelimits: { max: 100, diff --git a/server/api/api/v1/statuses/[id]/unpin.ts b/server/api/api/v1/statuses/[id]/unpin.ts index 8d10af6a..89ac2da4 100644 --- a/server/api/api/v1/statuses/[id]/unpin.ts +++ b/server/api/api/v1/statuses/[id]/unpin.ts @@ -2,9 +2,8 @@ import { apiRoute, applyConfig } from "@api"; import { errorResponse, jsonResponse } from "@response"; import { client } from "~database/datasource"; import { statusAndUserRelations, statusToAPI } from "~database/entities/Status"; -import type { APIRouteMeta } from "~types/api"; -export const meta: APIRouteMeta = applyConfig({ +export const meta = applyConfig({ allowedMethods: ["POST"], ratelimits: { max: 100, diff --git a/server/api/api/v1/statuses/[id]/unreblog.ts b/server/api/api/v1/statuses/[id]/unreblog.ts index 78179bc1..07d92b33 100644 --- a/server/api/api/v1/statuses/[id]/unreblog.ts +++ b/server/api/api/v1/statuses/[id]/unreblog.ts @@ -6,10 +6,9 @@ import { statusAndUserRelations, statusToAPI, } from "~database/entities/Status"; -import type { APIRouteMeta } from "~types/api"; import type { APIStatus } from "~types/entities/status"; -export const meta: APIRouteMeta = applyConfig({ +export const meta = applyConfig({ allowedMethods: ["POST"], ratelimits: { max: 100, diff --git a/server/api/api/v1/statuses/index.ts b/server/api/api/v1/statuses/index.ts index c51f5256..97eb34e5 100644 --- a/server/api/api/v1/statuses/index.ts +++ b/server/api/api/v1/statuses/index.ts @@ -11,9 +11,8 @@ import { statusToAPI, } from "~database/entities/Status"; import type { UserWithRelations } from "~database/entities/User"; -import type { APIRouteMeta } from "~types/api"; -export const meta: APIRouteMeta = applyConfig({ +export const meta = applyConfig({ allowedMethods: ["POST"], ratelimits: { max: 300, diff --git a/server/api/api/v1/timelines/home.ts b/server/api/api/v1/timelines/home.ts index 014d6f28..ce4e96aa 100644 --- a/server/api/api/v1/timelines/home.ts +++ b/server/api/api/v1/timelines/home.ts @@ -2,9 +2,8 @@ import { apiRoute, applyConfig } from "@api"; import { errorResponse, jsonResponse } from "@response"; import { client } from "~database/datasource"; import { statusAndUserRelations, statusToAPI } from "~database/entities/Status"; -import type { APIRouteMeta } from "~types/api"; -export const meta: APIRouteMeta = applyConfig({ +export const meta = applyConfig({ allowedMethods: ["GET"], ratelimits: { max: 200, diff --git a/server/api/api/v1/timelines/public.ts b/server/api/api/v1/timelines/public.ts index 62c3b72f..c58cfb15 100644 --- a/server/api/api/v1/timelines/public.ts +++ b/server/api/api/v1/timelines/public.ts @@ -2,9 +2,8 @@ import { apiRoute, applyConfig } from "@api"; import { errorResponse, jsonResponse } from "@response"; import { client } from "~database/datasource"; import { statusAndUserRelations, statusToAPI } from "~database/entities/Status"; -import type { APIRouteMeta } from "~types/api"; -export const meta: APIRouteMeta = applyConfig({ +export const meta = applyConfig({ allowedMethods: ["GET"], ratelimits: { max: 200, diff --git a/server/api/api/v2/media/index.ts b/server/api/api/v2/media/index.ts index eeadcbc6..2593ae05 100644 --- a/server/api/api/v2/media/index.ts +++ b/server/api/api/v2/media/index.ts @@ -2,12 +2,14 @@ import { apiRoute, applyConfig } from "@api"; import { errorResponse, jsonResponse } from "@response"; import { client } from "~database/datasource"; import { encode } from "blurhash"; -import type { APIRouteMeta } from "~types/api"; import sharp from "sharp"; -import { uploadFile } from "~classes/media"; import { attachmentToAPI, getUrl } from "~database/entities/Attachment"; +import type { MediaBackend } from "media-manager"; +import { MediaBackendType } from "media-manager"; +import { LocalMediaBackend } from "~packages/media-manager/backends/local"; +import { S3MediaBackend } from "~packages/media-manager/backends/s3"; -export const meta: APIRouteMeta = applyConfig({ +export const meta = applyConfig({ allowedMethods: ["POST"], ratelimits: { max: 10, @@ -88,18 +90,32 @@ export default apiRoute<{ let url = ""; - if (isImage) { - const hash = await uploadFile(file, config); + let mediaManager: MediaBackend; - url = hash ? getUrl(hash, config) : ""; + switch (config.media.backend as MediaBackendType) { + case MediaBackendType.LOCAL: + mediaManager = new LocalMediaBackend(config); + break; + case MediaBackendType.S3: + mediaManager = new S3MediaBackend(config); + break; + default: + // TODO: Replace with logger + throw new Error("Invalid media backend"); + } + + if (isImage) { + const { uploadedFile } = await mediaManager.addFile(file); + + url = getUrl(uploadedFile.name, config); } let thumbnailUrl = ""; if (thumbnail) { - const hash = await uploadFile(thumbnail as unknown as File, config); + const { uploadedFile } = await mediaManager.addFile(thumbnail); - thumbnailUrl = hash ? getUrl(hash, config) : ""; + thumbnailUrl = getUrl(uploadedFile.name, config); } const newAttachment = await client.attachment.create({ diff --git a/server/api/api/v2/search/index.ts b/server/api/api/v2/search/index.ts index 6a4d4a9c..c327dedb 100644 --- a/server/api/api/v2/search/index.ts +++ b/server/api/api/v2/search/index.ts @@ -4,9 +4,8 @@ import { errorResponse, jsonResponse } from "@response"; import { client } from "~database/datasource"; import { statusAndUserRelations, statusToAPI } from "~database/entities/Status"; import { userRelations, userToAPI } from "~database/entities/User"; -import type { APIRouteMeta } from "~types/api"; -export const meta: APIRouteMeta = applyConfig({ +export const meta = applyConfig({ allowedMethods: ["GET"], ratelimits: { max: 10, diff --git a/utils/api.ts b/utils/api.ts index a30ea4ed..283cf136 100644 --- a/utils/api.ts +++ b/utils/api.ts @@ -1,9 +1,10 @@ -import { getConfig } from "~classes/configmanager"; +import { ConfigManager } from "config-manager"; import type { RouteHandler } from "~server/api/routes.type"; import type { APIRouteMeta } from "~types/api"; +const config = await new ConfigManager({}).getConfig(); + export const applyConfig = (routeMeta: APIRouteMeta) => { - const config = getConfig(); const newMeta = routeMeta; // Apply ratelimits from config diff --git a/utils/config.ts b/utils/config.ts deleted file mode 100644 index a672a613..00000000 --- a/utils/config.ts +++ /dev/null @@ -1,405 +0,0 @@ -import { parse } from "@iarna/toml"; -import chalk from "chalk"; - -const scanConfig = async () => { - const config = Bun.file(process.cwd() + "/config/config.toml"); - - if (!(await config.exists())) { - console.error( - `${chalk.red(`✗`)} ${chalk.bold( - "Error while reading config: " - )} Config file not found` - ); - process.exit(1); - } - - return parse(await config.text()) as ConfigType; -}; - -let config = await scanConfig(); - -export interface ConfigType { - database: { - host: string; - port: number; - username: string; - password: string; - database: string; - }; - - redis: { - queue: { - host: string; - port: number; - password: string; - database: number | null; - }; - cache: { - host: string; - port: number; - password: string; - database: number | null; - enabled: boolean; - }; - }; - - meilisearch: { - host: string; - port: number; - api_key: string; - enabled: boolean; - }; - - signups: { - tos_url: string; - rules: string[]; - registration: boolean; - }; - - oidc: { - providers: { - name: string; - id: string; - url: string; - client_id: string; - client_secret: string; - icon: string; - }[]; - }; - - http: { - base_url: string; - bind: string; - bind_port: string; - banned_ips: string[]; - }; - - instance: { - name: string; - description: string; - banner: string; - logo: string; - }; - - smtp: { - server: string; - port: number; - username: string; - password: string; - tls: boolean; - }; - - validation: { - max_displayname_size: number; - max_bio_size: number; - max_username_size: number; - max_note_size: number; - max_avatar_size: number; - max_header_size: number; - max_media_size: number; - max_media_attachments: number; - max_media_description_size: number; - max_poll_options: number; - max_poll_option_size: number; - min_poll_duration: number; - max_poll_duration: number; - - username_blacklist: string[]; - blacklist_tempmail: boolean; - email_blacklist: string[]; - url_scheme_whitelist: string[]; - - enforce_mime_types: boolean; - allowed_mime_types: string[]; - }; - - media: { - backend: string; - deduplicate_media: boolean; - conversion: { - convert_images: boolean; - convert_to: string; - }; - }; - - s3: { - endpoint: string; - access_key: string; - secret_access_key: string; - region: string; - bucket_name: string; - public_url: string; - }; - - defaults: { - visibility: string; - language: string; - avatar: string; - header: string; - }; - - email: { - send_on_report: boolean; - send_on_suspend: boolean; - send_on_unsuspend: boolean; - }; - - activitypub: { - use_tombstones: boolean; - reject_activities: string[]; - force_followers_only: string[]; - discard_reports: string[]; - discard_deletes: string[]; - discard_banners: string[]; - discard_avatars: string[]; - discard_updates: string[]; - discard_follows: string[]; - force_sensitive: string[]; - remove_media: string[]; - fetch_all_collection_members: boolean; - authorized_fetch: boolean; - }; - - filters: { - note_filters: string[]; - username_filters: string[]; - displayname_filters: string[]; - bio_filters: string[]; - emoji_filters: string[]; - }; - - logging: { - log_requests: boolean; - log_requests_verbose: boolean; - log_filters: boolean; - }; - - ratelimits: { - duration_coeff: number; - max_coeff: number; - }; - - custom_ratelimits: Record< - string, - { - duration: number; - max: number; - } - >; - [key: string]: unknown; -} - -export const configDefaults: ConfigType = { - http: { - bind: "http://0.0.0.0", - bind_port: "8000", - base_url: "http://lysand.localhost:8000", - banned_ips: [], - }, - database: { - host: "localhost", - port: 5432, - username: "postgres", - password: "postgres", - database: "lysand", - }, - redis: { - queue: { - host: "localhost", - port: 6379, - password: "", - database: 0, - }, - cache: { - host: "localhost", - port: 6379, - password: "", - database: 1, - enabled: false, - }, - }, - meilisearch: { - host: "localhost", - port: 1491, - api_key: "", - enabled: false, - }, - signups: { - tos_url: "", - rules: [], - registration: false, - }, - oidc: { - providers: [], - }, - instance: { - banner: "", - description: "", - logo: "", - name: "", - }, - smtp: { - password: "", - port: 465, - server: "", - tls: true, - username: "", - }, - media: { - backend: "local", - deduplicate_media: true, - conversion: { - convert_images: false, - convert_to: "webp", - }, - }, - email: { - send_on_report: false, - send_on_suspend: false, - send_on_unsuspend: false, - }, - s3: { - access_key: "", - bucket_name: "", - endpoint: "", - public_url: "", - region: "", - secret_access_key: "", - }, - validation: { - max_displayname_size: 50, - max_bio_size: 6000, - max_note_size: 5000, - max_avatar_size: 5_000_000, - max_header_size: 5_000_000, - max_media_size: 40_000_000, - max_media_attachments: 10, - max_media_description_size: 1000, - max_poll_options: 20, - max_poll_option_size: 500, - min_poll_duration: 60, - max_poll_duration: 1893456000, - max_username_size: 30, - - username_blacklist: [ - ".well-known", - "~", - "about", - "activities", - "api", - "auth", - "dev", - "inbox", - "internal", - "main", - "media", - "nodeinfo", - "notice", - "oauth", - "objects", - "proxy", - "push", - "registration", - "relay", - "settings", - "status", - "tag", - "users", - "web", - "search", - "mfa", - ], - - blacklist_tempmail: false, - - email_blacklist: [], - - url_scheme_whitelist: [ - "http", - "https", - "ftp", - "dat", - "dweb", - "gopher", - "hyper", - "ipfs", - "ipns", - "irc", - "xmpp", - "ircs", - "magnet", - "mailto", - "mumble", - "ssb", - ], - - enforce_mime_types: false, - allowed_mime_types: [], - }, - defaults: { - visibility: "public", - language: "en", - avatar: "", - header: "", - }, - activitypub: { - use_tombstones: true, - reject_activities: [], - force_followers_only: [], - discard_reports: [], - discard_deletes: [], - discard_banners: [], - discard_avatars: [], - force_sensitive: [], - discard_updates: [], - discard_follows: [], - remove_media: [], - fetch_all_collection_members: false, - authorized_fetch: false, - }, - filters: { - note_filters: [], - username_filters: [], - displayname_filters: [], - bio_filters: [], - emoji_filters: [], - }, - logging: { - log_requests: false, - log_requests_verbose: false, - log_filters: true, - }, - ratelimits: { - duration_coeff: 1, - max_coeff: 1, - }, - custom_ratelimits: {}, -}; - -export const getConfig = () => { - return { - ...configDefaults, - ...config, - }; -}; - -export const getHost = () => { - const url = new URL(getConfig().http.base_url); - - return url.host; -}; - -// Refresh config every 5 seconds -setInterval(() => { - scanConfig() - .then(newConfig => { - if (newConfig !== config) { - config = newConfig; - } - }) - .catch(e => { - console.error(e); - }); -}, 5000); - -export { config }; diff --git a/utils/constants.ts b/utils/constants.ts index 3e9c7686..4de00425 100644 --- a/utils/constants.ts +++ b/utils/constants.ts @@ -1,6 +1,6 @@ -import { getConfig } from "~classes/configmanager"; +import { ConfigManager } from "config-manager"; -const config = getConfig(); +const config = await new ConfigManager({}).getConfig(); export const oauthRedirectUri = (issuer: string) => `${config.http.base_url}/oauth/callback/${issuer}`;