versia-pub/server
2
0
Fork 0
mirror of https://github.com/versia-pub/server.git synced 2025-12-06 16:38:19 +01:00
Code Issues Actions Packages Projects Releases Wiki Activity

Fix media code, clean up old types

Browse source
This commit is contained in:
Jesse Wierzbinski 2024-03-10 13:57:26 -10:00
parent 852efaea50
commit 0e4d6b401c
No known key found for this signature in database
34 changed files with 137 additions and 1204 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

BIN
bun.lockb
View file

Binary file not shown.

2
bunfig.toml Normal file
View file

@ -0,0 +1,2 @@
[install.scopes]
"@jsr" = "https://npm.jsr.io"

446
classes/configmanager.ts
View file

@ -1,446 +0,0 @@
/**
* @file configmanager.ts
* @summary ConfigManager system to retrieve and modify system configuration
* @description Can read from a hand-written file, config.toml, or from a machine-saved file, config.internal.toml
* @deprecated Use the new ConfigManager class instead
* Fuses both and provides a way to retrieve individual values
*/
/* import { parse, stringify } from "@iarna/toml";
import chalk from "chalk";
import merge from "merge-deep-ts";
const scanConfig = async () => {
const config = Bun.file(process.cwd() + "/config/config.toml");
if (!(await config.exists())) {
console.error(
`${chalk.red(`✗`)} ${chalk.bold(
"Error while reading config: "
)} Config file not found`
);
process.exit(1);
}
return parse(await config.text()) as ConfigType;
}; */
// Creates the internal config with nothing in it if it doesnt exist
/* const scanInternalConfig = async () => {
const config = Bun.file(process.cwd() + "/config/config.internal.toml");
if (!(await config.exists())) {
await Bun.write(config, "");
}
return parse(await config.text()) as ConfigType;
};
let config = await scanConfig();
const internalConfig = await scanInternalConfig();
export interface ConfigType {
database: {
host: string;
port: number;
username: string;
password: string;
database: string;
};
redis: {
queue: {
host: string;
port: number;
password: string;
database: number | null;
};
cache: {
host: string;
port: number;
password: string;
database: number | null;
enabled: boolean;
};
};
meilisearch: {
host: string;
port: number;
api_key: string;
enabled: boolean;
};
signups: {
tos_url: string;
rules: string[];
registration: boolean;
};
oidc: {
providers: {
name: string;
id: string;
url: string;
client_id: string;
client_secret: string;
icon: string;
}[];
};
http: {
base_url: string;
bind: string;
bind_port: string;
banned_ips: string[];
banned_user_agents: string[];
};
instance: {
name: string;
description: string;
banner: string;
logo: string;
};
smtp: {
server: string;
port: number;
username: string;
password: string;
tls: boolean;
};
validation: {
max_displayname_size: number;
max_bio_size: number;
max_username_size: number;
max_note_size: number;
max_avatar_size: number;
max_header_size: number;
max_media_size: number;
max_media_attachments: number;
max_media_description_size: number;
max_poll_options: number;
max_poll_option_size: number;
min_poll_duration: number;
max_poll_duration: number;
username_blacklist: string[];
blacklist_tempmail: boolean;
email_blacklist: string[];
url_scheme_whitelist: string[];
enforce_mime_types: boolean;
allowed_mime_types: string[];
};
media: {
backend: string;
deduplicate_media: boolean;
conversion: {
convert_images: boolean;
convert_to: string;
};
};
s3: {
endpoint: string;
access_key: string;
secret_access_key: string;
region: string;
bucket_name: string;
public_url: string;
};
defaults: {
visibility: string;
language: string;
avatar: string;
header: string;
};
email: {
send_on_report: boolean;
send_on_suspend: boolean;
send_on_unsuspend: boolean;
};
activitypub: {
use_tombstones: boolean;
reject_activities: string[];
force_followers_only: string[];
discard_reports: string[];
discard_deletes: string[];
discard_banners: string[];
discard_avatars: string[];
discard_updates: string[];
discard_follows: string[];
force_sensitive: string[];
remove_media: string[];
fetch_all_collection_members: boolean;
authorized_fetch: boolean;
};
filters: {
note_filters: string[];
username_filters: string[];
displayname_filters: string[];
bio_filters: string[];
emoji_filters: string[];
};
logging: {
log_requests: boolean;
log_requests_verbose: boolean;
log_filters: boolean;
};
ratelimits: {
duration_coeff: number;
max_coeff: number;
};
custom_ratelimits: Record<
string,
{
duration: number;
max: number;
}
>;
[key: string]: unknown;
}
export const configDefaults: ConfigType = {
http: {
bind: "http://0.0.0.0",
bind_port: "8000",
base_url: "http://lysand.localhost:8000",
banned_ips: [],
banned_user_agents: [],
},
database: {
host: "localhost",
port: 5432,
username: "postgres",
password: "postgres",
database: "lysand",
},
redis: {
queue: {
host: "localhost",
port: 6379,
password: "",
database: 0,
},
cache: {
host: "localhost",
port: 6379,
password: "",
database: 1,
enabled: false,
},
},
meilisearch: {
host: "localhost",
port: 1491,
api_key: "",
enabled: false,
},
signups: {
tos_url: "",
rules: [],
registration: false,
},
oidc: {
providers: [],
},
instance: {
banner: "",
description: "",
logo: "",
name: "",
},
smtp: {
password: "",
port: 465,
server: "",
tls: true,
username: "",
},
media: {
backend: "local",
deduplicate_media: true,
conversion: {
convert_images: false,
convert_to: "webp",
},
},
email: {
send_on_report: false,
send_on_suspend: false,
send_on_unsuspend: false,
},
s3: {
access_key: "",
bucket_name: "",
endpoint: "",
public_url: "",
region: "",
secret_access_key: "",
},
validation: {
max_displayname_size: 50,
max_bio_size: 6000,
max_note_size: 5000,
max_avatar_size: 5_000_000,
max_header_size: 5_000_000,
max_media_size: 40_000_000,
max_media_attachments: 10,
max_media_description_size: 1000,
max_poll_options: 20,
max_poll_option_size: 500,
min_poll_duration: 60,
max_poll_duration: 1893456000,
max_username_size: 30,
username_blacklist: [
".well-known",
"~",
"about",
"activities",
"api",
"auth",
"dev",
"inbox",
"internal",
"main",
"media",
"nodeinfo",
"notice",
"oauth",
"objects",
"proxy",
"push",
"registration",
"relay",
"settings",
"status",
"tag",
"users",
"web",
"search",
"mfa",
],
blacklist_tempmail: false,
email_blacklist: [],
url_scheme_whitelist: [
"http",
"https",
"ftp",
"dat",
"dweb",
"gopher",
"hyper",
"ipfs",
"ipns",
"irc",
"xmpp",
"ircs",
"magnet",
"mailto",
"mumble",
"ssb",
],
enforce_mime_types: false,
allowed_mime_types: [],
},
defaults: {
visibility: "public",
language: "en",
avatar: "",
header: "",
},
activitypub: {
use_tombstones: true,
reject_activities: [],
force_followers_only: [],
discard_reports: [],
discard_deletes: [],
discard_banners: [],
discard_avatars: [],
force_sensitive: [],
discard_updates: [],
discard_follows: [],
remove_media: [],
fetch_all_collection_members: false,
authorized_fetch: false,
},
filters: {
note_filters: [],
username_filters: [],
displayname_filters: [],
bio_filters: [],
emoji_filters: [],
},
logging: {
log_requests: false,
log_requests_verbose: false,
log_filters: true,
},
ratelimits: {
duration_coeff: 1,
max_coeff: 1,
},
custom_ratelimits: {},
}; */
/* export const getConfig = () => {
// Deeply merge configDefaults, config and internalConfig
return merge([configDefaults, config, internalConfig]) as any as ConfigType;
};
*/
/**
* Sets the internal config
* @param newConfig Any part of ConfigType
*/
/* export const setConfig = async (newConfig: Partial<ConfigType>) => {
const newInternalConfig = merge([
internalConfig,
newConfig,
]) as any as ConfigType;
// Prepend a warning comment and write the new TOML to the file
await Bun.write(
Bun.file(process.cwd() + "/config/config.internal.toml"),
`# This file is automatically generated. Do not modify it manually.\n${stringify(
newInternalConfig as any
)}`
);
}; */
/* export const getHost = () => {
const url = new URL(getConfig().http.base_url);
return url.host;
};
*/
// Refresh config every 5 seconds
/* setInterval(() => {
scanConfig()
.then(newConfig => {
if (newConfig !== config) {
config = newConfig;
}
})
.catch(e => {
console.error(e);
});
}, 5000); */
/* export { config };
*/

273
classes/media.ts
View file

@ -1,273 +0,0 @@
import type { GetObjectCommandOutput } from "@aws-sdk/client-s3";
import {
GetObjectCommand,
PutObjectCommand,
S3Client,
} from "@aws-sdk/client-s3";
import type { ConfigType } from "~classes/configmanager";
import sharp from "sharp";
import { exists, mkdir } from "fs/promises";
class MediaBackend {
backend: string;
constructor(backend: string) {
this.backend = backend;
}
/**
* Adds media to the media backend
* @param media
* @returns The hash of the file in SHA-256 (hex format) with the file extension added to it
*/
async addMedia(media: File) {
const hash = new Bun.SHA256()
.update(await media.arrayBuffer())
.digest("hex");
return `${hash}.${media.name.split(".").pop()}`;
}
async convertMedia(media: File, config: ConfigType) {
const sharpCommand = sharp(await media.arrayBuffer());
// Rename ".jpg" files to ".jpeg" to avoid sharp errors
let name = media.name;
if (media.name.endsWith(".jpg")) {
name = media.name.replace(".jpg", ".jpeg");
}
const fileFormatToConvertTo = config.media.conversion.convert_to;
switch (fileFormatToConvertTo) {
case "png":
return new File(
[(await sharpCommand.png().toBuffer()).buffer] as any,
// Replace the file extension with PNG
name.replace(/\.[^/.]+$/, ".png"),
{
type: "image/png",
}
);
case "webp":
return new File(
[(await sharpCommand.webp().toBuffer()).buffer] as any,
// Replace the file extension with WebP
name.replace(/\.[^/.]+$/, ".webp"),
{
type: "image/webp",
}
);
case "jpeg":
return new File(
[(await sharpCommand.jpeg().toBuffer()).buffer] as any,
// Replace the file extension with JPEG
name.replace(/\.[^/.]+$/, ".jpeg"),
{
type: "image/jpeg",
}
);
case "avif":
return new File(
[(await sharpCommand.avif().toBuffer()).buffer] as any,
// Replace the file extension with AVIF
name.replace(/\.[^/.]+$/, ".avif"),
{
type: "image/avif",
}
);
// Needs special build of libvips
case "jxl":
return new File(
[(await sharpCommand.jxl().toBuffer()).buffer] as any,
// Replace the file extension with JXL
name.replace(/\.[^/.]+$/, ".jxl"),
{
type: "image/jxl",
}
);
case "heif":
return new File(
[(await sharpCommand.heif().toBuffer()).buffer] as any,
// Replace the file extension with HEIF
name.replace(/\.[^/.]+$/, ".heif"),
{
type: "image/heif",
}
);
default:
return media;
}
}
/**
* Retrieves element from media backend by hash
* @param hash The hash of the element in SHA-256 hex format
* @param extension The extension of the file
* @returns The file as a File object
*/
// eslint-disable-next-line @typescript-eslint/require-await, @typescript-eslint/no-unused-vars
async getMediaByHash(
// eslint-disable-next-line @typescript-eslint/no-unused-vars
hash: string
): Promise<File | null> {
return new File([], "test");
}
}
/**
* S3 Backend, stores files in S3
*/
export class S3Backend extends MediaBackend {
client: S3Client;
config: ConfigType;
constructor(config: ConfigType) {
super("s3");
this.config = config;
this.client = new S3Client({
endpoint: this.config.s3.endpoint,
region: this.config.s3.region || "auto",
credentials: {
accessKeyId: this.config.s3.access_key,
secretAccessKey: this.config.s3.secret_access_key,
},
});
}
async addMedia(media: File): Promise<string> {
if (this.config.media.conversion.convert_images) {
media = await this.convertMedia(media, this.config);
}
const hash = await super.addMedia(media);
if (!hash) {
throw new Error("Failed to hash file");
}
// Check if file is already present
const existingFile = await this.getMediaByHash(hash);
if (existingFile) {
// File already exists, so return the hash without uploading it
return hash;
}
const command = new PutObjectCommand({
Bucket: this.config.s3.bucket_name,
Key: hash,
Body: Buffer.from(await media.arrayBuffer()),
ContentType: media.type,
ContentLength: media.size,
Metadata: {
"x-amz-meta-original-name": media.name,
},
});
const response = await this.client.send(command);
if (response.$metadata.httpStatusCode !== 200) {
throw new Error("Failed to upload file");
}
return hash;
}
async getMediaByHash(hash: string): Promise<File | null> {
const command = new GetObjectCommand({
Bucket: this.config.s3.bucket_name,
Key: hash,
});
let response: GetObjectCommandOutput;
try {
response = await this.client.send(command);
} catch {
return null;
}
if (response.$metadata.httpStatusCode !== 200) {
throw new Error("Failed to get file");
}
const body = await response.Body?.transformToByteArray();
if (!body) {
throw new Error("Failed to get file");
}
return new File([body], hash, {
type: response.ContentType,
});
}
}
/**
* Local backend, stores files on filesystem
*/
export class LocalBackend extends MediaBackend {
config: ConfigType;
constructor(config: ConfigType) {
super("local");
this.config = config;
}
async addMedia(media: File): Promise<string> {
if (this.config.media.conversion.convert_images) {
media = await this.convertMedia(media, this.config);
}
const hash = await super.addMedia(media);
if (!(await exists(`${process.cwd()}/uploads`))) {
await mkdir(`${process.cwd()}/uploads`);
}
await Bun.write(Bun.file(`${process.cwd()}/uploads/${hash}`), media);
return hash;
}
async getMediaByHash(hash: string): Promise<File | null> {
const file = Bun.file(`${process.cwd()}/uploads/${hash}`);
if (!(await file.exists())) {
return null;
}
return new File([await file.arrayBuffer()], `${hash}`, {
type: file.type,
});
}
}
export const uploadFile = (file: File, config: ConfigType) => {
const backend = config.media.backend;
if (backend === "local") {
return new LocalBackend(config).addMedia(file);
} else if (backend === "s3") {
return new S3Backend(config).addMedia(file);
}
};
export const getFile = (
hash: string,
extension: string,
config: ConfigType
) => {
const backend = config.media.backend;
if (backend === "local") {
return new LocalBackend(config).getMediaByHash(hash);
} else if (backend === "s3") {
return new S3Backend(config).getMediaByHash(hash);
}
return null;
};

15
database/entities/Attachment.ts
View file

@ -1,5 +1,6 @@
import type { ConfigType } from "~classes/configmanager";
import type { Attachment } from "@prisma/client"; import type { Attachment } from "@prisma/client";
import type { ConfigType } from "config-manager";
import { MediaBackendType } from "media-manager";
import type { APIAsyncAttachment } from "~types/entities/async_attachment"; import type { APIAsyncAttachment } from "~types/entities/async_attachment";
import type { APIAttachment } from "~types/entities/attachment"; import type { APIAttachment } from "~types/entities/attachment";
@ -56,11 +57,13 @@ export const attachmentToAPI = (
}; };
}; };
export const getUrl = (hash: string, config: ConfigType) => { export const getUrl = (name: string, config: ConfigType) => {
if (config.media.backend === "local") { // eslint-disable-next-line @typescript-eslint/no-unsafe-enum-comparison
return `${config.http.base_url}/media/${hash}`; if (config.media.backend === MediaBackendType.LOCAL) {
} else if (config.media.backend === "s3") { return `${config.http.base_url}/media/${name}`;
return `${config.s3.public_url}/${hash}`; // eslint-disable-next-line @typescript-eslint/no-unsafe-enum-comparison
} else if (config.media.backend === MediaBackendType.S3) {
return `${config.s3.public_url}/${name}`;
} }
return ""; return "";
}; };

5
package.json
View file

@ -111,6 +111,9 @@
"semver": "^7.5.4", "semver": "^7.5.4",
"sharp": "^0.33.0-rc.2", "sharp": "^0.33.0-rc.2",
"request-parser": "file:packages/request-parser", "request-parser": "file:packages/request-parser",
"config-manager": "file:packages/config-manager" "config-manager": "file:packages/config-manager",
"cli-parser": "file:packages/cli-parser",
"log-manager": "file:packages/log-manager",
"media-manager": "file:packages/media-manager"
} }
} }

4
packages/media-manager/backends/local.ts
View file

@ -4,8 +4,8 @@ import { MediaBackend, MediaBackendType, MediaHasher } from "..";
import type { ConfigType } from "config-manager"; import type { ConfigType } from "config-manager";
export class LocalMediaBackend extends MediaBackend { export class LocalMediaBackend extends MediaBackend {
constructor(private config: ConfigType) { constructor(config: ConfigType) {
super(MediaBackendType.LOCAL); super(config, MediaBackendType.LOCAL);
} }
public async addFile(file: File) { public async addFile(file: File) {

4
packages/media-manager/backends/s3.ts
View file

@ -6,7 +6,7 @@ import type { ConfigType } from "config-manager";
export class S3MediaBackend extends MediaBackend { export class S3MediaBackend extends MediaBackend {
constructor( constructor(
private config: ConfigType, config: ConfigType,
private s3Client = new S3Client({ private s3Client = new S3Client({
endPoint: config.s3.endpoint, endPoint: config.s3.endpoint,
useSSL: true, useSSL: true,
@ -16,7 +16,7 @@ export class S3MediaBackend extends MediaBackend {
secretKey: config.s3.secret_access_key, secretKey: config.s3.secret_access_key,
}) })
) { ) {
super(MediaBackendType.S3); super(config, MediaBackendType.S3);
} }
public async addFile(file: File) { public async addFile(file: File) {

5
packages/media-manager/index.ts
View file

@ -27,7 +27,10 @@ export class MediaHasher {
} }
export class MediaBackend { export class MediaBackend {
constructor(private backend: MediaBackendType) {} constructor(
public config: ConfigType,
public backend: MediaBackendType
) {}
public getBackendType() { public getBackendType() {
return this.backend; return this.backend;

2
packages/media-manager/tests/media-backends.test.ts
View file

@ -16,7 +16,6 @@ describe("MediaBackend", () => {
let mockConfig: ConfigType; let mockConfig: ConfigType;
beforeEach(() => { beforeEach(() => {
mediaBackend = new MediaBackend(MediaBackendType.S3);
mockConfig = { mockConfig = {
media: { media: {
conversion: { conversion: {
@ -24,6 +23,7 @@ describe("MediaBackend", () => {
}, },
}, },
} as ConfigType; } as ConfigType;
mediaBackend = new MediaBackend(mockConfig, MediaBackendType.S3);
}); });
it("should initialize with correct backend type", () => { it("should initialize with correct backend type", () => {

28
server/api/api/v1/accounts/update_credentials/index.ts
View file

@ -3,12 +3,16 @@ import { userRelations, userToAPI } from "~database/entities/User";
import { apiRoute, applyConfig } from "@api"; import { apiRoute, applyConfig } from "@api";
import { sanitize } from "isomorphic-dompurify"; import { sanitize } from "isomorphic-dompurify";
import { sanitizeHtml } from "@sanitization"; import { sanitizeHtml } from "@sanitization";
import { uploadFile } from "~classes/media";
import ISO6391 from "iso-639-1"; import ISO6391 from "iso-639-1";
import { parseEmojis } from "~database/entities/Emoji"; import { parseEmojis } from "~database/entities/Emoji";
import { client } from "~database/datasource"; import { client } from "~database/datasource";
import type { APISource } from "~types/entities/source"; import type { APISource } from "~types/entities/source";
import { convertTextToHtml } from "@formatting"; import { convertTextToHtml } from "@formatting";
import { MediaBackendType } from "media-manager";
import type { MediaBackend } from "media-manager";
import { LocalMediaBackend } from "~packages/media-manager/backends/local";
import { S3MediaBackend } from "~packages/media-manager/backends/s3";
import { getUrl } from "~database/entities/Attachment";
export const meta = applyConfig({ export const meta = applyConfig({
allowedMethods: ["PATCH"], allowedMethods: ["PATCH"],
@ -69,6 +73,20 @@ export default apiRoute<{
}; };
} }
let mediaManager: MediaBackend;
switch (config.media.backend as MediaBackendType) {
case MediaBackendType.LOCAL:
mediaManager = new LocalMediaBackend(config);
break;
case MediaBackendType.S3:
mediaManager = new S3MediaBackend(config);
break;
default:
// TODO: Replace with logger
throw new Error("Invalid media backend");
}
if (display_name) { if (display_name) {
// Check if within allowed display name lengths // Check if within allowed display name lengths
if ( if (
@ -167,9 +185,9 @@ export default apiRoute<{
); );
} }
const hash = await uploadFile(avatar, config); const { uploadedFile } = await mediaManager.addFile(avatar);
user.avatar = hash || ""; user.avatar = getUrl(uploadedFile.name, config);
} }
if (header) { if (header) {
@ -181,9 +199,9 @@ export default apiRoute<{
); );
} }
const hash = await uploadFile(header, config); const { uploadedFile } = await mediaManager.addFile(header);
user.header = hash || ""; user.header = getUrl(uploadedFile.name, config);
} }
if (locked) { if (locked) {

30
server/api/api/v1/media/[id]/index.ts
View file

@ -1,11 +1,13 @@
import { apiRoute, applyConfig } from "@api"; import { apiRoute, applyConfig } from "@api";
import { errorResponse, jsonResponse } from "@response"; import { errorResponse, jsonResponse } from "@response";
import { client } from "~database/datasource"; import { client } from "~database/datasource";
import type { APIRouteMeta } from "~types/api";
import { uploadFile } from "~classes/media";
import { attachmentToAPI, getUrl } from "~database/entities/Attachment"; import { attachmentToAPI, getUrl } from "~database/entities/Attachment";
import type { MediaBackend } from "media-manager";
import { MediaBackendType } from "media-manager";
import { LocalMediaBackend } from "~packages/media-manager/backends/local";
import { S3MediaBackend } from "~packages/media-manager/backends/s3";
export const meta: APIRouteMeta = applyConfig({ export const meta = applyConfig({
allowedMethods: ["GET", "PUT"], allowedMethods: ["GET", "PUT"],
ratelimits: { ratelimits: {
max: 10, max: 10,
@ -61,13 +63,23 @@ export default apiRoute<{
let thumbnailUrl = attachment.thumbnail_url; let thumbnailUrl = attachment.thumbnail_url;
if (thumbnail) { let mediaManager: MediaBackend;
const hash = await uploadFile(
thumbnail as unknown as File,
config
);
thumbnailUrl = hash ? getUrl(hash, config) : ""; switch (config.media.backend as MediaBackendType) {
case MediaBackendType.LOCAL:
mediaManager = new LocalMediaBackend(config);
break;
case MediaBackendType.S3:
mediaManager = new S3MediaBackend(config);
break;
default:
// TODO: Replace with logger
throw new Error("Invalid media backend");
}
if (thumbnail) {
const { uploadedFile } = await mediaManager.addFile(thumbnail);
thumbnailUrl = getUrl(uploadedFile.name, config);
} }
const descriptionText = description || attachment.description; const descriptionText = description || attachment.description;

30
server/api/api/v1/media/index.ts
View file

@ -2,12 +2,14 @@ import { apiRoute, applyConfig } from "@api";
import { errorResponse, jsonResponse } from "@response"; import { errorResponse, jsonResponse } from "@response";
import { client } from "~database/datasource"; import { client } from "~database/datasource";
import { encode } from "blurhash"; import { encode } from "blurhash";
import type { APIRouteMeta } from "~types/api";
import sharp from "sharp"; import sharp from "sharp";
import { uploadFile } from "~classes/media";
import { attachmentToAPI, getUrl } from "~database/entities/Attachment"; import { attachmentToAPI, getUrl } from "~database/entities/Attachment";
import { MediaBackendType } from "media-manager";
import type { MediaBackend } from "media-manager";
import { LocalMediaBackend } from "~packages/media-manager/backends/local";
import { S3MediaBackend } from "~packages/media-manager/backends/s3";
export const meta: APIRouteMeta = applyConfig({ export const meta = applyConfig({
allowedMethods: ["POST"], allowedMethods: ["POST"],
ratelimits: { ratelimits: {
max: 10, max: 10,
@ -88,16 +90,30 @@ export default apiRoute<{
let url = ""; let url = "";
const hash = await uploadFile(file, config); let mediaManager: MediaBackend;
url = hash ? getUrl(hash, config) : ""; switch (config.media.backend as MediaBackendType) {
case MediaBackendType.LOCAL:
mediaManager = new LocalMediaBackend(config);
break;
case MediaBackendType.S3:
mediaManager = new S3MediaBackend(config);
break;
default:
// TODO: Replace with logger
throw new Error("Invalid media backend");
}
const { uploadedFile } = await mediaManager.addFile(file);
url = getUrl(uploadedFile.name, config);
let thumbnailUrl = ""; let thumbnailUrl = "";
if (thumbnail) { if (thumbnail) {
const hash = await uploadFile(thumbnail as unknown as File, config); const { uploadedFile } = await mediaManager.addFile(thumbnail);
thumbnailUrl = hash ? getUrl(hash, config) : ""; thumbnailUrl = getUrl(uploadedFile.name, config);
} }
const newAttachment = await client.attachment.create({ const newAttachment = await client.attachment.create({

3
server/api/api/v1/profile/avatar.ts
View file

@ -2,9 +2,8 @@ import { apiRoute, applyConfig } from "@api";
import { errorResponse, jsonResponse } from "@response"; import { errorResponse, jsonResponse } from "@response";
import { client } from "~database/datasource"; import { client } from "~database/datasource";
import { userRelations, userToAPI } from "~database/entities/User"; import { userRelations, userToAPI } from "~database/entities/User";
import type { APIRouteMeta } from "~types/api";
export const meta: APIRouteMeta = applyConfig({ export const meta = applyConfig({
allowedMethods: ["DELETE"], allowedMethods: ["DELETE"],
ratelimits: { ratelimits: {
max: 10, max: 10,

3
server/api/api/v1/profile/header.ts
View file

@ -2,9 +2,8 @@ import { apiRoute, applyConfig } from "@api";
import { errorResponse, jsonResponse } from "@response"; import { errorResponse, jsonResponse } from "@response";
import { client } from "~database/datasource"; import { client } from "~database/datasource";
import { userRelations, userToAPI } from "~database/entities/User"; import { userRelations, userToAPI } from "~database/entities/User";
import type { APIRouteMeta } from "~types/api";
export const meta: APIRouteMeta = applyConfig({ export const meta = applyConfig({
allowedMethods: ["DELETE"], allowedMethods: ["DELETE"],
ratelimits: { ratelimits: {
max: 10, max: 10,

3
server/api/api/v1/statuses/[id]/context.ts
View file

@ -7,9 +7,8 @@ import {
statusAndUserRelations, statusAndUserRelations,
statusToAPI, statusToAPI,
} from "~database/entities/Status"; } from "~database/entities/Status";
import type { APIRouteMeta } from "~types/api";
export const meta: APIRouteMeta = applyConfig({ export const meta = applyConfig({
allowedMethods: ["GET"], allowedMethods: ["GET"],
ratelimits: { ratelimits: {
max: 8, max: 8,

3
server/api/api/v1/statuses/[id]/favourite.ts
View file

@ -8,10 +8,9 @@ import {
statusAndUserRelations, statusAndUserRelations,
statusToAPI, statusToAPI,
} from "~database/entities/Status"; } from "~database/entities/Status";
import type { APIRouteMeta } from "~types/api";
import type { APIStatus } from "~types/entities/status"; import type { APIStatus } from "~types/entities/status";
export const meta: APIRouteMeta = applyConfig({ export const meta = applyConfig({
allowedMethods: ["POST"], allowedMethods: ["POST"],
ratelimits: { ratelimits: {
max: 100, max: 100,

3
server/api/api/v1/statuses/[id]/favourited_by.ts
View file

@ -6,9 +6,8 @@ import {
statusAndUserRelations, statusAndUserRelations,
} from "~database/entities/Status"; } from "~database/entities/Status";
import { userRelations, userToAPI } from "~database/entities/User"; import { userRelations, userToAPI } from "~database/entities/User";
import type { APIRouteMeta } from "~types/api";
export const meta: APIRouteMeta = applyConfig({ export const meta = applyConfig({
allowedMethods: ["GET"], allowedMethods: ["GET"],
ratelimits: { ratelimits: {
max: 100, max: 100,

3
server/api/api/v1/statuses/[id]/index.ts
View file

@ -9,9 +9,8 @@ import {
statusAndUserRelations, statusAndUserRelations,
statusToAPI, statusToAPI,
} from "~database/entities/Status"; } from "~database/entities/Status";
import type { APIRouteMeta } from "~types/api";
export const meta: APIRouteMeta = applyConfig({ export const meta = applyConfig({
allowedMethods: ["GET", "DELETE", "PUT"], allowedMethods: ["GET", "DELETE", "PUT"],
ratelimits: { ratelimits: {
max: 100, max: 100,

3
server/api/api/v1/statuses/[id]/pin.ts
View file

@ -3,9 +3,8 @@ import { apiRoute, applyConfig } from "@api";
import { errorResponse, jsonResponse } from "@response"; import { errorResponse, jsonResponse } from "@response";
import { client } from "~database/datasource"; import { client } from "~database/datasource";
import { statusAndUserRelations, statusToAPI } from "~database/entities/Status"; import { statusAndUserRelations, statusToAPI } from "~database/entities/Status";
import type { APIRouteMeta } from "~types/api";
export const meta: APIRouteMeta = applyConfig({ export const meta = applyConfig({
allowedMethods: ["POST"], allowedMethods: ["POST"],
ratelimits: { ratelimits: {
max: 100, max: 100,

3
server/api/api/v1/statuses/[id]/reblog.ts
View file

@ -8,9 +8,8 @@ import {
statusToAPI, statusToAPI,
} from "~database/entities/Status"; } from "~database/entities/Status";
import { type UserWithRelations } from "~database/entities/User"; import { type UserWithRelations } from "~database/entities/User";
import type { APIRouteMeta } from "~types/api";
export const meta: APIRouteMeta = applyConfig({ export const meta = applyConfig({
allowedMethods: ["POST"], allowedMethods: ["POST"],
ratelimits: { ratelimits: {
max: 100, max: 100,

3
server/api/api/v1/statuses/[id]/reblogged_by.ts
View file

@ -6,9 +6,8 @@ import {
statusAndUserRelations, statusAndUserRelations,
} from "~database/entities/Status"; } from "~database/entities/Status";
import { userRelations, userToAPI } from "~database/entities/User"; import { userRelations, userToAPI } from "~database/entities/User";
import type { APIRouteMeta } from "~types/api";
export const meta: APIRouteMeta = applyConfig({ export const meta = applyConfig({
allowedMethods: ["GET"], allowedMethods: ["GET"],
ratelimits: { ratelimits: {
max: 100, max: 100,

3
server/api/api/v1/statuses/[id]/source.ts
View file

@ -5,9 +5,8 @@ import {
isViewableByUser, isViewableByUser,
statusAndUserRelations, statusAndUserRelations,
} from "~database/entities/Status"; } from "~database/entities/Status";
import type { APIRouteMeta } from "~types/api";
export const meta: APIRouteMeta = applyConfig({ export const meta = applyConfig({
allowedMethods: ["GET"], allowedMethods: ["GET"],
ratelimits: { ratelimits: {
max: 100, max: 100,

3
server/api/api/v1/statuses/[id]/unfavourite.ts
View file

@ -8,10 +8,9 @@ import {
statusAndUserRelations, statusAndUserRelations,
statusToAPI, statusToAPI,
} from "~database/entities/Status"; } from "~database/entities/Status";
import type { APIRouteMeta } from "~types/api";
import type { APIStatus } from "~types/entities/status"; import type { APIStatus } from "~types/entities/status";
export const meta: APIRouteMeta = applyConfig({ export const meta = applyConfig({
allowedMethods: ["POST"], allowedMethods: ["POST"],
ratelimits: { ratelimits: {
max: 100, max: 100,

3
server/api/api/v1/statuses/[id]/unpin.ts
View file

@ -2,9 +2,8 @@ import { apiRoute, applyConfig } from "@api";
import { errorResponse, jsonResponse } from "@response"; import { errorResponse, jsonResponse } from "@response";
import { client } from "~database/datasource"; import { client } from "~database/datasource";
import { statusAndUserRelations, statusToAPI } from "~database/entities/Status"; import { statusAndUserRelations, statusToAPI } from "~database/entities/Status";
import type { APIRouteMeta } from "~types/api";
export const meta: APIRouteMeta = applyConfig({ export const meta = applyConfig({
allowedMethods: ["POST"], allowedMethods: ["POST"],
ratelimits: { ratelimits: {
max: 100, max: 100,

3
server/api/api/v1/statuses/[id]/unreblog.ts
View file

@ -6,10 +6,9 @@ import {
statusAndUserRelations, statusAndUserRelations,
statusToAPI, statusToAPI,
} from "~database/entities/Status"; } from "~database/entities/Status";
import type { APIRouteMeta } from "~types/api";
import type { APIStatus } from "~types/entities/status"; import type { APIStatus } from "~types/entities/status";
export const meta: APIRouteMeta = applyConfig({ export const meta = applyConfig({
allowedMethods: ["POST"], allowedMethods: ["POST"],
ratelimits: { ratelimits: {
max: 100, max: 100,

3
server/api/api/v1/statuses/index.ts
View file

@ -11,9 +11,8 @@ import {
statusToAPI, statusToAPI,
} from "~database/entities/Status"; } from "~database/entities/Status";
import type { UserWithRelations } from "~database/entities/User"; import type { UserWithRelations } from "~database/entities/User";
import type { APIRouteMeta } from "~types/api";
export const meta: APIRouteMeta = applyConfig({ export const meta = applyConfig({
allowedMethods: ["POST"], allowedMethods: ["POST"],
ratelimits: { ratelimits: {
max: 300, max: 300,

3
server/api/api/v1/timelines/home.ts
View file

@ -2,9 +2,8 @@ import { apiRoute, applyConfig } from "@api";
import { errorResponse, jsonResponse } from "@response"; import { errorResponse, jsonResponse } from "@response";
import { client } from "~database/datasource"; import { client } from "~database/datasource";
import { statusAndUserRelations, statusToAPI } from "~database/entities/Status"; import { statusAndUserRelations, statusToAPI } from "~database/entities/Status";
import type { APIRouteMeta } from "~types/api";
export const meta: APIRouteMeta = applyConfig({ export const meta = applyConfig({
allowedMethods: ["GET"], allowedMethods: ["GET"],
ratelimits: { ratelimits: {
max: 200, max: 200,

3
server/api/api/v1/timelines/public.ts
View file

@ -2,9 +2,8 @@ import { apiRoute, applyConfig } from "@api";
import { errorResponse, jsonResponse } from "@response"; import { errorResponse, jsonResponse } from "@response";
import { client } from "~database/datasource"; import { client } from "~database/datasource";
import { statusAndUserRelations, statusToAPI } from "~database/entities/Status"; import { statusAndUserRelations, statusToAPI } from "~database/entities/Status";
import type { APIRouteMeta } from "~types/api";
export const meta: APIRouteMeta = applyConfig({ export const meta = applyConfig({
allowedMethods: ["GET"], allowedMethods: ["GET"],
ratelimits: { ratelimits: {
max: 200, max: 200,

32
server/api/api/v2/media/index.ts
View file

@ -2,12 +2,14 @@ import { apiRoute, applyConfig } from "@api";
import { errorResponse, jsonResponse } from "@response"; import { errorResponse, jsonResponse } from "@response";
import { client } from "~database/datasource"; import { client } from "~database/datasource";
import { encode } from "blurhash"; import { encode } from "blurhash";
import type { APIRouteMeta } from "~types/api";
import sharp from "sharp"; import sharp from "sharp";
import { uploadFile } from "~classes/media";
import { attachmentToAPI, getUrl } from "~database/entities/Attachment"; import { attachmentToAPI, getUrl } from "~database/entities/Attachment";
import type { MediaBackend } from "media-manager";
import { MediaBackendType } from "media-manager";
import { LocalMediaBackend } from "~packages/media-manager/backends/local";
import { S3MediaBackend } from "~packages/media-manager/backends/s3";
export const meta: APIRouteMeta = applyConfig({ export const meta = applyConfig({
allowedMethods: ["POST"], allowedMethods: ["POST"],
ratelimits: { ratelimits: {
max: 10, max: 10,
@ -88,18 +90,32 @@ export default apiRoute<{
let url = ""; let url = "";
if (isImage) { let mediaManager: MediaBackend;
const hash = await uploadFile(file, config);
url = hash ? getUrl(hash, config) : ""; switch (config.media.backend as MediaBackendType) {
case MediaBackendType.LOCAL:
mediaManager = new LocalMediaBackend(config);
break;
case MediaBackendType.S3:
mediaManager = new S3MediaBackend(config);
break;
default:
// TODO: Replace with logger
throw new Error("Invalid media backend");
}
if (isImage) {
const { uploadedFile } = await mediaManager.addFile(file);
url = getUrl(uploadedFile.name, config);
} }
let thumbnailUrl = ""; let thumbnailUrl = "";
if (thumbnail) { if (thumbnail) {
const hash = await uploadFile(thumbnail as unknown as File, config); const { uploadedFile } = await mediaManager.addFile(thumbnail);
thumbnailUrl = hash ? getUrl(hash, config) : ""; thumbnailUrl = getUrl(uploadedFile.name, config);
} }
const newAttachment = await client.attachment.create({ const newAttachment = await client.attachment.create({

3
server/api/api/v2/search/index.ts
View file

@ -4,9 +4,8 @@ import { errorResponse, jsonResponse } from "@response";
import { client } from "~database/datasource"; import { client } from "~database/datasource";
import { statusAndUserRelations, statusToAPI } from "~database/entities/Status"; import { statusAndUserRelations, statusToAPI } from "~database/entities/Status";
import { userRelations, userToAPI } from "~database/entities/User"; import { userRelations, userToAPI } from "~database/entities/User";
import type { APIRouteMeta } from "~types/api";
export const meta: APIRouteMeta = applyConfig({ export const meta = applyConfig({
allowedMethods: ["GET"], allowedMethods: ["GET"],
ratelimits: { ratelimits: {
max: 10, max: 10,

5
utils/api.ts
View file

@ -1,9 +1,10 @@
import { getConfig } from "~classes/configmanager"; import { ConfigManager } from "config-manager";
import type { RouteHandler } from "~server/api/routes.type"; import type { RouteHandler } from "~server/api/routes.type";
import type { APIRouteMeta } from "~types/api"; import type { APIRouteMeta } from "~types/api";
const config = await new ConfigManager({}).getConfig();
export const applyConfig = (routeMeta: APIRouteMeta) => { export const applyConfig = (routeMeta: APIRouteMeta) => {
const config = getConfig();
const newMeta = routeMeta; const newMeta = routeMeta;
// Apply ratelimits from config // Apply ratelimits from config

405
utils/config.ts
View file

@ -1,405 +0,0 @@
import { parse } from "@iarna/toml";
import chalk from "chalk";
const scanConfig = async () => {
const config = Bun.file(process.cwd() + "/config/config.toml");
if (!(await config.exists())) {
console.error(
`${chalk.red(`✗`)} ${chalk.bold(
"Error while reading config: "
)} Config file not found`
);
process.exit(1);
}
return parse(await config.text()) as ConfigType;
};
let config = await scanConfig();
export interface ConfigType {
database: {
host: string;
port: number;
username: string;
password: string;
database: string;
};
redis: {
queue: {
host: string;
port: number;
password: string;
database: number | null;
};
cache: {
host: string;
port: number;
password: string;
database: number | null;
enabled: boolean;
};
};
meilisearch: {
host: string;
port: number;
api_key: string;
enabled: boolean;
};
signups: {
tos_url: string;
rules: string[];
registration: boolean;
};
oidc: {
providers: {
name: string;
id: string;
url: string;
client_id: string;
client_secret: string;
icon: string;
}[];
};
http: {
base_url: string;
bind: string;
bind_port: string;
banned_ips: string[];
};
instance: {
name: string;
description: string;
banner: string;
logo: string;
};
smtp: {
server: string;
port: number;
username: string;
password: string;
tls: boolean;
};
validation: {
max_displayname_size: number;
max_bio_size: number;
max_username_size: number;
max_note_size: number;
max_avatar_size: number;
max_header_size: number;
max_media_size: number;
max_media_attachments: number;
max_media_description_size: number;
max_poll_options: number;
max_poll_option_size: number;
min_poll_duration: number;
max_poll_duration: number;
username_blacklist: string[];
blacklist_tempmail: boolean;
email_blacklist: string[];
url_scheme_whitelist: string[];
enforce_mime_types: boolean;
allowed_mime_types: string[];
};
media: {
backend: string;
deduplicate_media: boolean;
conversion: {
convert_images: boolean;
convert_to: string;
};
};
s3: {
endpoint: string;
access_key: string;
secret_access_key: string;
region: string;
bucket_name: string;
public_url: string;
};
defaults: {
visibility: string;
language: string;
avatar: string;
header: string;
};
email: {
send_on_report: boolean;
send_on_suspend: boolean;
send_on_unsuspend: boolean;
};
activitypub: {
use_tombstones: boolean;
reject_activities: string[];
force_followers_only: string[];
discard_reports: string[];
discard_deletes: string[];
discard_banners: string[];
discard_avatars: string[];
discard_updates: string[];
discard_follows: string[];
force_sensitive: string[];
remove_media: string[];
fetch_all_collection_members: boolean;
authorized_fetch: boolean;
};
filters: {
note_filters: string[];
username_filters: string[];
displayname_filters: string[];
bio_filters: string[];
emoji_filters: string[];
};
logging: {
log_requests: boolean;
log_requests_verbose: boolean;
log_filters: boolean;
};
ratelimits: {
duration_coeff: number;
max_coeff: number;
};
custom_ratelimits: Record<
string,
{
duration: number;
max: number;
}
>;
[key: string]: unknown;
}
export const configDefaults: ConfigType = {
http: {
bind: "http://0.0.0.0",
bind_port: "8000",
base_url: "http://lysand.localhost:8000",
banned_ips: [],
},
database: {
host: "localhost",
port: 5432,
username: "postgres",
password: "postgres",
database: "lysand",
},
redis: {
queue: {
host: "localhost",
port: 6379,
password: "",
database: 0,
},
cache: {
host: "localhost",
port: 6379,
password: "",
database: 1,
enabled: false,
},
},
meilisearch: {
host: "localhost",
port: 1491,
api_key: "",
enabled: false,
},
signups: {
tos_url: "",
rules: [],
registration: false,
},
oidc: {
providers: [],
},
instance: {
banner: "",
description: "",
logo: "",
name: "",
},
smtp: {
password: "",
port: 465,
server: "",
tls: true,
username: "",
},
media: {
backend: "local",
deduplicate_media: true,
conversion: {
convert_images: false,
convert_to: "webp",
},
},
email: {
send_on_report: false,
send_on_suspend: false,
send_on_unsuspend: false,
},
s3: {
access_key: "",
bucket_name: "",
endpoint: "",
public_url: "",
region: "",
secret_access_key: "",
},
validation: {
max_displayname_size: 50,
max_bio_size: 6000,
max_note_size: 5000,
max_avatar_size: 5_000_000,
max_header_size: 5_000_000,
max_media_size: 40_000_000,
max_media_attachments: 10,
max_media_description_size: 1000,
max_poll_options: 20,
max_poll_option_size: 500,
min_poll_duration: 60,
max_poll_duration: 1893456000,
max_username_size: 30,
username_blacklist: [
".well-known",
"~",
"about",
"activities",
"api",
"auth",
"dev",
"inbox",
"internal",
"main",
"media",
"nodeinfo",
"notice",
"oauth",
"objects",
"proxy",
"push",
"registration",
"relay",
"settings",
"status",
"tag",
"users",
"web",
"search",
"mfa",
],
blacklist_tempmail: false,
email_blacklist: [],
url_scheme_whitelist: [
"http",
"https",
"ftp",
"dat",
"dweb",
"gopher",
"hyper",
"ipfs",
"ipns",
"irc",
"xmpp",
"ircs",
"magnet",
"mailto",
"mumble",
"ssb",
],
enforce_mime_types: false,
allowed_mime_types: [],
},
defaults: {
visibility: "public",
language: "en",
avatar: "",
header: "",
},
activitypub: {
use_tombstones: true,
reject_activities: [],
force_followers_only: [],
discard_reports: [],
discard_deletes: [],
discard_banners: [],
discard_avatars: [],
force_sensitive: [],
discard_updates: [],
discard_follows: [],
remove_media: [],
fetch_all_collection_members: false,
authorized_fetch: false,
},
filters: {
note_filters: [],
username_filters: [],
displayname_filters: [],
bio_filters: [],
emoji_filters: [],
},
logging: {
log_requests: false,
log_requests_verbose: false,
log_filters: true,
},
ratelimits: {
duration_coeff: 1,
max_coeff: 1,
},
custom_ratelimits: {},
};
export const getConfig = () => {
return {
...configDefaults,
...config,
};
};
export const getHost = () => {
const url = new URL(getConfig().http.base_url);
return url.host;
};
// Refresh config every 5 seconds
setInterval(() => {
scanConfig()
.then(newConfig => {
if (newConfig !== config) {
config = newConfig;
}
})
.catch(e => {
console.error(e);
});
}, 5000);
export { config };

4
utils/constants.ts
View file

@ -1,6 +1,6 @@
import { getConfig } from "~classes/configmanager"; import { ConfigManager } from "config-manager";
const config = getConfig(); const config = await new ConfigManager({}).getConfig();
export const oauthRedirectUri = (issuer: string) => export const oauthRedirectUri = (issuer: string) =>
`${config.http.base_url}/oauth/callback/${issuer}`; `${config.http.base_url}/oauth/callback/${issuer}`;