mirror of
https://github.com/versia-pub/server.git
synced 2026-03-13 05:49:16 +01:00
Fix media code, clean up old types
This commit is contained in:
Jesse Wierzbinski
parent
852efaea50
commit
0e4d6b401c
34 changed files with 137 additions and 1204 deletions
|
|
@ -1,446 +0,0 @@
|
|||
/**
|
||||
* @file configmanager.ts
|
||||
* @summary ConfigManager system to retrieve and modify system configuration
|
||||
* @description Can read from a hand-written file, config.toml, or from a machine-saved file, config.internal.toml
|
||||
* @deprecated Use the new ConfigManager class instead
|
||||
* Fuses both and provides a way to retrieve individual values
|
||||
*/
|
||||
|
||||
/* import { parse, stringify } from "@iarna/toml";
|
||||
import chalk from "chalk";
|
||||
import merge from "merge-deep-ts";
|
||||
|
||||
const scanConfig = async () => {
|
||||
const config = Bun.file(process.cwd() + "/config/config.toml");
|
||||
|
||||
if (!(await config.exists())) {
|
||||
console.error(
|
||||
`${chalk.red(`✗`)} ${chalk.bold(
|
||||
"Error while reading config: "
|
||||
)} Config file not found`
|
||||
);
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
return parse(await config.text()) as ConfigType;
|
||||
}; */
|
||||
|
||||
// Creates the internal config with nothing in it if it doesnt exist
|
||||
/* const scanInternalConfig = async () => {
|
||||
const config = Bun.file(process.cwd() + "/config/config.internal.toml");
|
||||
|
||||
if (!(await config.exists())) {
|
||||
await Bun.write(config, "");
|
||||
}
|
||||
|
||||
return parse(await config.text()) as ConfigType;
|
||||
};
|
||||
|
||||
let config = await scanConfig();
|
||||
const internalConfig = await scanInternalConfig();
|
||||
|
||||
export interface ConfigType {
|
||||
database: {
|
||||
host: string;
|
||||
port: number;
|
||||
username: string;
|
||||
password: string;
|
||||
database: string;
|
||||
};
|
||||
|
||||
redis: {
|
||||
queue: {
|
||||
host: string;
|
||||
port: number;
|
||||
password: string;
|
||||
database: number | null;
|
||||
};
|
||||
cache: {
|
||||
host: string;
|
||||
port: number;
|
||||
password: string;
|
||||
database: number | null;
|
||||
enabled: boolean;
|
||||
};
|
||||
};
|
||||
|
||||
meilisearch: {
|
||||
host: string;
|
||||
port: number;
|
||||
api_key: string;
|
||||
enabled: boolean;
|
||||
};
|
||||
|
||||
signups: {
|
||||
tos_url: string;
|
||||
rules: string[];
|
||||
registration: boolean;
|
||||
};
|
||||
|
||||
oidc: {
|
||||
providers: {
|
||||
name: string;
|
||||
id: string;
|
||||
url: string;
|
||||
client_id: string;
|
||||
client_secret: string;
|
||||
icon: string;
|
||||
}[];
|
||||
};
|
||||
|
||||
http: {
|
||||
base_url: string;
|
||||
bind: string;
|
||||
bind_port: string;
|
||||
banned_ips: string[];
|
||||
banned_user_agents: string[];
|
||||
};
|
||||
|
||||
instance: {
|
||||
name: string;
|
||||
description: string;
|
||||
banner: string;
|
||||
logo: string;
|
||||
};
|
||||
|
||||
smtp: {
|
||||
server: string;
|
||||
port: number;
|
||||
username: string;
|
||||
password: string;
|
||||
tls: boolean;
|
||||
};
|
||||
|
||||
validation: {
|
||||
max_displayname_size: number;
|
||||
max_bio_size: number;
|
||||
max_username_size: number;
|
||||
max_note_size: number;
|
||||
max_avatar_size: number;
|
||||
max_header_size: number;
|
||||
max_media_size: number;
|
||||
max_media_attachments: number;
|
||||
max_media_description_size: number;
|
||||
max_poll_options: number;
|
||||
max_poll_option_size: number;
|
||||
min_poll_duration: number;
|
||||
max_poll_duration: number;
|
||||
|
||||
username_blacklist: string[];
|
||||
blacklist_tempmail: boolean;
|
||||
email_blacklist: string[];
|
||||
url_scheme_whitelist: string[];
|
||||
|
||||
enforce_mime_types: boolean;
|
||||
allowed_mime_types: string[];
|
||||
};
|
||||
|
||||
media: {
|
||||
backend: string;
|
||||
deduplicate_media: boolean;
|
||||
conversion: {
|
||||
convert_images: boolean;
|
||||
convert_to: string;
|
||||
};
|
||||
};
|
||||
|
||||
s3: {
|
||||
endpoint: string;
|
||||
access_key: string;
|
||||
secret_access_key: string;
|
||||
region: string;
|
||||
bucket_name: string;
|
||||
public_url: string;
|
||||
};
|
||||
|
||||
defaults: {
|
||||
visibility: string;
|
||||
language: string;
|
||||
avatar: string;
|
||||
header: string;
|
||||
};
|
||||
|
||||
email: {
|
||||
send_on_report: boolean;
|
||||
send_on_suspend: boolean;
|
||||
send_on_unsuspend: boolean;
|
||||
};
|
||||
|
||||
activitypub: {
|
||||
use_tombstones: boolean;
|
||||
reject_activities: string[];
|
||||
force_followers_only: string[];
|
||||
discard_reports: string[];
|
||||
discard_deletes: string[];
|
||||
discard_banners: string[];
|
||||
discard_avatars: string[];
|
||||
discard_updates: string[];
|
||||
discard_follows: string[];
|
||||
force_sensitive: string[];
|
||||
remove_media: string[];
|
||||
fetch_all_collection_members: boolean;
|
||||
authorized_fetch: boolean;
|
||||
};
|
||||
|
||||
filters: {
|
||||
note_filters: string[];
|
||||
username_filters: string[];
|
||||
displayname_filters: string[];
|
||||
bio_filters: string[];
|
||||
emoji_filters: string[];
|
||||
};
|
||||
|
||||
logging: {
|
||||
log_requests: boolean;
|
||||
log_requests_verbose: boolean;
|
||||
log_filters: boolean;
|
||||
};
|
||||
|
||||
ratelimits: {
|
||||
duration_coeff: number;
|
||||
max_coeff: number;
|
||||
};
|
||||
|
||||
custom_ratelimits: Record<
|
||||
string,
|
||||
{
|
||||
duration: number;
|
||||
max: number;
|
||||
}
|
||||
>;
|
||||
[key: string]: unknown;
|
||||
}
|
||||
|
||||
export const configDefaults: ConfigType = {
|
||||
http: {
|
||||
bind: "http://0.0.0.0",
|
||||
bind_port: "8000",
|
||||
base_url: "http://lysand.localhost:8000",
|
||||
banned_ips: [],
|
||||
banned_user_agents: [],
|
||||
},
|
||||
database: {
|
||||
host: "localhost",
|
||||
port: 5432,
|
||||
username: "postgres",
|
||||
password: "postgres",
|
||||
database: "lysand",
|
||||
},
|
||||
redis: {
|
||||
queue: {
|
||||
host: "localhost",
|
||||
port: 6379,
|
||||
password: "",
|
||||
database: 0,
|
||||
},
|
||||
cache: {
|
||||
host: "localhost",
|
||||
port: 6379,
|
||||
password: "",
|
||||
database: 1,
|
||||
enabled: false,
|
||||
},
|
||||
},
|
||||
meilisearch: {
|
||||
host: "localhost",
|
||||
port: 1491,
|
||||
api_key: "",
|
||||
enabled: false,
|
||||
},
|
||||
signups: {
|
||||
tos_url: "",
|
||||
rules: [],
|
||||
registration: false,
|
||||
},
|
||||
oidc: {
|
||||
providers: [],
|
||||
},
|
||||
instance: {
|
||||
banner: "",
|
||||
description: "",
|
||||
logo: "",
|
||||
name: "",
|
||||
},
|
||||
smtp: {
|
||||
password: "",
|
||||
port: 465,
|
||||
server: "",
|
||||
tls: true,
|
||||
username: "",
|
||||
},
|
||||
media: {
|
||||
backend: "local",
|
||||
deduplicate_media: true,
|
||||
conversion: {
|
||||
convert_images: false,
|
||||
convert_to: "webp",
|
||||
},
|
||||
},
|
||||
email: {
|
||||
send_on_report: false,
|
||||
send_on_suspend: false,
|
||||
send_on_unsuspend: false,
|
||||
},
|
||||
s3: {
|
||||
access_key: "",
|
||||
bucket_name: "",
|
||||
endpoint: "",
|
||||
public_url: "",
|
||||
region: "",
|
||||
secret_access_key: "",
|
||||
},
|
||||
validation: {
|
||||
max_displayname_size: 50,
|
||||
max_bio_size: 6000,
|
||||
max_note_size: 5000,
|
||||
max_avatar_size: 5_000_000,
|
||||
max_header_size: 5_000_000,
|
||||
max_media_size: 40_000_000,
|
||||
max_media_attachments: 10,
|
||||
max_media_description_size: 1000,
|
||||
max_poll_options: 20,
|
||||
max_poll_option_size: 500,
|
||||
min_poll_duration: 60,
|
||||
max_poll_duration: 1893456000,
|
||||
max_username_size: 30,
|
||||
|
||||
username_blacklist: [
|
||||
".well-known",
|
||||
"~",
|
||||
"about",
|
||||
"activities",
|
||||
"api",
|
||||
"auth",
|
||||
"dev",
|
||||
"inbox",
|
||||
"internal",
|
||||
"main",
|
||||
"media",
|
||||
"nodeinfo",
|
||||
"notice",
|
||||
"oauth",
|
||||
"objects",
|
||||
"proxy",
|
||||
"push",
|
||||
"registration",
|
||||
"relay",
|
||||
"settings",
|
||||
"status",
|
||||
"tag",
|
||||
"users",
|
||||
"web",
|
||||
"search",
|
||||
"mfa",
|
||||
],
|
||||
|
||||
blacklist_tempmail: false,
|
||||
|
||||
email_blacklist: [],
|
||||
|
||||
url_scheme_whitelist: [
|
||||
"http",
|
||||
"https",
|
||||
"ftp",
|
||||
"dat",
|
||||
"dweb",
|
||||
"gopher",
|
||||
"hyper",
|
||||
"ipfs",
|
||||
"ipns",
|
||||
"irc",
|
||||
"xmpp",
|
||||
"ircs",
|
||||
"magnet",
|
||||
"mailto",
|
||||
"mumble",
|
||||
"ssb",
|
||||
],
|
||||
|
||||
enforce_mime_types: false,
|
||||
allowed_mime_types: [],
|
||||
},
|
||||
defaults: {
|
||||
visibility: "public",
|
||||
language: "en",
|
||||
avatar: "",
|
||||
header: "",
|
||||
},
|
||||
activitypub: {
|
||||
use_tombstones: true,
|
||||
reject_activities: [],
|
||||
force_followers_only: [],
|
||||
discard_reports: [],
|
||||
discard_deletes: [],
|
||||
discard_banners: [],
|
||||
discard_avatars: [],
|
||||
force_sensitive: [],
|
||||
discard_updates: [],
|
||||
discard_follows: [],
|
||||
remove_media: [],
|
||||
fetch_all_collection_members: false,
|
||||
authorized_fetch: false,
|
||||
},
|
||||
filters: {
|
||||
note_filters: [],
|
||||
username_filters: [],
|
||||
displayname_filters: [],
|
||||
bio_filters: [],
|
||||
emoji_filters: [],
|
||||
},
|
||||
logging: {
|
||||
log_requests: false,
|
||||
log_requests_verbose: false,
|
||||
log_filters: true,
|
||||
},
|
||||
ratelimits: {
|
||||
duration_coeff: 1,
|
||||
max_coeff: 1,
|
||||
},
|
||||
custom_ratelimits: {},
|
||||
}; */
|
||||
|
||||
/* export const getConfig = () => {
|
||||
// Deeply merge configDefaults, config and internalConfig
|
||||
return merge([configDefaults, config, internalConfig]) as any as ConfigType;
|
||||
};
|
||||
*/
|
||||
/**
|
||||
* Sets the internal config
|
||||
* @param newConfig Any part of ConfigType
|
||||
*/
|
||||
/* export const setConfig = async (newConfig: Partial<ConfigType>) => {
|
||||
const newInternalConfig = merge([
|
||||
internalConfig,
|
||||
newConfig,
|
||||
]) as any as ConfigType;
|
||||
|
||||
// Prepend a warning comment and write the new TOML to the file
|
||||
await Bun.write(
|
||||
Bun.file(process.cwd() + "/config/config.internal.toml"),
|
||||
`# This file is automatically generated. Do not modify it manually.\n${stringify(
|
||||
newInternalConfig as any
|
||||
)}`
|
||||
);
|
||||
}; */
|
||||
|
||||
/* export const getHost = () => {
|
||||
const url = new URL(getConfig().http.base_url);
|
||||
|
||||
return url.host;
|
||||
};
|
||||
*/
|
||||
// Refresh config every 5 seconds
|
||||
/* setInterval(() => {
|
||||
scanConfig()
|
||||
.then(newConfig => {
|
||||
if (newConfig !== config) {
|
||||
config = newConfig;
|
||||
}
|
||||
})
|
||||
.catch(e => {
|
||||
console.error(e);
|
||||
});
|
||||
}, 5000); */
|
||||
|
||||
/* export { config };
|
||||
*/
|
||||
273
classes/media.ts
273
classes/media.ts
|
|
@ -1,273 +0,0 @@
|
|||
import type { GetObjectCommandOutput } from "@aws-sdk/client-s3";
|
||||
import {
|
||||
GetObjectCommand,
|
||||
PutObjectCommand,
|
||||
S3Client,
|
||||
} from "@aws-sdk/client-s3";
|
||||
import type { ConfigType } from "~classes/configmanager";
|
||||
import sharp from "sharp";
|
||||
import { exists, mkdir } from "fs/promises";
|
||||
class MediaBackend {
|
||||
backend: string;
|
||||
|
||||
constructor(backend: string) {
|
||||
this.backend = backend;
|
||||
}
|
||||
|
||||
/**
|
||||
* Adds media to the media backend
|
||||
* @param media
|
||||
* @returns The hash of the file in SHA-256 (hex format) with the file extension added to it
|
||||
*/
|
||||
async addMedia(media: File) {
|
||||
const hash = new Bun.SHA256()
|
||||
.update(await media.arrayBuffer())
|
||||
.digest("hex");
|
||||
|
||||
return `${hash}.${media.name.split(".").pop()}`;
|
||||
}
|
||||
|
||||
async convertMedia(media: File, config: ConfigType) {
|
||||
const sharpCommand = sharp(await media.arrayBuffer());
|
||||
|
||||
// Rename ".jpg" files to ".jpeg" to avoid sharp errors
|
||||
let name = media.name;
|
||||
if (media.name.endsWith(".jpg")) {
|
||||
name = media.name.replace(".jpg", ".jpeg");
|
||||
}
|
||||
|
||||
const fileFormatToConvertTo = config.media.conversion.convert_to;
|
||||
|
||||
switch (fileFormatToConvertTo) {
|
||||
case "png":
|
||||
return new File(
|
||||
[(await sharpCommand.png().toBuffer()).buffer] as any,
|
||||
// Replace the file extension with PNG
|
||||
name.replace(/\.[^/.]+$/, ".png"),
|
||||
{
|
||||
type: "image/png",
|
||||
}
|
||||
);
|
||||
case "webp":
|
||||
return new File(
|
||||
[(await sharpCommand.webp().toBuffer()).buffer] as any,
|
||||
// Replace the file extension with WebP
|
||||
name.replace(/\.[^/.]+$/, ".webp"),
|
||||
{
|
||||
type: "image/webp",
|
||||
}
|
||||
);
|
||||
case "jpeg":
|
||||
return new File(
|
||||
[(await sharpCommand.jpeg().toBuffer()).buffer] as any,
|
||||
// Replace the file extension with JPEG
|
||||
name.replace(/\.[^/.]+$/, ".jpeg"),
|
||||
{
|
||||
type: "image/jpeg",
|
||||
}
|
||||
);
|
||||
case "avif":
|
||||
return new File(
|
||||
[(await sharpCommand.avif().toBuffer()).buffer] as any,
|
||||
// Replace the file extension with AVIF
|
||||
name.replace(/\.[^/.]+$/, ".avif"),
|
||||
{
|
||||
type: "image/avif",
|
||||
}
|
||||
);
|
||||
// Needs special build of libvips
|
||||
case "jxl":
|
||||
return new File(
|
||||
[(await sharpCommand.jxl().toBuffer()).buffer] as any,
|
||||
// Replace the file extension with JXL
|
||||
name.replace(/\.[^/.]+$/, ".jxl"),
|
||||
{
|
||||
type: "image/jxl",
|
||||
}
|
||||
);
|
||||
case "heif":
|
||||
return new File(
|
||||
[(await sharpCommand.heif().toBuffer()).buffer] as any,
|
||||
// Replace the file extension with HEIF
|
||||
name.replace(/\.[^/.]+$/, ".heif"),
|
||||
{
|
||||
type: "image/heif",
|
||||
}
|
||||
);
|
||||
default:
|
||||
return media;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Retrieves element from media backend by hash
|
||||
* @param hash The hash of the element in SHA-256 hex format
|
||||
* @param extension The extension of the file
|
||||
* @returns The file as a File object
|
||||
*/
|
||||
// eslint-disable-next-line @typescript-eslint/require-await, @typescript-eslint/no-unused-vars
|
||||
async getMediaByHash(
|
||||
// eslint-disable-next-line @typescript-eslint/no-unused-vars
|
||||
hash: string
|
||||
): Promise<File | null> {
|
||||
return new File([], "test");
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* S3 Backend, stores files in S3
|
||||
*/
|
||||
export class S3Backend extends MediaBackend {
|
||||
client: S3Client;
|
||||
config: ConfigType;
|
||||
|
||||
constructor(config: ConfigType) {
|
||||
super("s3");
|
||||
|
||||
this.config = config;
|
||||
|
||||
this.client = new S3Client({
|
||||
endpoint: this.config.s3.endpoint,
|
||||
region: this.config.s3.region || "auto",
|
||||
credentials: {
|
||||
accessKeyId: this.config.s3.access_key,
|
||||
secretAccessKey: this.config.s3.secret_access_key,
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
async addMedia(media: File): Promise<string> {
|
||||
if (this.config.media.conversion.convert_images) {
|
||||
media = await this.convertMedia(media, this.config);
|
||||
}
|
||||
|
||||
const hash = await super.addMedia(media);
|
||||
|
||||
if (!hash) {
|
||||
throw new Error("Failed to hash file");
|
||||
}
|
||||
|
||||
// Check if file is already present
|
||||
const existingFile = await this.getMediaByHash(hash);
|
||||
|
||||
if (existingFile) {
|
||||
// File already exists, so return the hash without uploading it
|
||||
return hash;
|
||||
}
|
||||
|
||||
const command = new PutObjectCommand({
|
||||
Bucket: this.config.s3.bucket_name,
|
||||
Key: hash,
|
||||
Body: Buffer.from(await media.arrayBuffer()),
|
||||
ContentType: media.type,
|
||||
ContentLength: media.size,
|
||||
Metadata: {
|
||||
"x-amz-meta-original-name": media.name,
|
||||
},
|
||||
});
|
||||
|
||||
const response = await this.client.send(command);
|
||||
|
||||
if (response.$metadata.httpStatusCode !== 200) {
|
||||
throw new Error("Failed to upload file");
|
||||
}
|
||||
|
||||
return hash;
|
||||
}
|
||||
|
||||
async getMediaByHash(hash: string): Promise<File | null> {
|
||||
const command = new GetObjectCommand({
|
||||
Bucket: this.config.s3.bucket_name,
|
||||
Key: hash,
|
||||
});
|
||||
|
||||
let response: GetObjectCommandOutput;
|
||||
|
||||
try {
|
||||
response = await this.client.send(command);
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
|
||||
if (response.$metadata.httpStatusCode !== 200) {
|
||||
throw new Error("Failed to get file");
|
||||
}
|
||||
|
||||
const body = await response.Body?.transformToByteArray();
|
||||
|
||||
if (!body) {
|
||||
throw new Error("Failed to get file");
|
||||
}
|
||||
|
||||
return new File([body], hash, {
|
||||
type: response.ContentType,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Local backend, stores files on filesystem
|
||||
*/
|
||||
export class LocalBackend extends MediaBackend {
|
||||
config: ConfigType;
|
||||
|
||||
constructor(config: ConfigType) {
|
||||
super("local");
|
||||
|
||||
this.config = config;
|
||||
}
|
||||
|
||||
async addMedia(media: File): Promise<string> {
|
||||
if (this.config.media.conversion.convert_images) {
|
||||
media = await this.convertMedia(media, this.config);
|
||||
}
|
||||
|
||||
const hash = await super.addMedia(media);
|
||||
|
||||
if (!(await exists(`${process.cwd()}/uploads`))) {
|
||||
await mkdir(`${process.cwd()}/uploads`);
|
||||
}
|
||||
|
||||
await Bun.write(Bun.file(`${process.cwd()}/uploads/${hash}`), media);
|
||||
|
||||
return hash;
|
||||
}
|
||||
|
||||
async getMediaByHash(hash: string): Promise<File | null> {
|
||||
const file = Bun.file(`${process.cwd()}/uploads/${hash}`);
|
||||
|
||||
if (!(await file.exists())) {
|
||||
return null;
|
||||
}
|
||||
|
||||
return new File([await file.arrayBuffer()], `${hash}`, {
|
||||
type: file.type,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
export const uploadFile = (file: File, config: ConfigType) => {
|
||||
const backend = config.media.backend;
|
||||
|
||||
if (backend === "local") {
|
||||
return new LocalBackend(config).addMedia(file);
|
||||
} else if (backend === "s3") {
|
||||
return new S3Backend(config).addMedia(file);
|
||||
}
|
||||
};
|
||||
|
||||
export const getFile = (
|
||||
hash: string,
|
||||
extension: string,
|
||||
config: ConfigType
|
||||
) => {
|
||||
const backend = config.media.backend;
|
||||
|
||||
if (backend === "local") {
|
||||
return new LocalBackend(config).getMediaByHash(hash);
|
||||
} else if (backend === "s3") {
|
||||
return new S3Backend(config).getMediaByHash(hash);
|
||||
}
|
||||
|
||||
return null;
|
||||
};
|
||||
Loading…
Add table
Add a link
Reference in a new issue