versia-pub/server
2
0
Fork 0
mirror of https://github.com/versia-pub/server.git synced 2025-12-06 00:18:19 +01:00
Code Issues Actions Packages Projects Releases Wiki Activity

Refactor debugging and logging more

Browse source
This commit is contained in:
Jesse Wierzbinski 2024-04-14 01:20:55 -10:00
parent 82c6dc17a8
commit ab6fe6988c
No known key found for this signature in database
20 changed files with 230 additions and 123 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

160
config/config.example.toml
View file

@ -42,10 +42,10 @@ tos_url = "https://my-site.com/tos"
# Whether to enable registrations or not
registration = true
rules = [
"Do not harass others",
"Be nice to people",
"Don't spam",
"Don't post illegal content",
"Do not harass others",
"Be nice to people",
"Don't spam",
"Don't post illegal content",
]
# Delete this section if you don't want to use custom OAuth providers
@ -72,8 +72,8 @@ bind_port = "8080"
banned_ips = []
# Banned user agents, regex format
banned_user_agents = [
# "curl\/7.68.0",
# "wget\/1.20.3",
# "curl\/7.68.0",
# "wget\/1.20.3",
]
[http.bait]
@ -156,32 +156,32 @@ max_field_name_size = 1000
max_field_value_size = 1000
# Forbidden usernames, defaults are from Akkoma
username_blacklist = [
".well-known",
"~",
"about",
"activities",
"api",
"auth",
"dev",
"inbox",
"internal",
"main",
"media",
"nodeinfo",
"notice",
"oauth",
"objects",
"proxy",
"push",
"registration",
"relay",
"settings",
"status",
"tag",
"users",
"web",
"search",
"mfa",
".well-known",
"~",
"about",
"activities",
"api",
"auth",
"dev",
"inbox",
"internal",
"main",
"media",
"nodeinfo",
"notice",
"oauth",
"objects",
"proxy",
"push",
"registration",
"relay",
"settings",
"status",
"tag",
"users",
"web",
"search",
"mfa",
]
# Whether to blacklist known temporary email providers
blacklist_tempmail = false
@ -189,57 +189,57 @@ blacklist_tempmail = false
email_blacklist = []
# Valid URL schemes, otherwise the URL is parsed as text
url_scheme_whitelist = [
"http",
"https",
"ftp",
"dat",
"dweb",
"gopher",
"hyper",
"ipfs",
"ipns",
"irc",
"xmpp",
"ircs",
"magnet",
"mailto",
"mumble",
"ssb",
"gemini",
"http",
"https",
"ftp",
"dat",
"dweb",
"gopher",
"hyper",
"ipfs",
"ipns",
"irc",
"xmpp",
"ircs",
"magnet",
"mailto",
"mumble",
"ssb",
"gemini",
]
# Only allow those MIME types of data to be uploaded
# This can easily be spoofed, but if it is spoofed it will appear broken
# to normal clients until despoofed
enforce_mime_types = false
allowed_mime_types = [
"image/jpeg",
"image/png",
"image/gif",
"image/heic",
"image/heif",
"image/webp",
"image/avif",
"video/webm",
"video/mp4",
"video/quicktime",
"video/ogg",
"audio/wave",
"audio/wav",
"audio/x-wav",
"audio/x-pn-wave",
"audio/vnd.wave",
"audio/ogg",
"audio/vorbis",
"audio/mpeg",
"audio/mp3",
"audio/webm",
"audio/flac",
"audio/aac",
"audio/m4a",
"audio/x-m4a",
"audio/mp4",
"audio/3gpp",
"video/x-ms-asf",
"image/jpeg",
"image/png",
"image/gif",
"image/heic",
"image/heif",
"image/webp",
"image/avif",
"video/webm",
"video/mp4",
"video/quicktime",
"video/ogg",
"audio/wave",
"audio/wav",
"audio/x-wav",
"audio/x-pn-wave",
"audio/vnd.wave",
"audio/ogg",
"audio/vorbis",
"audio/mpeg",
"audio/mp3",
"audio/webm",
"audio/flac",
"audio/aac",
"audio/m4a",
"audio/x-m4a",
"audio/mp4",
"audio/3gpp",
"video/x-ms-asf",
]
[defaults]
@ -294,8 +294,8 @@ banner = ""
# Note contents
note_content = [
# "(https?://)?(www\\.)?youtube\\.com/watch\\?v=[a-zA-Z0-9_-]+",
# "(https?://)?(www\\.)?youtu\\.be/[a-zA-Z0-9_-]+",
# "(https?://)?(www\\.)?youtube\\.com/watch\\?v=[a-zA-Z0-9_-]+",
# "(https?://)?(www\\.)?youtu\\.be/[a-zA-Z0-9_-]+",
]
emoji = []
# These will drop users matching the filters
@ -308,6 +308,8 @@ bio = []
log_requests = false
# Log request and their contents (warning: this is a lot of data)
log_requests_verbose = false
# Available levels: debug, info, warning, error, critical
log_level = "info"
# For GDPR compliance, you can disable logging of IPs
log_ip = false

20
database/entities/Status.ts
View file

@ -35,6 +35,8 @@ import {
statusToMentions,
user,
} from "~drizzle/schema";
import { dualLogger } from "@loggers";
import { LogLevel } from "~packages/log-manager";
import type { Note } from "~types/lysand/Object";
import type { Attachment as APIAttachment } from "~types/mastodon/attachment";
import type { Status as APIStatus } from "~types/mastodon/status";
@ -601,7 +603,11 @@ export const resolveStatus = async (
for (const attachment of note.attachments ?? []) {
const resolvedAttachment = await attachmentFromLysand(attachment).catch(
(e) => {
console.error(e);
dualLogger.logError(
LogLevel.ERROR,
"Federation.StatusResolver",
e,
);
return null;
},
);
@ -616,7 +622,7 @@ export const resolveStatus = async (
for (const emoji of note.extensions?.["org.lysand:custom_emojis"]?.emojis ??
[]) {
const resolvedEmoji = await fetchEmoji(emoji).catch((e) => {
console.error(e);
dualLogger.logError(LogLevel.ERROR, "Federation.StatusResolver", e);
return null;
});
@ -1010,8 +1016,14 @@ export const federateStatus = async (status: StatusWithRelations) => {
const response = await fetch(request);
if (!response.ok) {
console.error(await response.text());
throw new Error(
dualLogger.log(
LogLevel.DEBUG,
"Federation.Status",
await response.text(),
);
dualLogger.log(
LogLevel.ERROR,
"Federation.Status",
`Failed to federate status ${status.id} to ${user.uri}`,
);
}

35
database/entities/User.ts
View file

@ -12,6 +12,8 @@ import {
relationship,
user,
} from "~drizzle/schema";
import { dualLogger } from "@loggers";
import { LogLevel } from "~packages/log-manager";
import type { Account as APIAccount } from "~types/mastodon/account";
import type { Source as APISource } from "~types/mastodon/source";
import {
@ -191,8 +193,15 @@ export const followRequestUser = async (
const response = await fetch(request);
if (!response.ok) {
console.error(await response.text());
console.error(
dualLogger.log(
LogLevel.DEBUG,
"Federation.FollowRequest",
await response.text(),
);
dualLogger.log(
LogLevel.ERROR,
"Federation.FollowRequest",
`Failed to federate follow request from ${follower.id} to ${followee.uri}`,
);
@ -230,8 +239,15 @@ export const sendFollowAccept = async (follower: User, followee: User) => {
const response = await fetch(request);
if (!response.ok) {
console.error(await response.text());
throw new Error(
dualLogger.log(
LogLevel.DEBUG,
"Federation.FollowAccept",
await response.text(),
);
dualLogger.log(
LogLevel.ERROR,
"Federation.FollowAccept",
`Failed to federate follow accept from ${followee.id} to ${follower.uri}`,
);
}
@ -249,8 +265,15 @@ export const sendFollowReject = async (follower: User, followee: User) => {
const response = await fetch(request);
if (!response.ok) {
console.error(await response.text());
throw new Error(
dualLogger.log(
LogLevel.DEBUG,
"Federation.FollowReject",
await response.text(),
);
dualLogger.log(
LogLevel.ERROR,
"Federation.FollowReject",
`Failed to federate follow reject from ${followee.id} to ${follower.uri}`,
);
}

21
index.ts
View file

@ -1,28 +1,15 @@
import { dualLogger } from "@loggers";
import { connectMeili } from "@meilisearch";
import { config } from "config-manager";
import { count } from "drizzle-orm";
import { LogLevel, LogManager, MultiLogManager } from "log-manager";
import { LogLevel } from "log-manager";
import { db, setupDatabase } from "~drizzle/db";
import { status } from "~drizzle/schema";
import { createServer } from "~server";
const timeAtStart = performance.now();
const requests_log = Bun.file(config.logging.storage.requests);
const isEntry = import.meta.path === Bun.main;
const noColors = process.env.NO_COLORS === "true";
const noFancyDates = process.env.NO_FANCY_DATES === "true";
// If imported as a module, redirect logs to /dev/null to not pollute console (e.g. in tests)
const logger = new LogManager(isEntry ? requests_log : Bun.file("/dev/null"));
const consoleLogger = new LogManager(
isEntry ? Bun.stdout : Bun.file("/dev/null"),
!noColors,
!noFancyDates,
);
const dualLogger = new MultiLogManager([logger, consoleLogger]);
await dualLogger.log(LogLevel.INFO, "Lysand", "Starting Lysand...");
await setupDatabase(dualLogger);
@ -43,11 +30,9 @@ try {
)[0].count;
} catch (e) {
const error = e as Error;
await logger.logError(LogLevel.CRITICAL, "Database", error);
await consoleLogger.logError(LogLevel.CRITICAL, "Database", error);
await dualLogger.logError(LogLevel.CRITICAL, "Database", error);
process.exit(1);
}
const server = createServer(config, dualLogger, true);
await dualLogger.log(

4
packages/config-manager/config.type.ts
View file

@ -336,6 +336,9 @@ export interface Config {
/** @default false */
log_requests_verbose: boolean;
/** @default "info" */
log_level: "info" | "debug" | "warning" | "error" | "critical";
/** @default false */
log_ip: boolean;
@ -591,6 +594,7 @@ export const defaultConfig: Config = {
logging: {
log_requests: false,
log_requests_verbose: false,
log_level: "info",
log_ip: false,
log_filters: true,
storage: {

16
packages/log-manager/index.ts
View file

@ -2,6 +2,7 @@ import { appendFile, exists, mkdir } from "node:fs/promises";
import { dirname } from "node:path";
import type { BunFile } from "bun";
import chalk from "chalk";
import { config } from "config-manager";
export enum LogLevel {
DEBUG = "debug",
@ -11,6 +12,14 @@ export enum LogLevel {
CRITICAL = "critical",
}
const logOrder = [
LogLevel.DEBUG,
LogLevel.INFO,
LogLevel.WARNING,
LogLevel.ERROR,
LogLevel.CRITICAL,
];
/**
* Class for handling logging to disk or to stdout
* @param output BunFile of output (can be a normal file or something like Bun.stdout)
@ -67,6 +76,12 @@ export class LogManager {
message: string,
showTimestamp = true,
) {
if (
logOrder.indexOf(level) <
logOrder.indexOf(config.logging.log_level as LogLevel)
)
return;
if (this.enableColors) {
await this.write(
`${
@ -113,6 +128,7 @@ export class LogManager {
* @param error Error to log
*/
async logError(level: LogLevel, entity: string, error: Error) {
error.stack && (await this.log(LogLevel.DEBUG, entity, error.stack));
await this.log(level, entity, error.message);
}

5
packages/server-handler/index.ts
View file

@ -164,6 +164,11 @@ export const processRoute = async (
return output;
} catch (err) {
await logger.log(
LogLevel.DEBUG,
"Server.RouteHandler",
(err as Error).toString(),
);
await logger.logError(
LogLevel.ERROR,
"Server.RouteHandler",

34
server.ts
View file

@ -24,8 +24,21 @@ export const createServer = (
return errorResponse("Forbidden", 403);
}
} catch (e) {
console.error(`[-] Error while parsing banned IP "${ip}" `);
throw e;
logger.log(
LogLevel.ERROR,
"Server.IPCheck",
`Error while parsing banned IP "${ip}" `,
);
logger.logError(
LogLevel.ERROR,
"Server.IPCheck",
e as Error,
);
return errorResponse(
`A server error occured: ${(e as Error).message}`,
500,
);
}
}
@ -57,10 +70,21 @@ export const createServer = (
);
}
} catch (e) {
console.error(
`[-] Error while parsing bait IP "${ip}" `,
logger.log(
LogLevel.ERROR,
"Server.IPCheck",
`Error while parsing bait IP "${ip}" `,
);
logger.logError(
LogLevel.ERROR,
"Server.IPCheck",
e as Error,
);
return errorResponse(
`A server error occured: ${(e as Error).message}`,
500,
);
throw e;
}
}

2
server/api/api/v1/accounts/[id]/index.test.ts
View file

@ -1,5 +1,5 @@
import { afterAll, beforeAll, describe, expect, test } from "bun:test";
import { config } from "~index";
import { config } from "config-manager";
import {
deleteOldTestUsers,
getTestStatuses,

2
server/api/api/v1/accounts/lookup/index.test.ts
View file

@ -1,5 +1,5 @@
import { afterAll, describe, expect, test } from "bun:test";
import { config } from "~index";
import { config } from "config-manager";
import {
deleteOldTestUsers,
getTestUsers,

8
server/api/api/v1/accounts/lookup/index.ts
View file

@ -17,6 +17,8 @@ import {
resolveWebFinger,
userToAPI,
} from "~database/entities/User";
import { dualLogger } from "@loggers";
import { LogLevel } from "~packages/log-manager";
export const meta = applyConfig({
allowedMethods: ["GET"],
@ -68,7 +70,11 @@ export default apiRoute<typeof meta, typeof schema>(
const [username, domain] = accountMatches[0].split("@");
const foundAccount = await resolveWebFinger(username, domain).catch(
(e) => {
console.error(e);
dualLogger.logError(
LogLevel.ERROR,
"WebFinger.Resolve",
e as Error,
);
return null;
},
);

2
server/api/api/v1/accounts/search/index.test.ts
View file

@ -1,5 +1,5 @@
import { afterAll, describe, expect, test } from "bun:test";
import { config } from "~index";
import { config } from "config-manager";
import {
deleteOldTestUsers,
getTestUsers,

2
server/api/api/v1/statuses/[id]/favourited_by.test.ts
View file

@ -1,5 +1,5 @@
import { afterAll, beforeAll, describe, expect, test } from "bun:test";
import { config } from "~index";
import { config } from "config-manager";
import {
deleteOldTestUsers,
getTestStatuses,

2
server/api/api/v1/statuses/[id]/reblogged_by.test.ts
View file

@ -1,5 +1,5 @@
import { afterAll, beforeAll, describe, expect, test } from "bun:test";
import { config } from "~index";
import { config } from "config-manager";
import {
deleteOldTestUsers,
getTestStatuses,

2
server/api/api/v1/statuses/index.test.ts
View file

@ -1,5 +1,5 @@
import { afterAll, describe, expect, test } from "bun:test";
import { config } from "~index";
import { config } from "config-manager";
import {
deleteOldTestUsers,
getTestUsers,

2
server/api/api/v1/timelines/home.test.ts
View file

@ -1,5 +1,5 @@
import { afterAll, describe, expect, test } from "bun:test";
import { config } from "~index";
import { config } from "config-manager";
import {
deleteOldTestUsers,
getTestStatuses,

2
server/api/api/v1/timelines/public.test.ts
View file

@ -1,5 +1,5 @@
import { afterAll, describe, expect, test } from "bun:test";
import { config } from "~index";
import { config } from "config-manager";
import {
deleteOldTestUsers,
getTestStatuses,

8
server/api/api/v2/search/index.ts
View file

@ -12,6 +12,8 @@ import {
} from "~database/entities/User";
import { db } from "~drizzle/db";
import { instance, user } from "~drizzle/schema";
import { dualLogger } from "@loggers";
import { LogLevel } from "~packages/log-manager";
export const meta = applyConfig({
allowedMethods: ["GET"],
@ -117,7 +119,11 @@ export default apiRoute<typeof meta, typeof schema>(
username,
domain,
).catch((e) => {
console.error(e);
dualLogger.logError(
LogLevel.ERROR,
"WebFinger.Resolve",
e,
);
return null;
});

8
server/api/users/[uuid]/inbox/index.ts
View file

@ -11,6 +11,8 @@ import {
} from "~database/entities/User";
import { db } from "~drizzle/db";
import { notification, relationship } from "~drizzle/schema";
import { dualLogger } from "@loggers";
import { LogLevel } from "~packages/log-manager";
export const meta = applyConfig({
allowedMethods: ["POST"],
@ -126,7 +128,11 @@ export default apiRoute(async (req, matchedRoute, extraData) => {
const newStatus = await resolveStatus(undefined, note).catch(
(e) => {
console.error(e);
dualLogger.logError(
LogLevel.ERROR,
"Inbox.NoteResolve",
e as Error,
);
return null;
},
);

18
utils/loggers.ts Normal file
View file

@ -0,0 +1,18 @@
import { LogManager, MultiLogManager } from "log-manager";
import { config } from "config-manager";
const noColors = process.env.NO_COLORS === "true";
const noFancyDates = process.env.NO_FANCY_DATES === "true";
const requests_log = Bun.file(config.logging.storage.requests);
export const logger = new LogManager(
true ? requests_log : Bun.file("/dev/null"),
);
export const consoleLogger = new LogManager(
true ? Bun.stdout : Bun.file("/dev/null"),
!noColors,
!noFancyDates,
);
export const dualLogger = new MultiLogManager([logger, consoleLogger]);