diff --git a/CHANGELOG.md b/CHANGELOG.md index a179b19d..84b27475 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -69,6 +69,7 @@ In the case that you've been running secret instances in the shadows, let us kno - 🐛 All media content-type is now correctly fetched, instead of guessed from the file extension as before. - 🐛 Fixed OpenAPI schema generation and `/docs` endpoint. - 🐛 Logs folder is now automatically created if it doesn't exist. +- 🐛 Media hosted on the configured S3 bucket and on the local filesystem is no longer unnecessarily proxied. # `0.7.0` • The Auth and APIs Update diff --git a/api/api/v1/emojis/index.test.ts b/api/api/v1/emojis/index.test.ts index e2848f4a..3bff570f 100644 --- a/api/api/v1/emojis/index.test.ts +++ b/api/api/v1/emojis/index.test.ts @@ -64,7 +64,7 @@ describe("/api/v1/emojis", () => { expect(ok).toBe(true); expect(data.shortcode).toBe("test1"); - expect(data.url).toContain("/media/proxy"); + expect(data.url).toContain("/media/"); }); test("should try to upload a non-image", async () => { @@ -116,7 +116,7 @@ describe("/api/v1/emojis", () => { expect(ok).toBe(true); expect(data.shortcode).toBe("test4"); - expect(data.url).toContain("/media/proxy"); + expect(data.url).toContain("/media/"); }); test("should fail when uploading an already existing global emoji", async () => { @@ -141,7 +141,7 @@ describe("/api/v1/emojis", () => { expect(ok).toBe(true); expect(data.shortcode).toBe("test4"); - expect(data.url).toContain("/media/proxy/"); + expect(data.url).toContain("/media/"); }); }); }); diff --git a/api/api/v1/instance/index.ts b/api/api/v1/instance/index.ts index 017036c9..afec20ed 100644 --- a/api/api/v1/instance/index.ts +++ b/api/api/v1/instance/index.ts @@ -1,5 +1,4 @@ import { apiRoute } from "@/api"; -import { proxyUrl } from "@/response"; import { InstanceV1 as InstanceV1Schema } from "@versia/client/schemas"; import { Instance, Note, User } from "@versia/kit/db"; import { Users } from "@versia/kit/tables"; @@ -8,6 +7,7 @@ import { describeRoute } from "hono-openapi"; import { resolver } from "hono-openapi/zod"; import type { z } from "zod"; import { markdownParse } from "~/classes/functions/status"; +import type { ProxiableUrl } from "~/classes/media/url"; import { config } from "~/config.ts"; import manifest from "~/package.json"; @@ -56,7 +56,7 @@ export default apiRoute((app) => providers?: { id: string; name: string; - icon: string; + icon?: ProxiableUrl; }[]; } | undefined; @@ -114,9 +114,7 @@ export default apiRoute((app) => status_count: statusCount, user_count: userCount, }, - thumbnail: config.instance.branding.logo - ? proxyUrl(config.instance.branding.logo).toString() - : null, + thumbnail: config.instance.branding.logo?.proxied ?? null, title: config.instance.name, uri: config.http.base_url.host, urls: { @@ -131,9 +129,7 @@ export default apiRoute((app) => providers: oidcConfig?.providers?.map((p) => ({ name: p.name, - icon: p.icon - ? proxyUrl(new URL(p.icon)).toString() - : undefined, + icon: p.icon?.proxied, id: p.id, })) ?? [], }, diff --git a/api/api/v2/instance/index.ts b/api/api/v2/instance/index.ts index daa94c6c..88536606 100644 --- a/api/api/v2/instance/index.ts +++ b/api/api/v2/instance/index.ts @@ -1,11 +1,11 @@ import { apiRoute } from "@/api"; -import { proxyUrl } from "@/response"; import { Instance as InstanceSchema } from "@versia/client/schemas"; import { User } from "@versia/kit/db"; import { Users } from "@versia/kit/tables"; import { and, eq, isNull } from "drizzle-orm"; import { describeRoute } from "hono-openapi"; import { resolver } from "hono-openapi/zod"; +import type { ProxiableUrl } from "~/classes/media/url"; import { config } from "~/config.ts"; import pkg from "~/package.json"; @@ -47,7 +47,7 @@ export default apiRoute((app) => providers?: { id: string; name: string; - icon: string; + icon?: ProxiableUrl; }[]; } | undefined; @@ -69,14 +69,10 @@ export default apiRoute((app) => mastodon: 1, }, thumbnail: { - url: config.instance.branding.logo - ? proxyUrl(config.instance.branding.logo).toString() - : pkg.icon, + url: config.instance.branding.logo?.proxied ?? pkg.icon, }, banner: { - url: config.instance.branding.banner - ? proxyUrl(config.instance.branding.banner).toString() - : null, + url: config.instance.branding.banner?.proxied ?? null, }, icon: [], languages: config.instance.languages, @@ -172,7 +168,7 @@ export default apiRoute((app) => providers: oidcConfig?.providers?.map((p) => ({ name: p.name, - icon: p.icon ? proxyUrl(new URL(p.icon)) : "", + icon: p.icon?.proxied, id: p.id, })) ?? [], }, diff --git a/classes/config/schema.ts b/classes/config/schema.ts index b59d63a5..1dd62be2 100644 --- a/classes/config/schema.ts +++ b/classes/config/schema.ts @@ -6,6 +6,7 @@ import { generateVAPIDKeys } from "web-push"; import { z } from "zod"; import { ZodError } from "zod"; import { fromZodError } from "zod-validation-error"; +import { ProxiableUrl } from "~/classes/media/url.ts"; import { RolePermission } from "~/packages/client/schemas/permissions.ts"; export const DEFAULT_ROLES = [ @@ -70,21 +71,21 @@ export enum MediaBackendType { // Need to declare this here instead of importing it otherwise we get cyclical import errors export const iso631 = z.enum(ISO6391.getAllCodes() as [string, ...string[]]); -const urlPath = z +export const urlPath = z .string() .trim() .min(1) // Remove trailing slashes, but keep the root slash .transform((arg) => (arg === "/" ? arg : arg.replace(/\/$/, ""))); -const url = z +export const url = z .string() .trim() .min(1) .refine((arg) => URL.canParse(arg), "Invalid url") - .transform((arg) => new URL(arg)); + .transform((arg) => new ProxiableUrl(arg)); -const unixPort = z +export const unixPort = z .number() .int() .min(1) diff --git a/classes/database/emoji.ts b/classes/database/emoji.ts index 14da1e69..bc0580a1 100644 --- a/classes/database/emoji.ts +++ b/classes/database/emoji.ts @@ -1,5 +1,4 @@ import { emojiValidatorWithColons, emojiValidatorWithIdentifiers } from "@/api"; -import { proxyUrl } from "@/response"; import type { CustomEmoji } from "@versia/client/schemas"; import type { CustomEmojiExtension } from "@versia/federation/types"; import { type Instance, Media, db } from "@versia/kit/db"; @@ -179,8 +178,8 @@ export class Emoji extends BaseInterface { return { id: this.id, shortcode: this.data.shortcode, - static_url: proxyUrl(this.media.getUrl()).toString(), - url: proxyUrl(this.media.getUrl()).toString(), + static_url: this.media.getUrl().proxied, + url: this.media.getUrl().proxied, visible_in_picker: this.data.visibleInPicker, category: this.data.category, global: this.data.ownerId === null, diff --git a/classes/database/media.ts b/classes/database/media.ts index 1e90dfcf..0cef287c 100644 --- a/classes/database/media.ts +++ b/classes/database/media.ts @@ -1,6 +1,5 @@ import { join } from "node:path"; import { mimeLookup } from "@/content_types.ts"; -import { proxyUrl } from "@/response"; import type { Attachment as AttachmentSchema } from "@versia/client/schemas"; import type { ContentFormat } from "@versia/federation/types"; import { db } from "@versia/kit/db"; @@ -20,6 +19,7 @@ import { MediaBackendType } from "~/classes/config/schema.ts"; import { config } from "~/config.ts"; import { ApiError } from "../errors/api-error.ts"; import { getMediaHash } from "../media/media-hasher.ts"; +import { ProxiableUrl } from "../media/url.ts"; import { MediaJobType, mediaQueue } from "../queues/media.ts"; import { BaseInterface } from "./base.ts"; @@ -369,10 +369,10 @@ export class Media extends BaseInterface { throw new Error("Unknown media backend"); } - public getUrl(): URL { + public getUrl(): ProxiableUrl { const type = this.getPreferredMimeType(); - return new URL(this.data.content[type]?.content ?? ""); + return new ProxiableUrl(this.data.content[type]?.content ?? ""); } /** @@ -510,10 +510,10 @@ export class Media extends BaseInterface { return { id: this.data.id, type: this.getMastodonType(), - url: proxyUrl(new URL(data.content)).toString(), + url: this.getUrl().proxied, remote_url: null, preview_url: thumbnailData?.content - ? proxyUrl(new URL(thumbnailData.content)).toString() + ? new ProxiableUrl(thumbnailData.content).proxied : null, meta: this.toApiMeta(), description: data.description || null, diff --git a/classes/database/role.ts b/classes/database/role.ts index e2ce30cf..478156bc 100644 --- a/classes/database/role.ts +++ b/classes/database/role.ts @@ -1,4 +1,3 @@ -import { proxyUrl } from "@/response"; import type { Role as RoleSchema } from "@versia/client/schemas"; import type { RolePermission } from "@versia/client/schemas"; import { db } from "@versia/kit/db"; @@ -14,11 +13,30 @@ import { } from "drizzle-orm"; import type { z } from "zod"; import { config } from "~/config.ts"; +import { ProxiableUrl } from "../media/url.ts"; import { BaseInterface } from "./base.ts"; type RoleType = InferSelectModel; export class Role extends BaseInterface { public static $type: RoleType; + public static defaultRole = new Role({ + id: "default", + name: "Default", + permissions: config.permissions.default, + priority: 0, + description: "Default role for all users", + visible: false, + icon: null, + }); + public static adminRole = new Role({ + id: "admin", + name: "Admin", + permissions: config.permissions.admin, + priority: 2 ** 31 - 1, + description: "Default role for all administrators", + visible: false, + icon: null, + }); public async reload(): Promise { const reloaded = await Role.fromId(this.data.id); @@ -59,24 +77,8 @@ export class Role extends BaseInterface { public static async getAll(): Promise { return (await Role.manyFromSql(undefined)).concat( - new Role({ - id: "default", - name: "Default", - permissions: config.permissions.default, - priority: 0, - description: "Default role for all users", - visible: false, - icon: null, - }), - new Role({ - id: "admin", - name: "Admin", - permissions: config.permissions.admin, - priority: 2 ** 31 - 1, - description: "Default role for all administrators", - visible: false, - icon: null, - }), + Role.defaultRole, + Role.adminRole, ); } @@ -215,7 +217,7 @@ export class Role extends BaseInterface { description: this.data.description ?? undefined, visible: this.data.visible, icon: this.data.icon - ? proxyUrl(new URL(this.data.icon)).toString() + ? new ProxiableUrl(this.data.icon).proxied : undefined, }; } diff --git a/classes/database/user.ts b/classes/database/user.ts index 87ceeea1..172d49cc 100644 --- a/classes/database/user.ts +++ b/classes/database/user.ts @@ -1,7 +1,6 @@ import { idValidator } from "@/api"; import { getBestContentType } from "@/content_types"; import { randomString } from "@/math"; -import { proxyUrl } from "@/response"; import { sentry } from "@/sentry"; import { getLogger } from "@logtape/logtape"; import type { @@ -56,6 +55,7 @@ import { findManyUsers } from "~/classes/functions/user"; import { searchManager } from "~/classes/search/search-manager"; import { config } from "~/config.ts"; import type { KnownEntity } from "~/types/api.ts"; +import { ProxiableUrl } from "../media/url.ts"; import { DeliveryJobType, deliveryQueue } from "../queues/delivery.ts"; import { PushJobType, pushQueue } from "../queues/push.ts"; import { BaseInterface } from "./base.ts"; @@ -180,19 +180,14 @@ export class User extends BaseInterface { } public getAllPermissions(): RolePermission[] { - return ( - this.data.roles - .flatMap((role) => role.permissions) + return Array.from( + new Set([ + ...this.data.roles.flatMap((role) => role.permissions), // Add default permissions - .concat(config.permissions.default) + ...config.permissions.default, // If admin, add admin permissions - .concat(this.data.isAdmin ? config.permissions.admin : []) - .reduce((acc, permission) => { - if (!acc.includes(permission)) { - acc.push(permission); - } - return acc; - }, [] as RolePermission[]) + ...(this.data.isAdmin ? config.permissions.admin : []), + ]), ); } @@ -415,7 +410,7 @@ export class User extends BaseInterface { id: string; name: string; url: string; - icon?: string; + icon?: ProxiableUrl; }[], ): Promise< { @@ -445,9 +440,7 @@ export class User extends BaseInterface { id: issuer.id, name: issuer.name, url: issuer.url, - icon: issuer.icon - ? proxyUrl(new URL(issuer.icon)).toString() - : undefined, + icon: issuer.icon?.proxied, server_id: account.serverId, }; }) @@ -831,11 +824,11 @@ export class User extends BaseInterface { * Get the user's avatar in raw URL format * @returns The raw URL for the user's avatar */ - public getAvatarUrl(): URL { + public getAvatarUrl(): ProxiableUrl { if (!this.avatar) { return ( config.defaults.avatar || - new URL( + new ProxiableUrl( `https://api.dicebear.com/8.x/${config.defaults.placeholder_style}/svg?seed=${this.data.username}`, ) ); @@ -928,10 +921,11 @@ export class User extends BaseInterface { * Get the user's header in raw URL format * @returns The raw URL for the user's header */ - public getHeaderUrl(): URL | null { + public getHeaderUrl(): ProxiableUrl | null { if (!this.header) { return config.defaults.header ?? null; } + return this.header.getUrl(); } @@ -974,7 +968,8 @@ export class User extends BaseInterface { newUser.isBot || newUser.isLocked || newUser.endpoints || - newUser.isDiscoverable) + newUser.isDiscoverable || + newUser.isIndexable) ) { await this.federateToFollowers(this.toVersia()); } @@ -1050,6 +1045,19 @@ export class User extends BaseInterface { return new FederationRequester(signatureConstructor); } + /** + * Get all remote followers of the user + * @returns The remote followers + */ + private getRemoteFollowers(): Promise { + return User.manyFromSql( + and( + sql`EXISTS (SELECT 1 FROM "Relationships" WHERE "Relationships"."subjectId" = ${this.id} AND "Relationships"."ownerId" = ${Users.id} AND "Relationships"."following" = true)`, + isNotNull(Users.instanceId), + ), + ); + } + /** * Federates an entity to all followers of the user * @@ -1057,12 +1065,7 @@ export class User extends BaseInterface { */ public async federateToFollowers(entity: KnownEntity): Promise { // Get followers - const followers = await User.manyFromSql( - and( - sql`EXISTS (SELECT 1 FROM "Relationships" WHERE "Relationships"."subjectId" = ${this.id} AND "Relationships"."ownerId" = ${Users.id} AND "Relationships"."following" = true)`, - isNotNull(Users.instanceId), - ), - ); + const followers = await this.getRemoteFollowers(); await deliveryQueue.addBulk( followers.map((follower) => ({ @@ -1130,10 +1133,8 @@ export class User extends BaseInterface { url: user.uri || new URL(`/@${user.username}`, config.http.base_url).toString(), - avatar: proxyUrl(this.getAvatarUrl()).toString(), - header: this.getHeaderUrl() - ? proxyUrl(this.getHeaderUrl() as URL).toString() - : "", + avatar: this.getAvatarUrl().proxied, + header: this.getHeaderUrl()?.proxied ?? "", locked: user.isLocked, created_at: new Date(user.createdAt).toISOString(), followers_count: @@ -1154,10 +1155,8 @@ export class User extends BaseInterface { bot: user.isBot, source: isOwnAccount ? user.source : undefined, // TODO: Add static avatar and header - avatar_static: proxyUrl(this.getAvatarUrl()).toString(), - header_static: this.getHeaderUrl() - ? proxyUrl(this.getHeaderUrl() as URL).toString() - : "", + avatar_static: this.getAvatarUrl().proxied, + header_static: this.getHeaderUrl()?.proxied ?? "", acct: this.getAcct(), // TODO: Add these fields limited: false, @@ -1168,33 +1167,8 @@ export class User extends BaseInterface { mute_expires_at: null, roles: user.roles .map((role) => new Role(role)) - .concat( - new Role({ - id: "default", - name: "Default", - permissions: config.permissions.default, - priority: 0, - description: "Default role for all users", - visible: false, - icon: null, - }), - ) - .concat( - user.isAdmin - ? [ - new Role({ - id: "admin", - name: "Admin", - permissions: config.permissions.admin, - priority: 2 ** 31 - 1, - description: - "Default role for all administrators", - visible: false, - icon: null, - }), - ] - : [], - ) + .concat(Role.defaultRole) + .concat(user.isAdmin ? Role.adminRole : []) .map((r) => r.toApi()), group: false, // TODO diff --git a/classes/media/url.ts b/classes/media/url.ts new file mode 100644 index 00000000..bd0e3302 --- /dev/null +++ b/classes/media/url.ts @@ -0,0 +1,25 @@ +import { config } from "~/config.ts"; + +export class ProxiableUrl extends URL { + private isAllowedOrigin(): boolean { + const allowedOrigins: URL[] = [config.http.base_url].concat( + config.s3?.public_url ?? [], + ); + + return allowedOrigins.some((origin) => + this.hostname.endsWith(origin.hostname), + ); + } + + public get proxied(): string { + // Don't proxy from CDN and self, since those sources are trusted + if (this.isAllowedOrigin()) { + return this.href; + } + + const urlAsBase64Url = Buffer.from(this.href).toString("base64url"); + + return new URL(`/media/proxy/${urlAsBase64Url}`, config.http.base_url) + .href; + } +} diff --git a/plugins/openid/index.ts b/plugins/openid/index.ts index 4ec0d565..011216cf 100644 --- a/plugins/openid/index.ts +++ b/plugins/openid/index.ts @@ -5,7 +5,7 @@ import { getCookie } from "hono/cookie"; import { jwtVerify } from "jose"; import { JOSEError, JWTExpired } from "jose/errors"; import { z } from "zod"; -import { keyPair, sensitiveString } from "~/classes/config/schema.ts"; +import { url, keyPair, sensitiveString } from "~/classes/config/schema.ts"; import { ApiError } from "~/classes/errors/api-error.ts"; import authorizeRoute from "./routes/authorize.ts"; import jwksRoute from "./routes/jwks.ts"; @@ -27,7 +27,7 @@ const configSchema = z.object({ url: z.string().min(1), client_id: z.string().min(1), client_secret: sensitiveString, - icon: z.string().min(1).optional(), + icon: url.optional(), }), ) .default([]), diff --git a/plugins/openid/routes/sso/:id/index.ts b/plugins/openid/routes/sso/:id/index.ts index 21358a7d..e611f43c 100644 --- a/plugins/openid/routes/sso/:id/index.ts +++ b/plugins/openid/routes/sso/:id/index.ts @@ -1,5 +1,4 @@ import { auth, handleZodError } from "@/api"; -import { proxyUrl } from "@/response"; import { RolePermission } from "@versia/client/schemas"; import { db } from "@versia/kit/db"; import { type SQL, eq } from "@versia/kit/drizzle"; @@ -77,9 +76,7 @@ export default (plugin: PluginType): void => { { id: issuer.id, name: issuer.name, - icon: issuer.icon - ? proxyUrl(new URL(issuer.icon)) - : undefined, + icon: issuer.icon?.proxied, }, 200, ); diff --git a/utils/response.ts b/utils/response.ts deleted file mode 100644 index 6aae4e0f..00000000 --- a/utils/response.ts +++ /dev/null @@ -1,17 +0,0 @@ -import { config } from "~/config.ts"; - -export type Json = - | string - | number - | boolean - | null - | undefined - | Json[] - | { [key: string]: Json }; - -export const proxyUrl = (url: URL): URL => { - const urlAsBase64Url = Buffer.from(url.toString() || "").toString( - "base64url", - ); - return new URL(`/media/proxy/${urlAsBase64Url}`, config.http.base_url); -}; diff --git a/utils/sanitization.ts b/utils/sanitization.ts index ee579a94..857ef17f 100644 --- a/utils/sanitization.ts +++ b/utils/sanitization.ts @@ -1,6 +1,6 @@ import { stringifyEntitiesLight } from "stringify-entities"; import xss, { type IFilterXSSOptions } from "xss"; -import { proxyUrl } from "./response.ts"; +import { ProxiableUrl } from "~/classes/media/url.ts"; export const sanitizedHtmlStrip = (html: string): Promise => { return sanitizeHtml(html, { @@ -137,9 +137,9 @@ export const sanitizeHtml = async ( element.setAttribute( "src", element.getAttribute("src") - ? proxyUrl( - new URL(element.getAttribute("src") as string), - ).toString() + ? new ProxiableUrl( + element.getAttribute("src") as string, + ).proxied : "", ); },