fix(media): 🐛 Don't proxy media from trusted origins, use new ProxiedUrl class

CHANGELOG.md

@@ -69,6 +69,7 @@ In the case that you've been running secret instances in the shadows, let us kno
 -   🐛 All media content-type is now correctly fetched, instead of guessed from the file extension as before.
 -   🐛 Fixed OpenAPI schema generation and `/docs` endpoint.
 -   🐛 Logs folder is now automatically created if it doesn't exist.
+-   🐛 Media hosted on the configured S3 bucket and on the local filesystem is no longer unnecessarily proxied.
 
 # `0.7.0` • The Auth and APIs Update
 

api/api/v1/emojis/index.test.ts

@@ -64,7 +64,7 @@ describe("/api/v1/emojis", () => {
 
             expect(ok).toBe(true);
             expect(data.shortcode).toBe("test1");
-            expect(data.url).toContain("/media/proxy");
+            expect(data.url).toContain("/media/");
         });
 
         test("should try to upload a non-image", async () => {
@@ -116,7 +116,7 @@ describe("/api/v1/emojis", () => {
 
             expect(ok).toBe(true);
             expect(data.shortcode).toBe("test4");
-            expect(data.url).toContain("/media/proxy");
+            expect(data.url).toContain("/media/");
         });
 
         test("should fail when uploading an already existing global emoji", async () => {
@@ -141,7 +141,7 @@ describe("/api/v1/emojis", () => {
 
             expect(ok).toBe(true);
             expect(data.shortcode).toBe("test4");
-            expect(data.url).toContain("/media/proxy/");
+            expect(data.url).toContain("/media/");
         });
     });
 });

api/api/v1/instance/index.ts

@@ -1,5 +1,4 @@
 import { apiRoute } from "@/api";
-import { proxyUrl } from "@/response";
 import { InstanceV1 as InstanceV1Schema } from "@versia/client/schemas";
 import { Instance, Note, User } from "@versia/kit/db";
 import { Users } from "@versia/kit/tables";
@@ -8,6 +7,7 @@ import { describeRoute } from "hono-openapi";
 import { resolver } from "hono-openapi/zod";
 import type { z } from "zod";
 import { markdownParse } from "~/classes/functions/status";
+import type { ProxiableUrl } from "~/classes/media/url";
 import { config } from "~/config.ts";
 import manifest from "~/package.json";
 
@@ -56,7 +56,7 @@ export default apiRoute((app) =>
                       providers?: {
                           id: string;
                           name: string;
-                          icon: string;
+                          icon?: ProxiableUrl;
                       }[];
                   }
                 | undefined;
@@ -114,9 +114,7 @@ export default apiRoute((app) =>
                     status_count: statusCount,
                     user_count: userCount,
                 },
-                thumbnail: config.instance.branding.logo
+                thumbnail: config.instance.branding.logo?.proxied ?? null,
-                    ? proxyUrl(config.instance.branding.logo).toString()
-                    : null,
                 title: config.instance.name,
                 uri: config.http.base_url.host,
                 urls: {
@@ -131,9 +129,7 @@ export default apiRoute((app) =>
                     providers:
                         oidcConfig?.providers?.map((p) => ({
                             name: p.name,
-                            icon: p.icon
+                            icon: p.icon?.proxied,
-                                ? proxyUrl(new URL(p.icon)).toString()
-                                : undefined,
                             id: p.id,
                         })) ?? [],
                 },

api/api/v2/instance/index.ts

@@ -1,11 +1,11 @@
 import { apiRoute } from "@/api";
-import { proxyUrl } from "@/response";
 import { Instance as InstanceSchema } from "@versia/client/schemas";
 import { User } from "@versia/kit/db";
 import { Users } from "@versia/kit/tables";
 import { and, eq, isNull } from "drizzle-orm";
 import { describeRoute } from "hono-openapi";
 import { resolver } from "hono-openapi/zod";
+import type { ProxiableUrl } from "~/classes/media/url";
 import { config } from "~/config.ts";
 import pkg from "~/package.json";
 
@@ -47,7 +47,7 @@ export default apiRoute((app) =>
                       providers?: {
                           id: string;
                           name: string;
-                          icon: string;
+                          icon?: ProxiableUrl;
                       }[];
                   }
                 | undefined;
@@ -69,14 +69,10 @@ export default apiRoute((app) =>
                     mastodon: 1,
                 },
                 thumbnail: {
-                    url: config.instance.branding.logo
+                    url: config.instance.branding.logo?.proxied ?? pkg.icon,
-                        ? proxyUrl(config.instance.branding.logo).toString()
-                        : pkg.icon,
                 },
                 banner: {
-                    url: config.instance.branding.banner
+                    url: config.instance.branding.banner?.proxied ?? null,
-                        ? proxyUrl(config.instance.branding.banner).toString()
-                        : null,
                 },
                 icon: [],
                 languages: config.instance.languages,
@@ -172,7 +168,7 @@ export default apiRoute((app) =>
                     providers:
                         oidcConfig?.providers?.map((p) => ({
                             name: p.name,
-                            icon: p.icon ? proxyUrl(new URL(p.icon)) : "",
+                            icon: p.icon?.proxied,
                             id: p.id,
                         })) ?? [],
                 },

classes/config/schema.ts

@@ -6,6 +6,7 @@ import { generateVAPIDKeys } from "web-push";
 import { z } from "zod";
 import { ZodError } from "zod";
 import { fromZodError } from "zod-validation-error";
+import { ProxiableUrl } from "~/classes/media/url.ts";
 import { RolePermission } from "~/packages/client/schemas/permissions.ts";
 
 export const DEFAULT_ROLES = [
@@ -70,21 +71,21 @@ export enum MediaBackendType {
 // Need to declare this here instead of importing it otherwise we get cyclical import errors
 export const iso631 = z.enum(ISO6391.getAllCodes() as [string, ...string[]]);
 
-const urlPath = z
+export const urlPath = z
     .string()
     .trim()
     .min(1)
     // Remove trailing slashes, but keep the root slash
     .transform((arg) => (arg === "/" ? arg : arg.replace(/\/$/, "")));
 
-const url = z
+export const url = z
     .string()
     .trim()
     .min(1)
     .refine((arg) => URL.canParse(arg), "Invalid url")
-    .transform((arg) => new URL(arg));
+    .transform((arg) => new ProxiableUrl(arg));
 
-const unixPort = z
+export const unixPort = z
     .number()
     .int()
     .min(1)

classes/database/emoji.ts

@@ -1,5 +1,4 @@
 import { emojiValidatorWithColons, emojiValidatorWithIdentifiers } from "@/api";
-import { proxyUrl } from "@/response";
 import type { CustomEmoji } from "@versia/client/schemas";
 import type { CustomEmojiExtension } from "@versia/federation/types";
 import { type Instance, Media, db } from "@versia/kit/db";
@@ -179,8 +178,8 @@ export class Emoji extends BaseInterface<typeof Emojis, EmojiType> {
         return {
             id: this.id,
             shortcode: this.data.shortcode,
-            static_url: proxyUrl(this.media.getUrl()).toString(),
+            static_url: this.media.getUrl().proxied,
-            url: proxyUrl(this.media.getUrl()).toString(),
+            url: this.media.getUrl().proxied,
             visible_in_picker: this.data.visibleInPicker,
             category: this.data.category,
             global: this.data.ownerId === null,

classes/database/media.ts

@@ -1,6 +1,5 @@
 import { join } from "node:path";
 import { mimeLookup } from "@/content_types.ts";
-import { proxyUrl } from "@/response";
 import type { Attachment as AttachmentSchema } from "@versia/client/schemas";
 import type { ContentFormat } from "@versia/federation/types";
 import { db } from "@versia/kit/db";
@@ -20,6 +19,7 @@ import { MediaBackendType } from "~/classes/config/schema.ts";
 import { config } from "~/config.ts";
 import { ApiError } from "../errors/api-error.ts";
 import { getMediaHash } from "../media/media-hasher.ts";
+import { ProxiableUrl } from "../media/url.ts";
 import { MediaJobType, mediaQueue } from "../queues/media.ts";
 import { BaseInterface } from "./base.ts";
 
@@ -369,10 +369,10 @@ export class Media extends BaseInterface<typeof Medias> {
         throw new Error("Unknown media backend");
     }
 
-    public getUrl(): URL {
+    public getUrl(): ProxiableUrl {
         const type = this.getPreferredMimeType();
 
-        return new URL(this.data.content[type]?.content ?? "");
+        return new ProxiableUrl(this.data.content[type]?.content ?? "");
     }
 
     /**
@@ -510,10 +510,10 @@ export class Media extends BaseInterface<typeof Medias> {
         return {
             id: this.data.id,
             type: this.getMastodonType(),
-            url: proxyUrl(new URL(data.content)).toString(),
+            url: this.getUrl().proxied,
             remote_url: null,
             preview_url: thumbnailData?.content
-                ? proxyUrl(new URL(thumbnailData.content)).toString()
+                ? new ProxiableUrl(thumbnailData.content).proxied
                 : null,
             meta: this.toApiMeta(),
             description: data.description || null,

classes/database/role.ts

@@ -1,4 +1,3 @@
-import { proxyUrl } from "@/response";
 import type { Role as RoleSchema } from "@versia/client/schemas";
 import type { RolePermission } from "@versia/client/schemas";
 import { db } from "@versia/kit/db";
@@ -14,11 +13,30 @@ import {
 } from "drizzle-orm";
 import type { z } from "zod";
 import { config } from "~/config.ts";
+import { ProxiableUrl } from "../media/url.ts";
 import { BaseInterface } from "./base.ts";
 type RoleType = InferSelectModel<typeof Roles>;
 
 export class Role extends BaseInterface<typeof Roles> {
     public static $type: RoleType;
+    public static defaultRole = new Role({
+        id: "default",
+        name: "Default",
+        permissions: config.permissions.default,
+        priority: 0,
+        description: "Default role for all users",
+        visible: false,
+        icon: null,
+    });
+    public static adminRole = new Role({
+        id: "admin",
+        name: "Admin",
+        permissions: config.permissions.admin,
+        priority: 2 ** 31 - 1,
+        description: "Default role for all administrators",
+        visible: false,
+        icon: null,
+    });
 
     public async reload(): Promise<void> {
         const reloaded = await Role.fromId(this.data.id);
@@ -59,24 +77,8 @@ export class Role extends BaseInterface<typeof Roles> {
 
     public static async getAll(): Promise<Role[]> {
         return (await Role.manyFromSql(undefined)).concat(
-            new Role({
+            Role.defaultRole,
-                id: "default",
+            Role.adminRole,
-                name: "Default",
-                permissions: config.permissions.default,
-                priority: 0,
-                description: "Default role for all users",
-                visible: false,
-                icon: null,
-            }),
-            new Role({
-                id: "admin",
-                name: "Admin",
-                permissions: config.permissions.admin,
-                priority: 2 ** 31 - 1,
-                description: "Default role for all administrators",
-                visible: false,
-                icon: null,
-            }),
         );
     }
 
@@ -215,7 +217,7 @@ export class Role extends BaseInterface<typeof Roles> {
             description: this.data.description ?? undefined,
             visible: this.data.visible,
             icon: this.data.icon
-                ? proxyUrl(new URL(this.data.icon)).toString()
+                ? new ProxiableUrl(this.data.icon).proxied
                 : undefined,
         };
     }

classes/database/user.ts

@@ -1,7 +1,6 @@
 import { idValidator } from "@/api";
 import { getBestContentType } from "@/content_types";
 import { randomString } from "@/math";
-import { proxyUrl } from "@/response";
 import { sentry } from "@/sentry";
 import { getLogger } from "@logtape/logtape";
 import type {
@@ -56,6 +55,7 @@ import { findManyUsers } from "~/classes/functions/user";
 import { searchManager } from "~/classes/search/search-manager";
 import { config } from "~/config.ts";
 import type { KnownEntity } from "~/types/api.ts";
+import { ProxiableUrl } from "../media/url.ts";
 import { DeliveryJobType, deliveryQueue } from "../queues/delivery.ts";
 import { PushJobType, pushQueue } from "../queues/push.ts";
 import { BaseInterface } from "./base.ts";
@@ -180,19 +180,14 @@ export class User extends BaseInterface<typeof Users, UserWithRelations> {
     }
 
     public getAllPermissions(): RolePermission[] {
-        return (
+        return Array.from(
-            this.data.roles
+            new Set([
-                .flatMap((role) => role.permissions)
+                ...this.data.roles.flatMap((role) => role.permissions),
                 // Add default permissions
-                .concat(config.permissions.default)
+                ...config.permissions.default,
                 // If admin, add admin permissions
-                .concat(this.data.isAdmin ? config.permissions.admin : [])
+                ...(this.data.isAdmin ? config.permissions.admin : []),
-                .reduce((acc, permission) => {
+            ]),
-                    if (!acc.includes(permission)) {
-                        acc.push(permission);
-                    }
-                    return acc;
-                }, [] as RolePermission[])
         );
     }
 
@@ -415,7 +410,7 @@ export class User extends BaseInterface<typeof Users, UserWithRelations> {
             id: string;
             name: string;
             url: string;
-            icon?: string;
+            icon?: ProxiableUrl;
         }[],
     ): Promise<
         {
@@ -445,9 +440,7 @@ export class User extends BaseInterface<typeof Users, UserWithRelations> {
                     id: issuer.id,
                     name: issuer.name,
                     url: issuer.url,
-                    icon: issuer.icon
+                    icon: issuer.icon?.proxied,
-                        ? proxyUrl(new URL(issuer.icon)).toString()
-                        : undefined,
                     server_id: account.serverId,
                 };
             })
@@ -831,11 +824,11 @@ export class User extends BaseInterface<typeof Users, UserWithRelations> {
      * Get the user's avatar in raw URL format
      * @returns The raw URL for the user's avatar
      */
-    public getAvatarUrl(): URL {
+    public getAvatarUrl(): ProxiableUrl {
         if (!this.avatar) {
             return (
                 config.defaults.avatar ||
-                new URL(
+                new ProxiableUrl(
                     `https://api.dicebear.com/8.x/${config.defaults.placeholder_style}/svg?seed=${this.data.username}`,
                 )
             );
@@ -928,10 +921,11 @@ export class User extends BaseInterface<typeof Users, UserWithRelations> {
      * Get the user's header in raw URL format
      * @returns The raw URL for the user's header
      */
-    public getHeaderUrl(): URL | null {
+    public getHeaderUrl(): ProxiableUrl | null {
         if (!this.header) {
             return config.defaults.header ?? null;
         }
+
         return this.header.getUrl();
     }
 
@@ -974,7 +968,8 @@ export class User extends BaseInterface<typeof Users, UserWithRelations> {
                 newUser.isBot ||
                 newUser.isLocked ||
                 newUser.endpoints ||
-                newUser.isDiscoverable)
+                newUser.isDiscoverable ||
+                newUser.isIndexable)
         ) {
             await this.federateToFollowers(this.toVersia());
         }
@@ -1050,6 +1045,19 @@ export class User extends BaseInterface<typeof Users, UserWithRelations> {
         return new FederationRequester(signatureConstructor);
     }
 
+    /**
+     * Get all remote followers of the user
+     * @returns The remote followers
+     */
+    private getRemoteFollowers(): Promise<User[]> {
+        return User.manyFromSql(
+            and(
+                sql`EXISTS (SELECT 1 FROM "Relationships" WHERE "Relationships"."subjectId" = ${this.id} AND "Relationships"."ownerId" = ${Users.id} AND "Relationships"."following" = true)`,
+                isNotNull(Users.instanceId),
+            ),
+        );
+    }
+
     /**
      * Federates an entity to all followers of the user
      *
@@ -1057,12 +1065,7 @@ export class User extends BaseInterface<typeof Users, UserWithRelations> {
      */
     public async federateToFollowers(entity: KnownEntity): Promise<void> {
         // Get followers
-        const followers = await User.manyFromSql(
+        const followers = await this.getRemoteFollowers();
-            and(
-                sql`EXISTS (SELECT 1 FROM "Relationships" WHERE "Relationships"."subjectId" = ${this.id} AND "Relationships"."ownerId" = ${Users.id} AND "Relationships"."following" = true)`,
-                isNotNull(Users.instanceId),
-            ),
-        );
 
         await deliveryQueue.addBulk(
             followers.map((follower) => ({
@@ -1130,10 +1133,8 @@ export class User extends BaseInterface<typeof Users, UserWithRelations> {
             url:
                 user.uri ||
                 new URL(`/@${user.username}`, config.http.base_url).toString(),
-            avatar: proxyUrl(this.getAvatarUrl()).toString(),
+            avatar: this.getAvatarUrl().proxied,
-            header: this.getHeaderUrl()
+            header: this.getHeaderUrl()?.proxied ?? "",
-                ? proxyUrl(this.getHeaderUrl() as URL).toString()
-                : "",
             locked: user.isLocked,
             created_at: new Date(user.createdAt).toISOString(),
             followers_count:
@@ -1154,10 +1155,8 @@ export class User extends BaseInterface<typeof Users, UserWithRelations> {
             bot: user.isBot,
             source: isOwnAccount ? user.source : undefined,
             // TODO: Add static avatar and header
-            avatar_static: proxyUrl(this.getAvatarUrl()).toString(),
+            avatar_static: this.getAvatarUrl().proxied,
-            header_static: this.getHeaderUrl()
+            header_static: this.getHeaderUrl()?.proxied ?? "",
-                ? proxyUrl(this.getHeaderUrl() as URL).toString()
-                : "",
             acct: this.getAcct(),
             // TODO: Add these fields
             limited: false,
@@ -1168,33 +1167,8 @@ export class User extends BaseInterface<typeof Users, UserWithRelations> {
             mute_expires_at: null,
             roles: user.roles
                 .map((role) => new Role(role))
-                .concat(
+                .concat(Role.defaultRole)
-                    new Role({
+                .concat(user.isAdmin ? Role.adminRole : [])
-                        id: "default",
-                        name: "Default",
-                        permissions: config.permissions.default,
-                        priority: 0,
-                        description: "Default role for all users",
-                        visible: false,
-                        icon: null,
-                    }),
-                )
-                .concat(
-                    user.isAdmin
-                        ? [
-                              new Role({
-                                  id: "admin",
-                                  name: "Admin",
-                                  permissions: config.permissions.admin,
-                                  priority: 2 ** 31 - 1,
-                                  description:
-                                      "Default role for all administrators",
-                                  visible: false,
-                                  icon: null,
-                              }),
-                          ]
-                        : [],
-                )
                 .map((r) => r.toApi()),
             group: false,
             // TODO

classes/media/url.ts

@@ -0,0 +1,25 @@
+import { config } from "~/config.ts";
+
+export class ProxiableUrl extends URL {
+    private isAllowedOrigin(): boolean {
+        const allowedOrigins: URL[] = [config.http.base_url].concat(
+            config.s3?.public_url ?? [],
+        );
+
+        return allowedOrigins.some((origin) =>
+            this.hostname.endsWith(origin.hostname),
+        );
+    }
+
+    public get proxied(): string {
+        // Don't proxy from CDN and self, since those sources are trusted
+        if (this.isAllowedOrigin()) {
+            return this.href;
+        }
+
+        const urlAsBase64Url = Buffer.from(this.href).toString("base64url");
+
+        return new URL(`/media/proxy/${urlAsBase64Url}`, config.http.base_url)
+            .href;
+    }
+}

plugins/openid/index.ts

@@ -5,7 +5,7 @@ import { getCookie } from "hono/cookie";
 import { jwtVerify } from "jose";
 import { JOSEError, JWTExpired } from "jose/errors";
 import { z } from "zod";
-import { keyPair, sensitiveString } from "~/classes/config/schema.ts";
+import { url, keyPair, sensitiveString } from "~/classes/config/schema.ts";
 import { ApiError } from "~/classes/errors/api-error.ts";
 import authorizeRoute from "./routes/authorize.ts";
 import jwksRoute from "./routes/jwks.ts";
@@ -27,7 +27,7 @@ const configSchema = z.object({
                 url: z.string().min(1),
                 client_id: z.string().min(1),
                 client_secret: sensitiveString,
-                icon: z.string().min(1).optional(),
+                icon: url.optional(),
             }),
         )
         .default([]),

plugins/openid/routes/sso/:id/index.ts

@@ -1,5 +1,4 @@
 import { auth, handleZodError } from "@/api";
-import { proxyUrl } from "@/response";
 import { RolePermission } from "@versia/client/schemas";
 import { db } from "@versia/kit/db";
 import { type SQL, eq } from "@versia/kit/drizzle";
@@ -77,9 +76,7 @@ export default (plugin: PluginType): void => {
                     {
                         id: issuer.id,
                         name: issuer.name,
-                        icon: issuer.icon
+                        icon: issuer.icon?.proxied,
-                            ? proxyUrl(new URL(issuer.icon))
-                            : undefined,
                     },
                     200,
                 );

utils/response.ts

@@ -1,17 +0,0 @@
-import { config } from "~/config.ts";
-
-export type Json =
-    | string
-    | number
-    | boolean
-    | null
-    | undefined
-    | Json[]
-    | { [key: string]: Json };
-
-export const proxyUrl = (url: URL): URL => {
-    const urlAsBase64Url = Buffer.from(url.toString() || "").toString(
-        "base64url",
-    );
-    return new URL(`/media/proxy/${urlAsBase64Url}`, config.http.base_url);
-};

utils/sanitization.ts

@@ -1,6 +1,6 @@
 import { stringifyEntitiesLight } from "stringify-entities";
 import xss, { type IFilterXSSOptions } from "xss";
-import { proxyUrl } from "./response.ts";
+import { ProxiableUrl } from "~/classes/media/url.ts";
 
 export const sanitizedHtmlStrip = (html: string): Promise<string> => {
     return sanitizeHtml(html, {
@@ -137,9 +137,9 @@ export const sanitizeHtml = async (
                 element.setAttribute(
                     "src",
                     element.getAttribute("src")
-                        ? proxyUrl(
+                        ? new ProxiableUrl(
-                              new URL(element.getAttribute("src") as string),
+                              element.getAttribute("src") as string,
-                          ).toString()
+                          ).proxied
                         : "",
                 );
             },