refactor(database): ♻️ Make emojis use a Media instead of just rawdogging the URI

api/api/v1/emojis/:id/index.ts

@@ -1,6 +1,7 @@
 import { apiRoute, auth, emojiValidator, jsonOrForm } from "@/api";
 import { mimeLookup } from "@/content_types";
 import { createRoute } from "@hono/zod-openapi";
+import type { ContentFormat } from "@versia/federation/types";
 import { Emoji, Media, db } from "@versia/kit/db";
 import { Emojis, RolePermissions } from "@versia/kit/tables";
 import { eq } from "drizzle-orm";
@@ -230,8 +231,6 @@ export default apiRoute((app) => {
             );
         }
 
-        const mediaManager = new MediaManager(config);
-
         const {
             global: emojiGlobal,
             alt,
@@ -248,11 +247,12 @@ export default apiRoute((app) => {
             );
         }
 
+        const modifiedMedia = structuredClone(emoji.data.media);
         const modified = structuredClone(emoji.data);
 
         if (element) {
             // Check of emoji is an image
-            let contentType =
+            const contentType =
                 element instanceof File
                     ? element.type
                     : await mimeLookup(element);
@@ -265,23 +265,33 @@ export default apiRoute((app) => {
                 );
             }
 
-            let url = "";
+            let contentFormat: ContentFormat | undefined;
 
             if (element instanceof File) {
-                const uploaded = await mediaManager.addFile(element);
+                const mediaManager = new MediaManager(config);
 
-                url = Media.getUrl(uploaded.path);
+                const { uploadedFile, path } =
-                contentType = uploaded.uploadedFile.type;
+                    await mediaManager.addFile(element);
+
+                contentFormat = await Media.fileToContentFormat(
+                    uploadedFile,
+                    Media.getUrl(path),
+                    { description: alt },
+                );
             } else {
-                url = element;
+                contentFormat = {
+                    [contentType]: {
+                        content: element,
+                        remote: true,
+                        description: alt,
+                    },
+                };
             }
 
-            modified.url = url;
+            modifiedMedia.content = contentFormat;
-            modified.contentType = contentType;
         }
 
         modified.shortcode = shortcode ?? modified.shortcode;
-        modified.alt = alt ?? modified.alt;
         modified.category = category ?? modified.category;
 
         if (emojiGlobal !== undefined) {
@@ -317,7 +327,7 @@ export default apiRoute((app) => {
 
         const mediaManager = new MediaManager(config);
 
-        await mediaManager.deleteFileByUrl(emoji.data.url);
+        await mediaManager.deleteFileByUrl(emoji.media.getUrl());
 
         await db.delete(Emojis).where(eq(Emojis.id, id));
 

api/api/v1/emojis/index.ts

@@ -1,6 +1,7 @@
 import { apiRoute, auth, emojiValidator, jsonOrForm } from "@/api";
 import { mimeLookup } from "@/content_types";
 import { createRoute } from "@hono/zod-openapi";
+import type { ContentFormat } from "@versia/federation/types";
 import { Emoji, Media } from "@versia/kit/db";
 import { Emojis, RolePermissions } from "@versia/kit/tables";
 import { and, eq, isNull, or } from "drizzle-orm";
@@ -130,10 +131,8 @@ export default apiRoute((app) =>
             );
         }
 
-        let url = "";
-
         // Check of emoji is an image
-        let contentType =
+        const contentType =
             element instanceof File ? element.type : await mimeLookup(element);
 
         if (!contentType.startsWith("image/")) {
@@ -144,25 +143,38 @@ export default apiRoute((app) =>
             );
         }
 
+        let contentFormat: ContentFormat | undefined;
+
         if (element instanceof File) {
             const mediaManager = new MediaManager(config);
 
-            const uploaded = await mediaManager.addFile(element);
+            const { uploadedFile, path } = await mediaManager.addFile(element);
 
-            url = Media.getUrl(uploaded.path);
+            contentFormat = await Media.fileToContentFormat(
-            contentType = uploaded.uploadedFile.type;
+                uploadedFile,
+                Media.getUrl(path),
+                { description: alt },
+            );
         } else {
-            url = element;
+            contentFormat = {
+                [contentType]: {
+                    content: element,
+                    remote: true,
+                    description: alt,
+                },
+            };
         }
 
+        const media = await Media.insert({
+            content: contentFormat,
+        });
+
         const emoji = await Emoji.insert({
             shortcode,
-            url,
+            mediaId: media.id,
             visibleInPicker: true,
             ownerId: global ? null : user.id,
             category,
-            contentType,
-            alt,
         });
 
         return context.json(emoji.toApi(), 201);

api/api/v1/statuses/index.test.ts

@@ -1,12 +1,13 @@
 import { afterAll, beforeAll, describe, expect, test } from "bun:test";
 import type { Status as ApiStatus } from "@versia/client/types";
-import { db } from "@versia/kit/db";
+import { Media, db } from "@versia/kit/db";
 import { Emojis } from "@versia/kit/tables";
 import { eq } from "drizzle-orm";
 import { config } from "~/packages/config-manager/index.ts";
 import { fakeRequest, getTestUsers } from "~/tests/utils";
 
 const { users, tokens, deleteUsers } = await getTestUsers(5);
+let media: Media;
 
 afterAll(async () => {
     await deleteUsers();
@@ -14,10 +15,17 @@ afterAll(async () => {
 });
 
 beforeAll(async () => {
+    media = await Media.insert({
+        content: {
+            "image/png": {
+                content: "https://example.com/test.png",
+                remote: true,
+            },
+        },
+    });
     await db.insert(Emojis).values({
-        contentType: "image/png",
         shortcode: "test",
-        url: "https://example.com/test.png",
+        mediaId: media.id,
         visibleInPicker: true,
     });
 });

classes/database/emoji.ts

@@ -2,8 +2,8 @@ import { emojiValidatorWithColons, emojiValidatorWithIdentifiers } from "@/api";
 import { proxyUrl } from "@/response";
 import type { Emoji as APIEmoji } from "@versia/client/types";
 import type { CustomEmojiExtension } from "@versia/federation/types";
-import { type Instance, db } from "@versia/kit/db";
+import { type Instance, Media, db } from "@versia/kit/db";
-import { Emojis, type Instances } from "@versia/kit/tables";
+import { Emojis, type Instances, type Medias } from "@versia/kit/tables";
 import {
     type InferInsertModel,
     type InferSelectModel,
@@ -17,11 +17,12 @@ import {
 import { z } from "zod";
 import { BaseInterface } from "./base.ts";
 
-type EmojiWithInstance = InferSelectModel<typeof Emojis> & {
+type EmojiType = InferSelectModel<typeof Emojis> & {
+    media: InferSelectModel<typeof Medias>;
     instance: InferSelectModel<typeof Instances> | null;
 };
 
-export class Emoji extends BaseInterface<typeof Emojis, EmojiWithInstance> {
+export class Emoji extends BaseInterface<typeof Emojis, EmojiType> {
     public static schema = z.object({
         id: z.string(),
         shortcode: z.string(),
@@ -32,7 +33,13 @@ export class Emoji extends BaseInterface<typeof Emojis, EmojiWithInstance> {
         global: z.boolean(),
     });
 
-    public static $type: EmojiWithInstance;
+    public static $type: EmojiType;
+    public media: Media;
+
+    public constructor(data: EmojiType) {
+        super(data);
+        this.media = new Media(data.media);
+    }
 
     public async reload(): Promise<void> {
         const reloaded = await Emoji.fromId(this.data.id);
@@ -65,6 +72,7 @@ export class Emoji extends BaseInterface<typeof Emojis, EmojiWithInstance> {
             orderBy,
             with: {
                 instance: true,
+                media: true,
             },
         });
 
@@ -86,15 +94,13 @@ export class Emoji extends BaseInterface<typeof Emojis, EmojiWithInstance> {
             orderBy,
             limit,
             offset,
-            with: { ...extra?.with, instance: true },
+            with: { ...extra?.with, instance: true, media: true },
         });
 
         return found.map((s) => new Emoji(s));
     }
 
-    public async update(
+    public async update(newEmoji: Partial<EmojiType>): Promise<EmojiType> {
-        newEmoji: Partial<EmojiWithInstance>,
-    ): Promise<EmojiWithInstance> {
         await db.update(Emojis).set(newEmoji).where(eq(Emojis.id, this.id));
 
         const updated = await Emoji.fromId(this.data.id);
@@ -107,7 +113,7 @@ export class Emoji extends BaseInterface<typeof Emojis, EmojiWithInstance> {
         return updated.data;
     }
 
-    public save(): Promise<EmojiWithInstance> {
+    public save(): Promise<EmojiType> {
         return this.update(this.data);
     }
 
@@ -182,29 +188,25 @@ export class Emoji extends BaseInterface<typeof Emojis, EmojiWithInstance> {
         return {
             id: this.id,
             shortcode: this.data.shortcode,
-            static_url: proxyUrl(this.data.url) ?? "", // TODO: Add static version
+            static_url: proxyUrl(this.media.getUrl()) ?? "", // TODO: Add static version
-            url: proxyUrl(this.data.url) ?? "",
+            url: proxyUrl(this.media.getUrl()) ?? "",
             visible_in_picker: this.data.visibleInPicker,
             category: this.data.category ?? undefined,
             global: this.data.ownerId === null,
-            description: this.data.alt ?? undefined,
+            description:
+                this.media.data.content[this.media.getPreferredMimeType()]
+                    .description ?? undefined,
         };
     }
 
     public toVersia(): CustomEmojiExtension["emojis"][0] {
         return {
             name: `:${this.data.shortcode}:`,
-            url: {
+            url: this.media.toVersia(),
-                [this.data.contentType]: {
-                    content: this.data.url,
-                    description: this.data.alt || undefined,
-                    remote: true,
-                },
-            },
         };
     }
 
-    public static fromVersia(
+    public static async fromVersia(
         emoji: CustomEmojiExtension["emojis"][0],
         instance: Instance,
     ): Promise<Emoji> {
@@ -217,11 +219,11 @@ export class Emoji extends BaseInterface<typeof Emojis, EmojiWithInstance> {
             throw new Error("Could not extract shortcode from emoji name");
         }
 
+        const media = await Media.fromVersia(emoji.url);
+
         return Emoji.insert({
             shortcode,
-            url: Object.entries(emoji.url)[0][1].content,
+            mediaId: media.id,
-            alt: Object.entries(emoji.url)[0][1].description || undefined,
-            contentType: Object.keys(emoji.url)[0],
             visibleInPicker: true,
             instanceId: instance.id,
         });

classes/database/reaction.ts

@@ -67,6 +67,7 @@ export class Reaction extends BaseInterface<typeof Reactions, ReactionType> {
                 emoji: {
                     with: {
                         instance: true,
+                        media: true,
                     },
                 },
                 author: true,
@@ -98,6 +99,7 @@ export class Reaction extends BaseInterface<typeof Reactions, ReactionType> {
                 emoji: {
                     with: {
                         instance: true,
+                        media: true,
                     },
                 },
                 author: true,

classes/functions/status.ts

@@ -48,6 +48,7 @@ export const findManyNotes = async (
                     emoji: {
                         with: {
                             instance: true,
+                            media: true,
                         },
                     },
                 },
@@ -79,6 +80,7 @@ export const findManyNotes = async (
                             emoji: {
                                 with: {
                                     instance: true,
+                                    media: true,
                                 },
                             },
                         },

classes/functions/user.ts

@@ -17,6 +17,7 @@ export const userRelations = {
             emoji: {
                 with: {
                     instance: true,
+                    media: true,
                 },
             },
         },

cli/classes.ts

@@ -1,9 +1,9 @@
 import { parseUserAddress, userAddressValidator } from "@/api";
 import { Args, type Command, Flags, type Interfaces } from "@oclif/core";
-import { type Emoji, Instance, User, db } from "@versia/kit/db";
+import { Emoji, Instance, User } from "@versia/kit/db";
 import { Emojis, Instances, Users } from "@versia/kit/tables";
 import chalk from "chalk";
-import { and, eq, getTableColumns, like } from "drizzle-orm";
+import { and, eq, inArray, like } from "drizzle-orm";
 import { BaseCommand } from "./base.ts";
 
 export type FlagsType<T extends typeof Command> = Interfaces.InferredFlags<
@@ -203,14 +203,7 @@ export abstract class EmojiFinderCommand<
         this.args = args as ArgsType<T>;
     }
 
-    public async findEmojis(): Promise<
+    public async findEmojis(): Promise<Emoji[]> {
-        Omit<
-            typeof Emoji.$type & {
-                instanceUrl: string | null;
-            },
-            "instance"
-        >[]
-    > {
         // Check if there are asterisks in the identifier but no pattern flag, warn the user if so
         if (this.args.identifier.includes("*") && !this.flags.pattern) {
             this.log(
@@ -228,22 +221,26 @@ export abstract class EmojiFinderCommand<
             ? this.args.identifier.replace(/\*/g, "%")
             : this.args.identifier;
 
-        return await db
+        const instanceIds =
-            .select({
+            this.flags.type === "instance"
-                ...getTableColumns(Emojis),
+                ? (
-                instanceUrl: Instances.baseUrl,
+                      await Instance.manyFromSql(
-            })
+                          operator(Instances.baseUrl, identifier),
-            .from(Emojis)
+                      )
-            .leftJoin(Instances, eq(Emojis.instanceId, Instances.id))
+                  ).map((instance) => instance.id)
-            .where(
+                : undefined;
-                and(
+
-                    this.flags.type === "shortcode"
+        return await Emoji.manyFromSql(
-                        ? operator(Emojis.shortcode, identifier)
+            and(
-                        : undefined,
+                this.flags.type === "shortcode"
-                    this.flags.type === "instance"
+                    ? operator(Emojis.shortcode, identifier)
-                        ? operator(Instances.baseUrl, identifier)
+                    : undefined,
-                        : undefined,
+                instanceIds && instanceIds.length > 0
-                ),
+                    ? inArray(Emojis.instanceId, instanceIds)
-            );
+                    : undefined,
+            ),
+            undefined,
+            this.flags.limit,
+        );
     }
 }

cli/commands/emoji/add.ts

@@ -4,7 +4,6 @@ import { Emojis } from "@versia/kit/tables";
 import chalk from "chalk";
 import { and, eq, isNull } from "drizzle-orm";
 import ora from "ora";
-import { MediaManager } from "~/classes/media/media-manager";
 import { BaseCommand } from "~/cli/base";
 import { config } from "~/packages/config-manager";
 
@@ -97,35 +96,22 @@ export default class EmojiAdd extends BaseCommand<typeof EmojiAdd> {
             );
         }
 
-        const mediaManager = new MediaManager(config);
-
         const spinner = ora("Uploading emoji").start();
 
-        const uploaded = await mediaManager.addFile(file).catch((e: Error) => {
+        const media = await Media.fromFile(file);
-            spinner.fail();
-            this.log(`${chalk.red("✗")} Error: ${chalk.red(e.message)}`);
-            return null;
-        });
-
-        if (!uploaded) {
-            return this.exit(1);
-        }
 
         spinner.succeed();
 
         await Emoji.insert({
             shortcode: args.shortcode,
-            url: Media.getUrl(uploaded.path),
+            mediaId: media.id,
             visibleInPicker: true,
-            contentType: uploaded.uploadedFile.type,
         });
 
         this.log(
             `${chalk.green("✓")} Created emoji ${chalk.green(
                 args.shortcode,
-            )} with url ${chalk.blue(
+            )} with url ${chalk.blue(chalk.underline(media.getUrl()))}`,
-                chalk.underline(Media.getUrl(uploaded.path)),
-            )}`,
         );
 
         this.exit(0);

cli/commands/emoji/delete.ts

@@ -55,13 +55,10 @@ export default class EmojiDelete extends EmojiFinderCommand<
 
         flags.print &&
             this.log(
-                formatArray(emojis, [
+                formatArray(
-                    "id",
+                    emojis.map((e) => e.data),
-                    "shortcode",
+                    ["id", "shortcode", "alt", "contentType", "instanceUrl"],
-                    "alt",
+                ),
-                    "contentType",
-                    "instanceUrl",
-                ]),
             );
 
         if (flags.confirm) {
@@ -80,13 +77,13 @@ export default class EmojiDelete extends EmojiFinderCommand<
         const spinner = ora("Deleting emoji(s)").start();
 
         for (const emoji of emojis) {
-            spinner.text = `Deleting emoji ${chalk.gray(emoji.shortcode)} (${
+            spinner.text = `Deleting emoji ${chalk.gray(emoji.data.shortcode)} (${
                 emojis.findIndex((e) => e.id === emoji.id) + 1
             }/${emojis.length})`;
 
             const mediaManager = new MediaManager(config);
 
-            await mediaManager.deleteFileByUrl(emoji.url);
+            await mediaManager.deleteFileByUrl(emoji.media.getUrl());
 
             await db.delete(Emojis).where(eq(Emojis.id, emoji.id));
         }

cli/commands/emoji/import.ts

@@ -6,7 +6,6 @@ import { and, inArray, isNull } from "drizzle-orm";
 import { lookup } from "mime-types";
 import ora from "ora";
 import { unzip } from "unzipit";
-import { MediaManager } from "~/classes/media/media-manager";
 import { BaseCommand } from "~/cli/base";
 import { config } from "~/packages/config-manager";
 
@@ -169,8 +168,6 @@ export default class EmojiImport extends BaseCommand<typeof EmojiImport> {
 
         const importSpinner = ora("Importing emojis").start();
 
-        const mediaManager = new MediaManager(config);
-
         const successfullyImported: MetaType["emojis"] = [];
 
         for (const emoji of newEmojis) {
@@ -197,26 +194,12 @@ export default class EmojiImport extends BaseCommand<typeof EmojiImport> {
                 type: contentType,
             });
 
-            const uploaded = await mediaManager
+            const media = await Media.fromFile(newFile);
-                .addFile(newFile)
-                .catch((e: Error) => {
-                    this.log(
-                        `${chalk.red("✗")} Error uploading ${chalk.red(
-                            emoji.emoji.name,
-                        )}: ${chalk.red(e.message)}`,
-                    );
-                    return null;
-                });
-
-            if (!uploaded) {
-                continue;
-            }
 
             await Emoji.insert({
                 shortcode: emoji.emoji.name,
-                url: Media.getUrl(uploaded.path),
+                mediaId: media.id,
                 visibleInPicker: true,
-                contentType: uploaded.uploadedFile.type,
             });
 
             successfullyImported.push(emoji);

drizzle.config.ts

@@ -1,20 +1,25 @@
 import type { Config } from "drizzle-kit";
+import { config } from "./packages/config-manager/index.ts";
 
+/**
+ * Drizzle can't properly resolve imports with top-level await, so uncomment
+ * this line when generating migrations.
+ */
 export default {
     dialect: "postgresql",
     out: "./drizzle/migrations",
     schema: "./drizzle/schema.ts",
     dbCredentials: {
-        host: "localhost",
+        /* host: "localhost",
         port: 40000,
         user: "lysand",
         password: "lysand",
-        database: "lysand",
+        database: "lysand", */
-        /* host: config.database.host,
+        host: config.database.host,
         port: Number(config.database.port),
         user: config.database.username,
         password: config.database.password,
-        database: config.database.database, */
+        database: config.database.database,
     },
     // Print all statements
     verbose: true,

drizzle/migrations/0043_mute_jigsaw.sql

@@ -0,0 +1,5 @@
+ALTER TABLE "Emojis" ADD COLUMN "mediaId" uuid;--> statement-breakpoint
+ALTER TABLE "Emojis" ADD CONSTRAINT "Emojis_mediaId_Medias_id_fk" FOREIGN KEY ("mediaId") REFERENCES "public"."Medias"("id") ON DELETE cascade ON UPDATE cascade;--> statement-breakpoint
+ALTER TABLE "Emojis" DROP COLUMN "url";--> statement-breakpoint
+ALTER TABLE "Emojis" DROP COLUMN "alt";--> statement-breakpoint
+ALTER TABLE "Emojis" DROP COLUMN "content_type";

drizzle/migrations/0044_quiet_jasper_sitwell.sql

@@ -0,0 +1 @@
+ALTER TABLE "Emojis" ALTER COLUMN "mediaId" SET NOT NULL;

drizzle/migrations/meta/_journal.json

@@ -302,6 +302,20 @@
             "when": 1737660317024,
             "tag": "0042_swift_the_phantom",
             "breakpoints": true
+        },
+        {
+            "idx": 43,
+            "version": "7",
+            "when": 1738080562679,
+            "tag": "0043_mute_jigsaw",
+            "breakpoints": true
+        },
+        {
+            "idx": 44,
+            "version": "7",
+            "when": 1738082427051,
+            "tag": "0044_quiet_jasper_sitwell",
+            "breakpoints": true
         }
     ]
 }

drizzle/schema.ts

@@ -52,10 +52,13 @@ export const Challenges = pgTable("Challenges", {
 export const Emojis = pgTable("Emojis", {
     id: id(),
     shortcode: text("shortcode").notNull(),
-    url: text("url").notNull(),
+    mediaId: uuid("mediaId")
+        .references(() => Medias.id, {
+            onDelete: "cascade",
+            onUpdate: "cascade",
+        })
+        .notNull(),
     visibleInPicker: boolean("visible_in_picker").notNull(),
-    alt: text("alt"),
-    contentType: text("content_type").notNull(),
     instanceId: uuid("instanceId").references(() => Instances.id, {
         onDelete: "cascade",
         onUpdate: "cascade",
@@ -68,6 +71,10 @@ export const Emojis = pgTable("Emojis", {
 });
 
 export const EmojisRelations = relations(Emojis, ({ one, many }) => ({
+    media: one(Medias, {
+        fields: [Emojis.mediaId],
+        references: [Medias.id],
+    }),
     instance: one(Instances, {
         fields: [Emojis.instanceId],
         references: [Instances.id],
@@ -357,6 +364,7 @@ export const Medias = pgTable("Medias", {
 
 export const MediasRelations = relations(Medias, ({ many }) => ({
     notes: many(Notes),
+    emojis: many(Emojis),
 }));
 
 export const Notifications = pgTable("Notifications", {