versia-pub/server
2
0
Fork 0
mirror of https://github.com/versia-pub/server.git synced 2025-12-06 08:28:19 +01:00
Code Issues Actions Packages Projects Releases Wiki Activity

refactor(database): ♻️ Make emojis use a Media instead of just rawdogging the URI

Browse source
This commit is contained in:
Jesse Wierzbinski 2025-01-28 17:43:43 +01:00
parent c7aae24d42
commit cf1104d762
No known key found for this signature in database
18 changed files with 4823 additions and 128 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

34
api/api/v1/emojis/:id/index.ts
View file

@ -1,6 +1,7 @@
import { apiRoute, auth, emojiValidator, jsonOrForm } from "@/api";
import { mimeLookup } from "@/content_types";
import { createRoute } from "@hono/zod-openapi";
import type { ContentFormat } from "@versia/federation/types";
import { Emoji, Media, db } from "@versia/kit/db";
import { Emojis, RolePermissions } from "@versia/kit/tables";
import { eq } from "drizzle-orm";
@ -230,8 +231,6 @@ export default apiRoute((app) => {
);
}
const mediaManager = new MediaManager(config);
const {
global: emojiGlobal,
alt,
@ -248,11 +247,12 @@ export default apiRoute((app) => {
);
}
const modifiedMedia = structuredClone(emoji.data.media);
const modified = structuredClone(emoji.data);
if (element) {
// Check of emoji is an image
let contentType =
const contentType =
element instanceof File
? element.type
: await mimeLookup(element);
@ -265,23 +265,33 @@ export default apiRoute((app) => {
);
}
let url = "";
let contentFormat: ContentFormat | undefined;
if (element instanceof File) {
const uploaded = await mediaManager.addFile(element);
const mediaManager = new MediaManager(config);
url = Media.getUrl(uploaded.path);
contentType = uploaded.uploadedFile.type;
const { uploadedFile, path } =
await mediaManager.addFile(element);
contentFormat = await Media.fileToContentFormat(
uploadedFile,
Media.getUrl(path),
{ description: alt },
);
} else {
url = element;
contentFormat = {
[contentType]: {
content: element,
remote: true,
description: alt,
},
};
}
modified.url = url;
modified.contentType = contentType;
modifiedMedia.content = contentFormat;
}
modified.shortcode = shortcode ?? modified.shortcode;
modified.alt = alt ?? modified.alt;
modified.category = category ?? modified.category;
if (emojiGlobal !== undefined) {
@ -317,7 +327,7 @@ export default apiRoute((app) => {
const mediaManager = new MediaManager(config);
await mediaManager.deleteFileByUrl(emoji.data.url);
await mediaManager.deleteFileByUrl(emoji.media.getUrl());
await db.delete(Emojis).where(eq(Emojis.id, id));

32
api/api/v1/emojis/index.ts
View file

@ -1,6 +1,7 @@
import { apiRoute, auth, emojiValidator, jsonOrForm } from "@/api";
import { mimeLookup } from "@/content_types";
import { createRoute } from "@hono/zod-openapi";
import type { ContentFormat } from "@versia/federation/types";
import { Emoji, Media } from "@versia/kit/db";
import { Emojis, RolePermissions } from "@versia/kit/tables";
import { and, eq, isNull, or } from "drizzle-orm";
@ -130,10 +131,8 @@ export default apiRoute((app) =>
);
}
let url = "";
// Check of emoji is an image
let contentType =
const contentType =
element instanceof File ? element.type : await mimeLookup(element);
if (!contentType.startsWith("image/")) {
@ -144,25 +143,38 @@ export default apiRoute((app) =>
);
}
let contentFormat: ContentFormat | undefined;
if (element instanceof File) {
const mediaManager = new MediaManager(config);
const uploaded = await mediaManager.addFile(element);
const { uploadedFile, path } = await mediaManager.addFile(element);
url = Media.getUrl(uploaded.path);
contentType = uploaded.uploadedFile.type;
contentFormat = await Media.fileToContentFormat(
uploadedFile,
Media.getUrl(path),
{ description: alt },
);
} else {
url = element;
contentFormat = {
[contentType]: {
content: element,
remote: true,
description: alt,
},
};
}
const media = await Media.insert({
content: contentFormat,
});
const emoji = await Emoji.insert({
shortcode,
url,
mediaId: media.id,
visibleInPicker: true,
ownerId: global ? null : user.id,
category,
contentType,
alt,
});
return context.json(emoji.toApi(), 201);

14
api/api/v1/statuses/index.test.ts
View file

@ -1,12 +1,13 @@
import { afterAll, beforeAll, describe, expect, test } from "bun:test";
import type { Status as ApiStatus } from "@versia/client/types";
import { db } from "@versia/kit/db";
import { Media, db } from "@versia/kit/db";
import { Emojis } from "@versia/kit/tables";
import { eq } from "drizzle-orm";
import { config } from "~/packages/config-manager/index.ts";
import { fakeRequest, getTestUsers } from "~/tests/utils";
const { users, tokens, deleteUsers } = await getTestUsers(5);
let media: Media;
afterAll(async () => {
await deleteUsers();
@ -14,10 +15,17 @@ afterAll(async () => {
});
beforeAll(async () => {
media = await Media.insert({
content: {
"image/png": {
content: "https://example.com/test.png",
remote: true,
},
},
});
await db.insert(Emojis).values({
contentType: "image/png",
shortcode: "test",
url: "https://example.com/test.png",
mediaId: media.id,
visibleInPicker: true,
});
});

50
classes/database/emoji.ts
View file

@ -2,8 +2,8 @@ import { emojiValidatorWithColons, emojiValidatorWithIdentifiers } from "@/api";
import { proxyUrl } from "@/response";
import type { Emoji as APIEmoji } from "@versia/client/types";
import type { CustomEmojiExtension } from "@versia/federation/types";
import { type Instance, db } from "@versia/kit/db";
import { Emojis, type Instances } from "@versia/kit/tables";
import { type Instance, Media, db } from "@versia/kit/db";
import { Emojis, type Instances, type Medias } from "@versia/kit/tables";
import {
type InferInsertModel,
type InferSelectModel,
@ -17,11 +17,12 @@ import {
import { z } from "zod";
import { BaseInterface } from "./base.ts";
type EmojiWithInstance = InferSelectModel<typeof Emojis> & {
type EmojiType = InferSelectModel<typeof Emojis> & {
media: InferSelectModel<typeof Medias>;
instance: InferSelectModel<typeof Instances> | null;
};
export class Emoji extends BaseInterface<typeof Emojis, EmojiWithInstance> {
export class Emoji extends BaseInterface<typeof Emojis, EmojiType> {
public static schema = z.object({
id: z.string(),
shortcode: z.string(),
@ -32,7 +33,13 @@ export class Emoji extends BaseInterface<typeof Emojis, EmojiWithInstance> {
global: z.boolean(),
});
public static $type: EmojiWithInstance;
public static $type: EmojiType;
public media: Media;
public constructor(data: EmojiType) {
super(data);
this.media = new Media(data.media);
}
public async reload(): Promise<void> {
const reloaded = await Emoji.fromId(this.data.id);
@ -65,6 +72,7 @@ export class Emoji extends BaseInterface<typeof Emojis, EmojiWithInstance> {
orderBy,
with: {
instance: true,
media: true,
},
});
@ -86,15 +94,13 @@ export class Emoji extends BaseInterface<typeof Emojis, EmojiWithInstance> {
orderBy,
limit,
offset,
with: { ...extra?.with, instance: true },
with: { ...extra?.with, instance: true, media: true },
});
return found.map((s) => new Emoji(s));
}
public async update(
newEmoji: Partial<EmojiWithInstance>,
): Promise<EmojiWithInstance> {
public async update(newEmoji: Partial<EmojiType>): Promise<EmojiType> {
await db.update(Emojis).set(newEmoji).where(eq(Emojis.id, this.id));
const updated = await Emoji.fromId(this.data.id);
@ -107,7 +113,7 @@ export class Emoji extends BaseInterface<typeof Emojis, EmojiWithInstance> {
return updated.data;
}
public save(): Promise<EmojiWithInstance> {
public save(): Promise<EmojiType> {
return this.update(this.data);
}
@ -182,29 +188,25 @@ export class Emoji extends BaseInterface<typeof Emojis, EmojiWithInstance> {
return {
id: this.id,
shortcode: this.data.shortcode,
static_url: proxyUrl(this.data.url) ?? "", // TODO: Add static version
url: proxyUrl(this.data.url) ?? "",
static_url: proxyUrl(this.media.getUrl()) ?? "", // TODO: Add static version
url: proxyUrl(this.media.getUrl()) ?? "",
visible_in_picker: this.data.visibleInPicker,
category: this.data.category ?? undefined,
global: this.data.ownerId === null,
description: this.data.alt ?? undefined,
description:
this.media.data.content[this.media.getPreferredMimeType()]
.description ?? undefined,
};
}
public toVersia(): CustomEmojiExtension["emojis"][0] {
return {
name: `:${this.data.shortcode}:`,
url: {
[this.data.contentType]: {
content: this.data.url,
description: this.data.alt || undefined,
remote: true,
},
},
url: this.media.toVersia(),
};
}
public static fromVersia(
public static async fromVersia(
emoji: CustomEmojiExtension["emojis"][0],
instance: Instance,
): Promise<Emoji> {
@ -217,11 +219,11 @@ export class Emoji extends BaseInterface<typeof Emojis, EmojiWithInstance> {
throw new Error("Could not extract shortcode from emoji name");
}
const media = await Media.fromVersia(emoji.url);
return Emoji.insert({
shortcode,
url: Object.entries(emoji.url)[0][1].content,
alt: Object.entries(emoji.url)[0][1].description || undefined,
contentType: Object.keys(emoji.url)[0],
mediaId: media.id,
visibleInPicker: true,
instanceId: instance.id,
});

2
classes/database/reaction.ts
View file

@ -67,6 +67,7 @@ export class Reaction extends BaseInterface<typeof Reactions, ReactionType> {
emoji: {
with: {
instance: true,
media: true,
},
},
author: true,
@ -98,6 +99,7 @@ export class Reaction extends BaseInterface<typeof Reactions, ReactionType> {
emoji: {
with: {
instance: true,
media: true,
},
},
author: true,

2
classes/functions/status.ts
View file

@ -48,6 +48,7 @@ export const findManyNotes = async (
emoji: {
with: {
instance: true,
media: true,
},
},
},
@ -79,6 +80,7 @@ export const findManyNotes = async (
emoji: {
with: {
instance: true,
media: true,
},
},
},

1
classes/functions/user.ts
View file

@ -17,6 +17,7 @@ export const userRelations = {
emoji: {
with: {
instance: true,
media: true,
},
},
},

51
cli/classes.ts
View file

@ -1,9 +1,9 @@
import { parseUserAddress, userAddressValidator } from "@/api";
import { Args, type Command, Flags, type Interfaces } from "@oclif/core";
import { type Emoji, Instance, User, db } from "@versia/kit/db";
import { Emoji, Instance, User } from "@versia/kit/db";
import { Emojis, Instances, Users } from "@versia/kit/tables";
import chalk from "chalk";
import { and, eq, getTableColumns, like } from "drizzle-orm";
import { and, eq, inArray, like } from "drizzle-orm";
import { BaseCommand } from "./base.ts";
export type FlagsType<T extends typeof Command> = Interfaces.InferredFlags<
@ -203,14 +203,7 @@ export abstract class EmojiFinderCommand<
this.args = args as ArgsType<T>;
}
public async findEmojis(): Promise<
Omit<
typeof Emoji.$type & {
instanceUrl: string | null;
},
"instance"
>[]
> {
public async findEmojis(): Promise<Emoji[]> {
// Check if there are asterisks in the identifier but no pattern flag, warn the user if so
if (this.args.identifier.includes("*") && !this.flags.pattern) {
this.log(
@ -228,22 +221,26 @@ export abstract class EmojiFinderCommand<
? this.args.identifier.replace(/\*/g, "%")
: this.args.identifier;
return await db
.select({
...getTableColumns(Emojis),
instanceUrl: Instances.baseUrl,
})
.from(Emojis)
.leftJoin(Instances, eq(Emojis.instanceId, Instances.id))
.where(
and(
this.flags.type === "shortcode"
? operator(Emojis.shortcode, identifier)
: undefined,
this.flags.type === "instance"
? operator(Instances.baseUrl, identifier)
: undefined,
),
);
const instanceIds =
this.flags.type === "instance"
? (
await Instance.manyFromSql(
operator(Instances.baseUrl, identifier),
)
).map((instance) => instance.id)
: undefined;
return await Emoji.manyFromSql(
and(
this.flags.type === "shortcode"
? operator(Emojis.shortcode, identifier)
: undefined,
instanceIds && instanceIds.length > 0
? inArray(Emojis.instanceId, instanceIds)
: undefined,
),
undefined,
this.flags.limit,
);
}
}

20
cli/commands/emoji/add.ts
View file

@ -4,7 +4,6 @@ import { Emojis } from "@versia/kit/tables";
import chalk from "chalk";
import { and, eq, isNull } from "drizzle-orm";
import ora from "ora";
import { MediaManager } from "~/classes/media/media-manager";
import { BaseCommand } from "~/cli/base";
import { config } from "~/packages/config-manager";
@ -97,35 +96,22 @@ export default class EmojiAdd extends BaseCommand<typeof EmojiAdd> {
);
}
const mediaManager = new MediaManager(config);
const spinner = ora("Uploading emoji").start();
const uploaded = await mediaManager.addFile(file).catch((e: Error) => {
spinner.fail();
this.log(`${chalk.red("✗")} Error: ${chalk.red(e.message)}`);
return null;
});
if (!uploaded) {
return this.exit(1);
}
const media = await Media.fromFile(file);
spinner.succeed();
await Emoji.insert({
shortcode: args.shortcode,
url: Media.getUrl(uploaded.path),
mediaId: media.id,
visibleInPicker: true,
contentType: uploaded.uploadedFile.type,
});
this.log(
`${chalk.green("✓")} Created emoji ${chalk.green(
args.shortcode,
)} with url ${chalk.blue(
chalk.underline(Media.getUrl(uploaded.path)),
)}`,
)} with url ${chalk.blue(chalk.underline(media.getUrl()))}`,
);
this.exit(0);

15
cli/commands/emoji/delete.ts
View file

@ -55,13 +55,10 @@ export default class EmojiDelete extends EmojiFinderCommand<
flags.print &&
this.log(
formatArray(emojis, [
"id",
"shortcode",
"alt",
"contentType",
"instanceUrl",
]),
formatArray(
emojis.map((e) => e.data),
["id", "shortcode", "alt", "contentType", "instanceUrl"],
),
);
if (flags.confirm) {
@ -80,13 +77,13 @@ export default class EmojiDelete extends EmojiFinderCommand<
const spinner = ora("Deleting emoji(s)").start();
for (const emoji of emojis) {
spinner.text = `Deleting emoji ${chalk.gray(emoji.shortcode)} (${
spinner.text = `Deleting emoji ${chalk.gray(emoji.data.shortcode)} (${
emojis.findIndex((e) => e.id === emoji.id) + 1
}/${emojis.length})`;
const mediaManager = new MediaManager(config);
await mediaManager.deleteFileByUrl(emoji.url);
await mediaManager.deleteFileByUrl(emoji.media.getUrl());
await db.delete(Emojis).where(eq(Emojis.id, emoji.id));
}

21
cli/commands/emoji/import.ts
View file

@ -6,7 +6,6 @@ import { and, inArray, isNull } from "drizzle-orm";
import { lookup } from "mime-types";
import ora from "ora";
import { unzip } from "unzipit";
import { MediaManager } from "~/classes/media/media-manager";
import { BaseCommand } from "~/cli/base";
import { config } from "~/packages/config-manager";
@ -169,8 +168,6 @@ export default class EmojiImport extends BaseCommand<typeof EmojiImport> {
const importSpinner = ora("Importing emojis").start();
const mediaManager = new MediaManager(config);
const successfullyImported: MetaType["emojis"] = [];
for (const emoji of newEmojis) {
@ -197,26 +194,12 @@ export default class EmojiImport extends BaseCommand<typeof EmojiImport> {
type: contentType,
});
const uploaded = await mediaManager
.addFile(newFile)
.catch((e: Error) => {
this.log(
`${chalk.red("✗")} Error uploading ${chalk.red(
emoji.emoji.name,
)}: ${chalk.red(e.message)}`,
);
return null;
});
if (!uploaded) {
continue;
}
const media = await Media.fromFile(newFile);
await Emoji.insert({
shortcode: emoji.emoji.name,
url: Media.getUrl(uploaded.path),
mediaId: media.id,
visibleInPicker: true,
contentType: uploaded.uploadedFile.type,
});
successfullyImported.push(emoji);

13
drizzle.config.ts
View file

@ -1,20 +1,25 @@
import type { Config } from "drizzle-kit";
import { config } from "./packages/config-manager/index.ts";
/**
* Drizzle can't properly resolve imports with top-level await, so uncomment
* this line when generating migrations.
*/
export default {
dialect: "postgresql",
out: "./drizzle/migrations",
schema: "./drizzle/schema.ts",
dbCredentials: {
host: "localhost",
/* host: "localhost",
port: 40000,
user: "lysand",
password: "lysand",
database: "lysand",
/* host: config.database.host,
database: "lysand", */
host: config.database.host,
port: Number(config.database.port),
user: config.database.username,
password: config.database.password,
database: config.database.database, */
database: config.database.database,
},
// Print all statements
verbose: true,

5
drizzle/migrations/0043_mute_jigsaw.sql Normal file
View file

@ -0,0 +1,5 @@
ALTER TABLE "Emojis" ADD COLUMN "mediaId" uuid;--> statement-breakpoint
ALTER TABLE "Emojis" ADD CONSTRAINT "Emojis_mediaId_Medias_id_fk" FOREIGN KEY ("mediaId") REFERENCES "public"."Medias"("id") ON DELETE cascade ON UPDATE cascade;--> statement-breakpoint
ALTER TABLE "Emojis" DROP COLUMN "url";--> statement-breakpoint
ALTER TABLE "Emojis" DROP COLUMN "alt";--> statement-breakpoint
ALTER TABLE "Emojis" DROP COLUMN "content_type";

1
drizzle/migrations/0044_quiet_jasper_sitwell.sql Normal file
View file

@ -0,0 +1 @@
ALTER TABLE "Emojis" ALTER COLUMN "mediaId" SET NOT NULL;

2331
drizzle/migrations/meta/0043_snapshot.json Normal file
View file

File diff suppressed because it is too large Load diff

2331
drizzle/migrations/meta/0044_snapshot.json Normal file
View file

File diff suppressed because it is too large Load diff

14
drizzle/migrations/meta/_journal.json
View file

@ -302,6 +302,20 @@
"when": 1737660317024,
"tag": "0042_swift_the_phantom",
"breakpoints": true
},
{
"idx": 43,
"version": "7",
"when": 1738080562679,
"tag": "0043_mute_jigsaw",
"breakpoints": true
},
{
"idx": 44,
"version": "7",
"when": 1738082427051,
"tag": "0044_quiet_jasper_sitwell",
"breakpoints": true
}
]
}

14
drizzle/schema.ts
View file

@ -52,10 +52,13 @@ export const Challenges = pgTable("Challenges", {
export const Emojis = pgTable("Emojis", {
id: id(),
shortcode: text("shortcode").notNull(),
url: text("url").notNull(),
mediaId: uuid("mediaId")
.references(() => Medias.id, {
onDelete: "cascade",
onUpdate: "cascade",
})
.notNull(),
visibleInPicker: boolean("visible_in_picker").notNull(),
alt: text("alt"),
contentType: text("content_type").notNull(),
instanceId: uuid("instanceId").references(() => Instances.id, {
onDelete: "cascade",
onUpdate: "cascade",
@ -68,6 +71,10 @@ export const Emojis = pgTable("Emojis", {
});
export const EmojisRelations = relations(Emojis, ({ one, many }) => ({
media: one(Medias, {
fields: [Emojis.mediaId],
references: [Medias.id],
}),
instance: one(Instances, {
fields: [Emojis.instanceId],
references: [Instances.id],
@ -357,6 +364,7 @@ export const Medias = pgTable("Medias", {
export const MediasRelations = relations(Medias, ({ many }) => ({
notes: many(Notes),
emojis: many(Emojis),
}));
export const Notifications = pgTable("Notifications", {