refactor(worker): 🚚 Move queue code to plugin-kit package

packages/plugin-kit/queues/delivery.ts

@@ -0,0 +1,97 @@
+import type { JSONObject } from "@versia/sdk";
+import * as VersiaEntities from "@versia/sdk/entities";
+import { config } from "@versia-server/config";
+import { Queue, Worker } from "bullmq";
+import chalk from "chalk";
+import { User } from "../db/user.ts";
+import { connection } from "../redis.ts";
+
+export enum DeliveryJobType {
+    FederateEntity = "federateEntity",
+}
+
+export type DeliveryJobData = {
+    entity: JSONObject;
+    recipientId: string;
+    senderId: string;
+};
+
+export const deliveryQueue = new Queue<DeliveryJobData, void, DeliveryJobType>(
+    "delivery",
+    {
+        connection,
+    },
+);
+
+export const getDeliveryWorker = (): Worker<
+    DeliveryJobData,
+    void,
+    DeliveryJobType
+> =>
+    new Worker<DeliveryJobData, void, DeliveryJobType>(
+        deliveryQueue.name,
+        async (job) => {
+            switch (job.name) {
+                case DeliveryJobType.FederateEntity: {
+                    const { entity, recipientId, senderId } = job.data;
+
+                    const sender = await User.fromId(senderId);
+
+                    if (!sender) {
+                        throw new Error(
+                            `Could not resolve sender ID ${chalk.gray(
+                                senderId,
+                            )}`,
+                        );
+                    }
+
+                    const recipient = await User.fromId(recipientId);
+
+                    if (!recipient) {
+                        throw new Error(
+                            `Could not resolve recipient ID ${chalk.gray(
+                                recipientId,
+                            )}`,
+                        );
+                    }
+
+                    await job.log(
+                        `Federating entity [${
+                            entity.id
+                        }] from @${sender.getAcct()} to @${recipient.getAcct()}`,
+                    );
+
+                    const type = entity.type;
+                    const entityCtor = Object.values(VersiaEntities).find(
+                        (ctor) => ctor.name === type,
+                    ) as typeof VersiaEntities.Entity | undefined;
+
+                    if (!entityCtor) {
+                        throw new Error(
+                            `Could not resolve entity type ${chalk.gray(
+                                type,
+                            )} for entity [${entity.id}]`,
+                        );
+                    }
+
+                    await sender.federateToUser(
+                        await entityCtor.fromJSON(entity),
+                        recipient,
+                    );
+
+                    await job.log(
+                        `✔ Finished federating entity [${entity.id}]`,
+                    );
+                }
+            }
+        },
+        {
+            connection,
+            removeOnComplete: {
+                age: config.queues.delivery?.remove_after_complete_seconds,
+            },
+            removeOnFail: {
+                age: config.queues.delivery?.remove_after_failure_seconds,
+            },
+        },
+    );

packages/plugin-kit/queues/fetch.ts

@@ -0,0 +1,71 @@
+import { config } from "@versia-server/config";
+import { Queue, Worker } from "bullmq";
+import { eq } from "drizzle-orm";
+import { Instance } from "../db/instance.ts";
+import { connection } from "../redis.ts";
+import { Instances } from "../tables/schema.ts";
+
+export enum FetchJobType {
+    Instance = "instance",
+    User = "user",
+    Note = "user",
+}
+
+export type FetchJobData = {
+    uri: string;
+    refetcher?: string;
+};
+
+export const fetchQueue = new Queue<FetchJobData, void, FetchJobType>("fetch", {
+    connection,
+});
+
+export const getFetchWorker = (): Worker<FetchJobData, void, FetchJobType> =>
+    new Worker<FetchJobData, void, FetchJobType>(
+        fetchQueue.name,
+        async (job) => {
+            switch (job.name) {
+                case FetchJobType.Instance: {
+                    const { uri } = job.data;
+
+                    await job.log(`Fetching instance metadata from [${uri}]`);
+
+                    // Check if exists
+                    const host = new URL(uri).host;
+
+                    const existingInstance = await Instance.fromSql(
+                        eq(Instances.baseUrl, host),
+                    );
+
+                    if (existingInstance) {
+                        await job.log(
+                            "Instance is known, refetching remote data.",
+                        );
+
+                        await existingInstance.updateFromRemote();
+
+                        await job.log(
+                            `Instance [${uri}] successfully refetched`,
+                        );
+
+                        return;
+                    }
+
+                    await Instance.resolve(new URL(uri));
+
+                    await job.log(
+                        `✔ Finished fetching instance metadata from [${uri}]`,
+                    );
+                }
+            }
+        },
+        {
+            connection,
+            removeOnComplete: {
+                age: config.queues.fetch?.remove_after_complete_seconds,
+            },
+            removeOnFail: {
+                age: config.queues.fetch?.remove_after_failure_seconds,
+            },
+        },
+    );

packages/plugin-kit/queues/inbox.ts

@@ -0,0 +1,218 @@
+import type { JSONObject } from "@versia/sdk";
+import { config } from "@versia-server/config";
+import { Queue, Worker } from "bullmq";
+import type { SocketAddress } from "bun";
+import { ApiError } from "../api-error.ts";
+import { Instance } from "../db/instance.ts";
+import { User } from "../db/user.ts";
+import { InboxProcessor } from "../inbox-processor.ts";
+import { connection } from "../redis.ts";
+
+export enum InboxJobType {
+    ProcessEntity = "processEntity",
+}
+
+export type InboxJobData = {
+    data: JSONObject;
+    headers: {
+        "versia-signature"?: string;
+        "versia-signed-at"?: number;
+        "versia-signed-by"?: string;
+        authorization?: string;
+    };
+    request: {
+        url: string;
+        method: string;
+        body: string;
+    };
+    ip: SocketAddress | null;
+};
+
+export const inboxQueue = new Queue<InboxJobData, Response, InboxJobType>(
+    "inbox",
+    {
+        connection,
+    },
+);
+
+export const getInboxWorker = (): Worker<InboxJobData, void, InboxJobType> =>
+    new Worker<InboxJobData, void, InboxJobType>(
+        inboxQueue.name,
+        async (job) => {
+            switch (job.name) {
+                case InboxJobType.ProcessEntity: {
+                    const { data, headers, request, ip } = job.data;
+
+                    await job.log(`Processing entity [${data.id}]`);
+
+                    const req = new Request(request.url, {
+                        method: request.method,
+                        headers: new Headers(
+                            Object.entries(headers)
+                                .map(([k, v]) => [k, String(v)])
+                                .concat([
+                                    ["content-type", "application/json"],
+                                ]) as [string, string][],
+                        ),
+                        body: request.body,
+                    });
+
+                    if (headers.authorization) {
+                        try {
+                            const processor = new InboxProcessor(
+                                req,
+                                data,
+                                null,
+                                headers.authorization,
+                                ip,
+                            );
+
+                            await job.log(
+                                `Entity [${data.id}] is potentially from a bridge`,
+                            );
+
+                            await processor.process();
+                        } catch (e) {
+                            if (e instanceof ApiError) {
+                                // Error occurred
+                                await job.log(
+                                    `Error during processing: ${e.message}`,
+                                );
+
+                                await job.log(
+                                    `Failed processing entity [${data.id}]`,
+                                );
+
+                                return;
+                            }
+
+                            throw e;
+                        }
+
+                        await job.log(
+                            `✔ Finished processing entity [${data.id}]`,
+                        );
+
+                        return;
+                    }
+
+                    const { "versia-signed-by": signedBy } = headers as {
+                        "versia-signed-by": string;
+                    };
+
+                    const sender = await User.resolve(new URL(signedBy));
+
+                    if (!(sender || signedBy.startsWith("instance "))) {
+                        await job.log(
+                            `Could not resolve sender URI [${signedBy}]`,
+                        );
+
+                        return;
+                    }
+
+                    if (sender?.local) {
+                        throw new Error(
+                            "Cannot process federation requests from local users",
+                        );
+                    }
+
+                    const remoteInstance = sender
+                        ? await Instance.fromUser(sender)
+                        : await Instance.resolveFromHost(
+                              signedBy.split(" ")[1],
+                          );
+
+                    if (!remoteInstance) {
+                        await job.log("Could not resolve the remote instance.");
+
+                        return;
+                    }
+
+                    await job.log(
+                        `Entity [${data.id}] is from remote instance [${remoteInstance.data.baseUrl}]`,
+                    );
+
+                    if (!remoteInstance.data.publicKey?.key) {
+                        throw new Error(
+                            `Instance ${remoteInstance.data.baseUrl} has no public key stored in database`,
+                        );
+                    }
+
+                    const key = await crypto.subtle.importKey(
+                        "spki",
+                        Buffer.from(
+                            sender?.data.publicKey ??
+                                remoteInstance.data.publicKey.key,
+                            "base64",
+                        ),
+                        "Ed25519",
+                        false,
+                        ["verify"],
+                    );
+
+                    try {
+                        const processor = new InboxProcessor(
+                            req,
+                            data,
+                            {
+                                instance: remoteInstance,
+                                key,
+                            },
+                            undefined,
+                            ip,
+                        );
+
+                        await processor.process();
+                    } catch (e) {
+                        if (e instanceof ApiError) {
+                            // Error occurred
+                            await job.log(
+                                `Error during processing: ${e.message}`,
+                            );
+
+                            await job.log(
+                                `Failed processing entity [${data.id}]`,
+                            );
+
+                            await job.log(
+                                `Sending error message to instance [${remoteInstance.data.baseUrl}]`,
+                            );
+
+                            await remoteInstance.sendMessage(
+                                `Failed processing entity [${
+                                    data.uri
+                                }] delivered to inbox. Returned error:\n\n${JSON.stringify(
+                                    e.message,
+                                    null,
+                                    4,
+                                )}`,
+                            );
+
+                            await job.log("Message sent");
+
+                            return;
+                        }
+
+                        throw e;
+                    }
+
+                    await job.log(`Finished processing entity [${data.id}]`);
+
+                    return;
+                }
+
+                default: {
+                    throw new Error(`Unknown job type: ${job.name}`);
+                }
+            }
+        },
+        {
+            connection,
+            removeOnComplete: {
+                age: config.queues.inbox?.remove_after_complete_seconds,
+            },
+            removeOnFail: {
+                age: config.queues.inbox?.remove_after_failure_seconds,
+            },
+        },
+    );

packages/plugin-kit/queues/media.ts

@@ -0,0 +1,125 @@
+import { config } from "@versia-server/config";
+import { Queue, Worker } from "bullmq";
+import { calculateBlurhash } from "../../../classes/media/preprocessors/blurhash.ts";
+import { convertImage } from "../../../classes/media/preprocessors/image-conversion.ts";
+import { Media } from "../db/media.ts";
+import { connection } from "../redis.ts";
+
+export enum MediaJobType {
+    ConvertMedia = "convertMedia",
+    CalculateMetadata = "calculateMetadata",
+}
+
+export type MediaJobData = {
+    attachmentId: string;
+    filename: string;
+};
+
+export const mediaQueue = new Queue<MediaJobData, void, MediaJobType>("media", {
+    connection,
+});
+
+export const getMediaWorker = (): Worker<MediaJobData, void, MediaJobType> =>
+    new Worker<MediaJobData, void, MediaJobType>(
+        mediaQueue.name,
+        async (job) => {
+            switch (job.name) {
+                case MediaJobType.ConvertMedia: {
+                    const { attachmentId, filename } = job.data;
+
+                    await job.log(`Fetching attachment ID [${attachmentId}]`);
+
+                    const attachment = await Media.fromId(attachmentId);
+
+                    if (!attachment) {
+                        throw new Error(
+                            `Attachment not found: [${attachmentId}]`,
+                        );
+                    }
+
+                    await job.log(`Processing attachment [${attachmentId}]`);
+                    await job.log(
+                        `Fetching file from [${attachment.getUrl()}]`,
+                    );
+
+                    // Download the file and process it.
+                    const blob = await (
+                        await fetch(attachment.getUrl())
+                    ).blob();
+
+                    const file = new File([blob], filename);
+
+                    await job.log(`Converting attachment [${attachmentId}]`);
+
+                    const processedFile = await convertImage(
+                        file,
+                        config.media.conversion.convert_to,
+                        {
+                            convertVectors:
+                                config.media.conversion.convert_vectors,
+                        },
+                    );
+
+                    await job.log(`Uploading attachment [${attachmentId}]`);
+
+                    await attachment.updateFromFile(processedFile);
+
+                    await job.log(
+                        `✔ Finished processing attachment [${attachmentId}]`,
+                    );
+
+                    break;
+                }
+                case MediaJobType.CalculateMetadata: {
+                    // Calculate blurhash
+                    const { attachmentId } = job.data;
+
+                    await job.log(`Fetching attachment ID [${attachmentId}]`);
+
+                    const attachment = await Media.fromId(attachmentId);
+
+                    if (!attachment) {
+                        throw new Error(
+                            `Attachment not found: [${attachmentId}]`,
+                        );
+                    }
+
+                    await job.log(`Processing attachment [${attachmentId}]`);
+                    await job.log(
+                        `Fetching file from [${attachment.getUrl()}]`,
+                    );
+
+                    // Download the file and process it.
+                    const blob = await (
+                        await fetch(attachment.getUrl())
+                    ).blob();
+
+                    // Filename is not important for blurhash
+                    const file = new File([blob], "");
+
+                    await job.log(`Generating blurhash for [${attachmentId}]`);
+
+                    const blurhash = await calculateBlurhash(file);
+
+                    await attachment.update({
+                        blurhash,
+                    });
+
+                    await job.log(
+                        `✔ Finished processing attachment [${attachmentId}]`,
+                    );
+
+                    break;
+                }
+            }
+        },
+        {
+            connection,
+            removeOnComplete: {
+                age: config.queues.media?.remove_after_complete_seconds,
+            },
+            removeOnFail: {
+                age: config.queues.media?.remove_after_failure_seconds,
+            },
+        },
+    );

packages/plugin-kit/queues/push.ts

@@ -0,0 +1,168 @@
+import { config } from "@versia-server/config";
+import { Queue, Worker } from "bullmq";
+import { sendNotification } from "web-push";
+import { htmlToText } from "@/content_types.ts";
+import { Note } from "../db/note.ts";
+import { PushSubscription } from "../db/pushsubscription.ts";
+import { Token } from "../db/token.ts";
+import { User } from "../db/user.ts";
+import { connection } from "../redis.ts";
+
+export enum PushJobType {
+    Notify = "notify",
+}
+
+export type PushJobData = {
+    psId: string;
+    type: string;
+    relatedUserId: string;
+    noteId?: string;
+    notificationId: string;
+};
+
+export const pushQueue = new Queue<PushJobData, void, PushJobType>("push", {
+    connection,
+});
+
+export const getPushWorker = (): Worker<PushJobData, void, PushJobType> =>
+    new Worker<PushJobData, void, PushJobType>(
+        pushQueue.name,
+        async (job) => {
+            const {
+                data: { psId, relatedUserId, type, noteId, notificationId },
+            } = job;
+
+            if (!config.notifications.push) {
+                await job.log("Push notifications are disabled");
+                return;
+            }
+
+            if (
+                !(
+                    config.notifications.push.vapid_keys.private ||
+                    config.notifications.push.vapid_keys.public
+                )
+            ) {
+                await job.log("Push notifications are not configured");
+                return;
+            }
+
+            await job.log(
+                `Sending push notification for note [${notificationId}]`,
+            );
+
+            const ps = await PushSubscription.fromId(psId);
+
+            if (!ps) {
+                throw new Error(
+                    `Could not resolve push subscription ID ${psId}`,
+                );
+            }
+
+            const token = await Token.fromId(ps.data.tokenId);
+
+            if (!token) {
+                throw new Error(
+                    `Could not resolve token ID ${ps.data.tokenId}`,
+                );
+            }
+
+            const relatedUser = await User.fromId(relatedUserId);
+
+            if (!relatedUser) {
+                throw new Error(
+                    `Could not resolve related user ID ${relatedUserId}`,
+                );
+            }
+
+            const note = noteId ? await Note.fromId(noteId) : null;
+
+            const truncate = (str: string, len: number): string => {
+                if (str.length <= len) {
+                    return str;
+                }
+
+                return `${str.slice(0, len)}...`;
+            };
+
+            const name = truncate(
+                relatedUser.data.displayName || relatedUser.data.username,
+                50,
+            );
+
+            let title = name;
+
+            switch (type) {
+                case "mention":
+                    title = `${name} mentioned you`;
+                    break;
+                case "reply":
+                    title = `${name} replied to you`;
+                    break;
+                case "favourite":
+                    title = `${name} liked your note`;
+                    break;
+                case "reblog":
+                    title = `${name} reblogged your note`;
+                    break;
+                case "follow":
+                    title = `${name} followed you`;
+                    break;
+                case "follow_request":
+                    title = `${name} requested to follow you`;
+                    break;
+                case "poll":
+                    title = "Poll ended";
+                    break;
+            }
+
+            const body = note
+                ? htmlToText(note.data.spoilerText || note.data.content)
+                : htmlToText(relatedUser.data.note);
+
+            await sendNotification(
+                {
+                    endpoint: ps.data.endpoint,
+                    keys: {
+                        auth: ps.data.authSecret,
+                        p256dh: ps.data.publicKey,
+                    },
+                },
+                JSON.stringify({
+                    access_token: token.data.accessToken,
+                    // FIXME
+                    preferred_locale: "en-US",
+                    notification_id: notificationId,
+                    notification_type: type,
+                    icon: relatedUser.getAvatarUrl(),
+                    title,
+                    body: truncate(body, 140),
+                }),
+                {
+                    vapidDetails: {
+                        subject:
+                            config.notifications.push.subject ||
+                            config.http.base_url.origin,
+                        privateKey:
+                            config.notifications.push.vapid_keys.private ?? "",
+                        publicKey:
+                            config.notifications.push.vapid_keys.public ?? "",
+                    },
+                    contentEncoding: "aesgcm",
+                },
+            );
+
+            await job.log(
+                `✔ Finished delivering push notification for note [${notificationId}]`,
+            );
+        },
+        {
+            connection,
+            removeOnComplete: {
+                age: config.queues.push?.remove_after_complete_seconds,
+            },
+            removeOnFail: {
+                age: config.queues.push?.remove_after_failure_seconds,
+            },
+        },
+    );

packages/plugin-kit/queues/relationships.ts

@@ -0,0 +1,67 @@
+import { config } from "@versia-server/config";
+import { Queue, Worker } from "bullmq";
+import { Relationship } from "../db/relationship.ts";
+import { User } from "../db/user.ts";
+import { connection } from "../redis.ts";
+
+export enum RelationshipJobType {
+    Unmute = "unmute",
+}
+
+export type RelationshipJobData = {
+    ownerId: string;
+    subjectId: string;
+};
+
+export const relationshipQueue = new Queue<
+    RelationshipJobData,
+    void,
+    RelationshipJobType
+>("relationships", {
+    connection,
+});
+
+export const getRelationshipWorker = (): Worker<
+    RelationshipJobData,
+    void,
+    RelationshipJobType
+> =>
+    new Worker<RelationshipJobData, void, RelationshipJobType>(
+        relationshipQueue.name,
+        async (job) => {
+            switch (job.name) {
+                case RelationshipJobType.Unmute: {
+                    const { ownerId, subjectId } = job.data;
+
+                    const owner = await User.fromId(ownerId);
+                    const subject = await User.fromId(subjectId);
+
+                    if (!(owner && subject)) {
+                        await job.log("Users not found");
+                        return;
+                    }
+
+                    const foundRelationship =
+                        await Relationship.fromOwnerAndSubject(owner, subject);
+
+                    if (foundRelationship.data.muting) {
+                        await foundRelationship.update({
+                            muting: false,
+                            mutingNotifications: false,
+                        });
+                    }
+
+                    await job.log(`✔ Finished unmuting [${subjectId}]`);
+                }
+            }
+        },
+        {
+            connection,
+            removeOnComplete: {
+                age: config.queues.fetch?.remove_after_complete_seconds,
+            },
+            removeOnFail: {
+                age: config.queues.fetch?.remove_after_failure_seconds,
+            },
+        },
+    );