refactor(worker): 🚚 Move queue code to plugin-kit package

packages/api/app.ts

@@ -13,14 +13,14 @@ import { secureHeaders } from "hono/secure-headers";
 import { openAPISpecs } from "hono-openapi";
 import { Youch } from "youch";
 import { applyToHono } from "@/bull-board.ts";
-import pkg from "~/package.json" with { type: "application/json" };
-import { PluginLoader } from "../../classes/plugin/loader.ts";
+import pkg from "../../package.json" with { type: "application/json" };
 import type { ApiRouteExports, HonoEnv } from "../../types/api.ts";
 import { agentBans } from "./middlewares/agent-bans.ts";
 import { boundaryCheck } from "./middlewares/boundary-check.ts";
 import { ipBans } from "./middlewares/ip-bans.ts";
 import { logger } from "./middlewares/logger.ts";
 import { rateLimit } from "./middlewares/rate-limit.ts";
+import { PluginLoader } from "./plugin-loader.ts";
 import { routes } from "./routes.ts";
 // Extends Zod with OpenAPI schema generation
 import "zod-openapi/extend";

packages/api/package.json

@@ -68,6 +68,7 @@
         "magic-regexp": "catalog:",
         "altcha-lib": "catalog:",
         "@hono/zod-validator": "catalog:",
-        "zod-validation-error": "catalog:"
+        "zod-validation-error": "catalog:",
+        "confbox": "catalog:"
     }
 }

packages/api/plugin-loader.ts

@@ -0,0 +1,253 @@
+import { readdir } from "node:fs/promises";
+import { config } from "@versia-server/config";
+import { type Manifest, manifestSchema, Plugin } from "@versia-server/kit";
+import { pluginLogger, serverLogger } from "@versia-server/logging";
+import { file, sleep } from "bun";
+import chalk from "chalk";
+import { parseJSON5, parseJSONC } from "confbox";
+import type { Hono } from "hono";
+import type { ZodTypeAny } from "zod";
+import { fromZodError, type ValidationError } from "zod-validation-error";
+import type { HonoEnv } from "~/types/api";
+
+/**
+ * Class to manage plugins.
+ */
+export class PluginLoader {
+    /**
+     * Get all directories in a given directory.
+     * @param {string} dir - The directory to search.
+     * @returns {Promise<string[]>} - An array of directory names.
+     */
+    private static async getDirectories(dir: string): Promise<string[]> {
+        const files = await readdir(dir, { withFileTypes: true });
+        return files.filter((f) => f.isDirectory()).map((f) => f.name);
+    }
+
+    /**
+     * Find the manifest file in a given directory.
+     * @param {string} dir - The directory to search.
+     * @returns {Promise<string | undefined>} - The manifest file name if found, otherwise undefined.
+     */
+    private static async findManifestFile(
+        dir: string,
+    ): Promise<string | undefined> {
+        const files = await readdir(dir);
+        return files.find((f) => f.match(/^manifest\.(json|json5|jsonc)$/));
+    }
+
+    /**
+     * Check if a directory has an entrypoint file (index.{ts,js}).
+     * @param {string} dir - The directory to search.
+     * @returns {Promise<boolean>} - True if the entrypoint file is found, otherwise false.
+     */
+    private static async hasEntrypoint(dir: string): Promise<boolean> {
+        const files = await readdir(dir);
+        return files.includes("index.ts") || files.includes("index.js");
+    }
+
+    /**
+     * Parse the manifest file based on its type.
+     * @param {string} manifestPath - The path to the manifest file.
+     * @param {string} manifestFile - The manifest file name.
+     * @returns {Promise<unknown>} - The parsed manifest content.
+     * @throws Will throw an error if the manifest file cannot be parsed.
+     */
+    private async parseManifestFile(
+        manifestPath: string,
+        manifestFile: string,
+    ): Promise<unknown> {
+        const manifestText = await file(manifestPath).text();
+
+        try {
+            if (manifestFile.endsWith(".json")) {
+                return JSON.parse(manifestText);
+            }
+            if (manifestFile.endsWith(".json5")) {
+                return parseJSON5(manifestText);
+            }
+            if (manifestFile.endsWith(".jsonc")) {
+                return parseJSONC(manifestText);
+            }
+
+            throw new Error(`Unsupported manifest file type: ${manifestFile}`);
+        } catch (e) {
+            pluginLogger.fatal`Could not parse plugin manifest ${chalk.blue(manifestPath)} as ${manifestFile.split(".").pop()?.toUpperCase()}.`;
+            throw e;
+        }
+    }
+
+    /**
+     * Find all direct subdirectories with a valid manifest file and entrypoint (index.{ts,js}).
+     * @param {string} dir - The directory to search.
+     * @returns {Promise<string[]>} - An array of plugin directories.
+     */
+    public static async findPlugins(dir: string): Promise<string[]> {
+        const directories = await PluginLoader.getDirectories(dir);
+        const plugins: string[] = [];
+
+        for (const directory of directories) {
+            const manifestFile = await PluginLoader.findManifestFile(
+                `${dir}/${directory}`,
+            );
+            if (
+                manifestFile &&
+                (await PluginLoader.hasEntrypoint(`${dir}/${directory}`))
+            ) {
+                plugins.push(directory);
+            }
+        }
+
+        return plugins;
+    }
+
+    /**
+     * Parse the manifest file of a plugin.
+     * @param {string} dir - The directory containing the plugin.
+     * @param {string} plugin - The plugin directory name.
+     * @returns {Promise<Manifest>} - The parsed manifest object.
+     * @throws Will throw an error if the manifest file is missing or invalid.
+     */
+    public async parseManifest(dir: string, plugin: string): Promise<Manifest> {
+        const manifestFile = await PluginLoader.findManifestFile(
+            `${dir}/${plugin}`,
+        );
+
+        if (!manifestFile) {
+            throw new Error(`Plugin ${plugin} is missing a manifest file`);
+        }
+
+        const manifestPath = `${dir}/${plugin}/${manifestFile}`;
+        const manifest = await this.parseManifestFile(
+            manifestPath,
+            manifestFile,
+        );
+
+        const result = await manifestSchema.safeParseAsync(manifest);
+
+        if (!result.success) {
+            pluginLogger.fatal`Plugin manifest ${chalk.blue(manifestPath)} is invalid.`;
+            throw fromZodError(result.error);
+        }
+
+        return result.data;
+    }
+
+    /**
+     * Loads an entrypoint's default export and check if it's a Plugin.
+     * @param {string} dir - The directory containing the entrypoint.
+     * @param {string} entrypoint - The entrypoint file name.
+     * @returns {Promise<Plugin<ZodTypeAny>>} - The loaded Plugin instance.
+     * @throws Will throw an error if the entrypoint's default export is not a Plugin.
+     */
+    public async loadPlugin(
+        dir: string,
+        entrypoint: string,
+    ): Promise<Plugin<ZodTypeAny>> {
+        const plugin = (await import(`${dir}/${entrypoint}`)).default;
+
+        if (plugin instanceof Plugin) {
+            return plugin;
+        }
+
+        pluginLogger.fatal`Default export of entrypoint ${chalk.blue(entrypoint)} at ${chalk.blue(dir)} is not a Plugin.`;
+
+        throw new Error("Entrypoint is not a Plugin");
+    }
+
+    /**
+     * Load all plugins in a given directory.
+     * @param {string} dir - The directory to search.
+     * @returns An array of objects containing the manifest and plugin instance.
+     */
+    public async loadPlugins(
+        dir: string,
+        autoload: boolean,
+        enabled?: string[],
+        disabled?: string[],
+    ): Promise<{ manifest: Manifest; plugin: Plugin<ZodTypeAny> }[]> {
+        const plugins = await PluginLoader.findPlugins(dir);
+
+        const enabledOn = (enabled?.length ?? 0) > 0;
+        const disabledOn = (disabled?.length ?? 0) > 0;
+
+        if (enabledOn && disabledOn) {
+            pluginLogger.fatal`Both enabled and disabled lists are specified. Only one of them can be used.`;
+            throw new Error("Invalid configuration");
+        }
+
+        return Promise.all(
+            plugins.map(async (plugin) => {
+                const manifest = await this.parseManifest(dir, plugin);
+
+                // If autoload is disabled, only load plugins explicitly enabled
+                if (
+                    !(autoload || enabledOn || enabled?.includes(manifest.name))
+                ) {
+                    return null;
+                }
+
+                // If enabled is specified, only load plugins in the enabled list
+                // If disabled is specified, only load plugins not in the disabled list
+                if (enabledOn && !enabled?.includes(manifest.name)) {
+                    return null;
+                }
+
+                if (disabled?.includes(manifest.name)) {
+                    return null;
+                }
+
+                const pluginInstance = await this.loadPlugin(
+                    dir,
+                    `${plugin}/index`,
+                );
+
+                return { manifest, plugin: pluginInstance };
+            }),
+        ).then((data) => data.filter((d) => d !== null));
+    }
+
+    public static async addToApp(
+        plugins: {
+            manifest: Manifest;
+            plugin: Plugin<ZodTypeAny>;
+        }[],
+        app: Hono<HonoEnv>,
+    ): Promise<void> {
+        for (const data of plugins) {
+            serverLogger.info`Loading plugin ${chalk.blueBright(data.manifest.name)} ${chalk.blueBright(data.manifest.version)} ${chalk.gray(`[${plugins.indexOf(data) + 1}/${plugins.length}]`)}`;
+
+            const time1 = performance.now();
+
+            try {
+                // biome-ignore lint/complexity/useLiteralKeys: loadConfig is a private method
+                await data.plugin["_loadConfig"](
+                    config.plugins?.config?.[data.manifest.name],
+                );
+            } catch (e) {
+                serverLogger.fatal`Error encountered while loading plugin ${chalk.blueBright(data.manifest.name)} ${chalk.blueBright(data.manifest.version)} configuration.`;
+                serverLogger.fatal`This is due to invalid, missing or incomplete configuration.`;
+                serverLogger.fatal`Put your configuration at ${chalk.blueBright(
+                    "plugins.config.<plugin-name>",
+                )}`;
+                serverLogger.fatal`Here is the error message, please fix the configuration file accordingly:`;
+                serverLogger.fatal`${(e as ValidationError).message}`;
+
+                await sleep(Number.POSITIVE_INFINITY);
+            }
+
+            const time2 = performance.now();
+
+            // biome-ignore lint/complexity/useLiteralKeys: AddToApp is a private method
+            await data.plugin["_addToApp"](app);
+
+            const time3 = performance.now();
+
+            serverLogger.info`Plugin ${chalk.blueBright(data.manifest.name)} ${chalk.blueBright(
+                data.manifest.version,
+            )} loaded in ${chalk.gray(
+                `${(time2 - time1).toFixed(2)}ms`,
+            )} and added to app in ${chalk.gray(`${(time3 - time2).toFixed(2)}ms`)}`;
+        }
+    }
+}

packages/api/routes/api/v1/accounts/[id]/mute.ts

@@ -10,13 +10,13 @@ import {
     withUserParam,
 } from "@versia-server/kit/api";
 import { Relationship } from "@versia-server/kit/db";
-import { describeRoute } from "hono-openapi";
-import { resolver, validator } from "hono-openapi/zod";
-import { z } from "zod";
 import {
     RelationshipJobType,
     relationshipQueue,
-} from "~/classes/queues/relationships";
+} from "@versia-server/kit/queues/relationships";
+import { describeRoute } from "hono-openapi";
+import { resolver, validator } from "hono-openapi/zod";
+import { z } from "zod";
 
 export default apiRoute((app) =>
     app.post(

packages/api/routes/api/v1/instance/index.ts

@@ -8,7 +8,7 @@ import { and, eq, isNull } from "drizzle-orm";
 import { describeRoute } from "hono-openapi";
 import { resolver } from "hono-openapi/zod";
 import type { z } from "zod";
-import manifest from "~/package.json" with { type: "json" };
+import manifest from "../../../../../../package.json" with { type: "json" };
 
 export default apiRoute((app) =>
     app.get(

packages/api/routes/api/v2/instance/index.ts

@@ -6,7 +6,7 @@ import { Users } from "@versia-server/kit/tables";
 import { and, eq, isNull } from "drizzle-orm";
 import { describeRoute } from "hono-openapi";
 import { resolver } from "hono-openapi/zod";
-import pkg from "~/package.json" with { type: "json" };
+import pkg from "../../../../../../package.json" with { type: "json" };
 
 export default apiRoute((app) =>
     app.get(

packages/api/routes/inbox/index.ts

@@ -1,8 +1,8 @@
 import { apiRoute, handleZodError } from "@versia-server/kit/api";
+import { InboxJobType, inboxQueue } from "@versia-server/kit/queues/inbox";
 import { describeRoute } from "hono-openapi";
 import { validator } from "hono-openapi/zod";
 import { z } from "zod";
-import { InboxJobType, inboxQueue } from "~/classes/queues/inbox";
 
 export default apiRoute((app) =>
     app.post(

packages/api/routes/users/[uuid]/inbox/index.ts

@@ -1,9 +1,9 @@
 import { ApiError } from "@versia-server/kit";
 import { apiRoute, handleZodError } from "@versia-server/kit/api";
+import { InboxJobType, inboxQueue } from "@versia-server/kit/queues/inbox";
 import { describeRoute } from "hono-openapi";
 import { resolver, validator } from "hono-openapi/zod";
 import { z } from "zod";
-import { InboxJobType, inboxQueue } from "~/classes/queues/inbox";
 
 export default apiRoute((app) =>
     app.post(

packages/api/routes/well-known/nodeinfo/2.0/index.ts

@@ -4,7 +4,7 @@ import { Note, User } from "@versia-server/kit/db";
 import { describeRoute } from "hono-openapi";
 import { resolver } from "hono-openapi/zod";
 import { z } from "zod";
-import manifest from "~/package.json" with { type: "json" };
+import manifest from "../../../../../../package.json" with { type: "json" };
 
 export default apiRoute((app) =>
     app.get(

packages/api/routes/well-known/versia.ts

@@ -7,7 +7,7 @@ import { asc } from "drizzle-orm";
 import { describeRoute } from "hono-openapi";
 import { resolver } from "hono-openapi/zod";
 import { urlToContentFormat } from "@/content_types";
-import pkg from "~/package.json" with { type: "json" };
+import pkg from "../../../../package.json" with { type: "json" };
 
 export default apiRoute((app) =>
     app.get(