feat(federation): ✨ Implement queue for fetches

worker.ts

@@ -4,8 +4,10 @@ import { Instance, User } from "@versia/kit/db";
 import { Queue, Worker } from "bullmq";
 import type { SocketAddress } from "bun";
 import chalk from "chalk";
+import { eq } from "drizzle-orm";
 import IORedis from "ioredis";
 import { InboxProcessor } from "./classes/inbox/processor.ts";
+import { Instances } from "./drizzle/schema.ts";
 import { config } from "./packages/config-manager/index.ts";
 import type { KnownEntity } from "./types/api.ts";
 
@@ -25,6 +27,12 @@ export enum InboxJobType {
     ProcessEntity = "processEntity",
 }
 
+export enum FetchJobType {
+    Instance = "instance",
+    User = "user",
+    Note = "user",
+}
+
 export type InboxJobData = {
     data: Entity;
     headers: {
@@ -47,6 +55,11 @@ export type DeliveryJobData = {
     senderId: string;
 };
 
+export type FetchJobData = {
+    uri: string;
+    refetcher?: string;
+};
+
 export const deliveryQueue = new Queue<DeliveryJobData, void, DeliveryJobType>(
     "delivery",
     {
@@ -61,6 +74,10 @@ export const inboxQueue = new Queue<InboxJobData, Response, InboxJobType>(
     },
 );
 
+export const fetchQueue = new Queue<FetchJobData, void, FetchJobType>("fetch", {
+    connection,
+});
+
 export const deliveryWorker = new Worker<
     DeliveryJobData,
     void,
@@ -260,3 +277,50 @@ export const inboxWorker = new Worker<InboxJobData, Response, InboxJobType>(
         },
     },
 );
+
+export const fetchWorker = new Worker<FetchJobData, void, FetchJobType>(
+    fetchQueue.name,
+    async (job) => {
+        switch (job.name) {
+            case FetchJobType.Instance: {
+                const { uri } = job.data;
+
+                await job.log(`Fetching instance metadata from [${uri}]`);
+
+                // Check if exists
+                const host = new URL(uri).host;
+
+                const existingInstance = await Instance.fromSql(
+                    eq(Instances.baseUrl, host),
+                );
+
+                if (existingInstance) {
+                    await job.log("Instance is known, refetching remote data.");
+
+                    await existingInstance.updateFromRemote();
+
+                    await job.log(`Instance [${uri}] successfully refetched`);
+
+                    return;
+                }
+
+                await Instance.resolve(uri);
+
+                await job.log(
+                    `${chalk.green(
+                        "✔",
+                    )} Finished fetching instance metadata from [${uri}]`,
+                );
+            }
+        }
+    },
+    {
+        connection,
+        removeOnComplete: {
+            age: config.queues.fetch.remove_on_complete,
+        },
+        removeOnFail: {
+            age: config.queues.fetch.remove_on_failure,
+        },
+    },
+);