refactor: ♻️ Store instance federation protocol in database, refactor fetcher

classes/functions/instance.ts

@@ -1,51 +0,0 @@
-import type { ServerMetadata } from "@lysand-org/federation/types";
-import { db } from "~/drizzle/db";
-import { Instances } from "~/drizzle/schema";
-import { config } from "~/packages/config-manager";
-
-/**
- * Represents an instance in the database.
- */
-
-/**
- * Adds an instance to the database if it doesn't already exist.
- * @param url
- * @returns Either the database instance if it already exists, or a newly created instance.
- */
-export const addInstanceIfNotExists = async (url: string) => {
-    const origin = new URL(url).origin;
-    const host = new URL(url).host;
-
-    const found = await db.query.Instances.findFirst({
-        where: (instance, { eq }) => eq(instance.baseUrl, host),
-    });
-
-    if (found) {
-        return found;
-    }
-
-    // Fetch the instance configuration
-    const metadata = (await fetch(new URL("/.well-known/lysand", origin), {
-        proxy: config.http.proxy.address,
-    }).then((res) => res.json())) as ServerMetadata;
-
-    if (metadata.type !== "ServerMetadata") {
-        throw new Error("Invalid instance metadata (wrong type)");
-    }
-
-    if (!(metadata.name && metadata.version)) {
-        throw new Error("Invalid instance metadata (missing name or version)");
-    }
-
-    return (
-        await db
-            .insert(Instances)
-            .values({
-                baseUrl: host,
-                name: metadata.name,
-                version: metadata.version,
-                logo: metadata.logo,
-            })
-            .returning()
-    )[0];
-};

cli/commands/federation/instance/fetch.ts

@@ -0,0 +1,48 @@
+import { Args } from "@oclif/core";
+import ora from "ora";
+import { BaseCommand } from "~/cli/base";
+import { formatArray } from "~/cli/utils/format";
+import { Instance } from "~/packages/database-interface/instance";
+
+export default class FederationInstanceFetch extends BaseCommand<
+    typeof FederationInstanceFetch
+> {
+    static override args = {
+        url: Args.string({
+            description: "URL of the remote instance",
+            required: true,
+        }),
+    };
+
+    static override description = "Fetch metadata from remote instances";
+
+    static override examples = ["<%= config.bin %> <%= command.id %>"];
+
+    static override flags = {};
+
+    public async run(): Promise<void> {
+        const { args } = await this.parse(FederationInstanceFetch);
+
+        const spinner = ora("Fetching instance metadata").start();
+
+        const data = await Instance.fetchMetadata(args.url);
+
+        if (!data) {
+            spinner.fail("Failed to fetch instance metadata");
+            this.exit(1);
+        }
+
+        spinner.succeed("Fetched instance metadata");
+
+        const { metadata, protocol } = data;
+
+        this.log(
+            formatArray(
+                [{ ...metadata, protocol }],
+                ["name", "description", "version", "protocol"],
+            ),
+        );
+
+        this.exit(0);
+    }
+}

cli/index.ts

@@ -4,6 +4,7 @@ import EmojiAdd from "./commands/emoji/add";
 import EmojiDelete from "./commands/emoji/delete";
 import EmojiImport from "./commands/emoji/import";
 import EmojiList from "./commands/emoji/list";
+import FederationInstanceFetch from "./commands/federation/instance/fetch";
 import IndexRebuild from "./commands/index/rebuild";
 import Start from "./commands/start";
 import UserCreate from "./commands/user/create";
@@ -26,6 +27,7 @@ export const commands = {
     "emoji:list": EmojiList,
     "emoji:import": EmojiImport,
     "index:rebuild": IndexRebuild,
+    "federation:instance:fetch": FederationInstanceFetch,
     start: Start,
 };
 

drizzle/migrations/0028_unique_fat_cobra.sql

@@ -0,0 +1,2 @@
+ALTER TABLE "Challenges" ALTER COLUMN "expires_at" SET NOT NULL;--> statement-breakpoint
+ALTER TABLE "Instances" ADD COLUMN "protocol" text DEFAULT 'lysand' NOT NULL;

drizzle/migrations/meta/_journal.json

@@ -197,6 +197,13 @@
             "when": 1718327596823,
             "tag": "0027_peaceful_whistler",
             "breakpoints": true
+        },
+        {
+            "idx": 28,
+            "version": "7",
+            "when": 1719726234826,
+            "tag": "0028_unique_fat_cobra",
+            "breakpoints": true
         }
     ]
 }

drizzle/schema.ts

@@ -348,6 +348,10 @@ export const Instances = pgTable("Instances", {
     disableAutomoderation: boolean("disable_automoderation")
         .default(false)
         .notNull(),
+    protocol: text("protocol")
+        .notNull()
+        .$type<"lysand" | "activitypub">()
+        .default("lysand"),
 });
 
 export const OpenIdAccounts = pgTable("OpenIdAccounts", {

packages/database-interface/emoji.ts

@@ -10,10 +10,10 @@ import {
     inArray,
 } from "drizzle-orm";
 import type { EmojiWithInstance } from "~/classes/functions/emoji";
-import { addInstanceIfNotExists } from "~/classes/functions/instance";
 import { db } from "~/drizzle/db";
 import { Emojis, Instances } from "~/drizzle/schema";
 import { BaseInterface } from "./base";
+import { Instance } from "./instance";
 
 export class Emoji extends BaseInterface<typeof Emojis, EmojiWithInstance> {
     async reload(): Promise<void> {
@@ -143,7 +143,7 @@ export class Emoji extends BaseInterface<typeof Emojis, EmojiWithInstance> {
             return found;
         }
 
-        const foundInstance = host ? await addInstanceIfNotExists(host) : null;
+        const foundInstance = host ? await Instance.resolve(host) : null;
 
         return await Emoji.fromLysand(emojiToFetch, foundInstance?.id ?? null);
     }

packages/database-interface/instance.ts

@@ -0,0 +1,278 @@
+import { getLogger } from "@logtape/logtape";
+import { EntityValidator, type ValidationError } from "@lysand-org/federation";
+import type { ServerMetadata } from "@lysand-org/federation/types";
+import chalk from "chalk";
+import { config } from "config-manager";
+import {
+    type InferInsertModel,
+    type InferSelectModel,
+    type SQL,
+    desc,
+    eq,
+    inArray,
+} from "drizzle-orm";
+import { db } from "~/drizzle/db";
+import { Instances } from "~/drizzle/schema";
+import { BaseInterface } from "./base";
+
+export type AttachmentType = InferSelectModel<typeof Instances>;
+
+export class Instance extends BaseInterface<typeof Instances> {
+    async reload(): Promise<void> {
+        const reloaded = await Instance.fromId(this.data.id);
+
+        if (!reloaded) {
+            throw new Error("Failed to reload instance");
+        }
+
+        this.data = reloaded.data;
+    }
+
+    public static async fromId(id: string | null): Promise<Instance | null> {
+        if (!id) {
+            return null;
+        }
+
+        return await Instance.fromSql(eq(Instances.id, id));
+    }
+
+    public static async fromIds(ids: string[]): Promise<Instance[]> {
+        return await Instance.manyFromSql(inArray(Instances.id, ids));
+    }
+
+    public static async fromSql(
+        sql: SQL<unknown> | undefined,
+        orderBy: SQL<unknown> | undefined = desc(Instances.id),
+    ): Promise<Instance | null> {
+        const found = await db.query.Instances.findFirst({
+            where: sql,
+            orderBy,
+        });
+
+        if (!found) {
+            return null;
+        }
+        return new Instance(found);
+    }
+
+    public static async manyFromSql(
+        sql: SQL<unknown> | undefined,
+        orderBy: SQL<unknown> | undefined = desc(Instances.id),
+        limit?: number,
+        offset?: number,
+        extra?: Parameters<typeof db.query.Instances.findMany>[0],
+    ): Promise<Instance[]> {
+        const found = await db.query.Instances.findMany({
+            where: sql,
+            orderBy,
+            limit,
+            offset,
+            with: extra?.with,
+        });
+
+        return found.map((s) => new Instance(s));
+    }
+
+    async update(
+        newInstance: Partial<AttachmentType>,
+    ): Promise<AttachmentType> {
+        await db
+            .update(Instances)
+            .set(newInstance)
+            .where(eq(Instances.id, this.id));
+
+        const updated = await Instance.fromId(this.data.id);
+
+        if (!updated) {
+            throw new Error("Failed to update instance");
+        }
+
+        this.data = updated.data;
+        return updated.data;
+    }
+
+    save(): Promise<AttachmentType> {
+        return this.update(this.data);
+    }
+
+    async delete(ids: string[]): Promise<void>;
+    async delete(): Promise<void>;
+    async delete(ids?: unknown): Promise<void> {
+        if (Array.isArray(ids)) {
+            await db.delete(Instances).where(inArray(Instances.id, ids));
+        } else {
+            await db.delete(Instances).where(eq(Instances.id, this.id));
+        }
+    }
+
+    public static async insert(
+        data: InferInsertModel<typeof Instances>,
+    ): Promise<Instance> {
+        const inserted = (
+            await db.insert(Instances).values(data).returning()
+        )[0];
+
+        const instance = await Instance.fromId(inserted.id);
+
+        if (!instance) {
+            throw new Error("Failed to insert instance");
+        }
+
+        return instance;
+    }
+
+    get id() {
+        return this.data.id;
+    }
+
+    static async fetchMetadata(url: string): Promise<{
+        metadata: ServerMetadata;
+        protocol: "lysand" | "activitypub";
+    } | null> {
+        const origin = new URL(url).origin;
+        const wellKnownUrl = new URL("/.well-known/lysand", origin);
+        const logger = getLogger("federation");
+
+        try {
+            const response = await fetch(wellKnownUrl, {
+                proxy: config.http.proxy.address,
+            });
+
+            if (!response.ok) {
+                // If the server doesn't have a Lysand well-known endpoint, it's not a Lysand instance
+                // Try to resolve ActivityPub metadata instead
+                const data = await Instance.fetchActivityPubMetadata(url);
+
+                if (!data) {
+                    return null;
+                }
+
+                return {
+                    metadata: data,
+                    protocol: "activitypub",
+                };
+            }
+
+            try {
+                const metadata = await new EntityValidator().ServerMetadata(
+                    await response.json(),
+                );
+
+                return { metadata, protocol: "lysand" };
+            } catch (error) {
+                logger.error`Instance ${chalk.bold(
+                    origin,
+                )} has invalid metadata: ${(error as ValidationError).message}`;
+                return null;
+            }
+        } catch (error) {
+            logger.error`Failed to fetch Lysand metadata for instance ${chalk.bold(
+                origin,
+            )} - Error! ${error}`;
+            return null;
+        }
+    }
+
+    private static async fetchActivityPubMetadata(
+        url: string,
+    ): Promise<ServerMetadata | null> {
+        const origin = new URL(url).origin;
+        const wellKnownUrl = new URL("/.well-known/nodeinfo", origin);
+
+        // Go to endpoint, then follow the links to the actual metadata
+
+        const logger = getLogger("federation");
+
+        try {
+            const response = await fetch(wellKnownUrl, {
+                proxy: config.http.proxy.address,
+            });
+
+            if (!response.ok) {
+                logger.error`Failed to fetch ActivityPub metadata for instance ${chalk.bold(
+                    origin,
+                )} - HTTP ${response.status}`;
+                return null;
+            }
+
+            const wellKnown = await response.json();
+
+            if (!wellKnown.links) {
+                logger.error`Failed to fetch ActivityPub metadata for instance ${chalk.bold(
+                    origin,
+                )} - No links found`;
+                return null;
+            }
+
+            const metadataUrl = wellKnown.links.find(
+                (link: { rel: string }) =>
+                    link.rel ===
+                    "http://nodeinfo.diaspora.software/ns/schema/2.0",
+            );
+
+            if (!metadataUrl) {
+                logger.error`Failed to fetch ActivityPub metadata for instance ${chalk.bold(
+                    origin,
+                )} - No metadata URL found`;
+                return null;
+            }
+
+            const metadataResponse = await fetch(metadataUrl.href, {
+                proxy: config.http.proxy.address,
+            });
+
+            if (!metadataResponse.ok) {
+                logger.error`Failed to fetch ActivityPub metadata for instance ${chalk.bold(
+                    origin,
+                )} - HTTP ${metadataResponse.status}`;
+                return null;
+            }
+
+            const metadata = await metadataResponse.json();
+
+            return {
+                name:
+                    metadata.metadata.nodeName || metadata.metadata.title || "",
+                version: metadata.software.version,
+                logo: undefined,
+                type: "ServerMetadata",
+                supported_extensions: [],
+            };
+        } catch (error) {
+            logger.error`Failed to fetch ActivityPub metadata for instance ${chalk.bold(
+                origin,
+            )} - Error! ${error}`;
+            return null;
+        }
+    }
+
+    static async resolve(url: string): Promise<Instance> {
+        const logger = getLogger("federation");
+        const host = new URL(url).host;
+
+        const existingInstance = await Instance.fromSql(
+            eq(Instances.baseUrl, host),
+        );
+
+        if (existingInstance) {
+            return existingInstance;
+        }
+
+        const output = await Instance.fetchMetadata(url);
+
+        if (!output) {
+            logger.error`Failed to resolve instance ${chalk.bold(host)}`;
+            throw new Error("Failed to resolve instance");
+        }
+
+        const { metadata, protocol } = output;
+
+        return Instance.insert({
+            baseUrl: host,
+            name: metadata.name,
+            version: metadata.version,
+            logo: metadata.logo,
+            protocol: protocol,
+        });
+    }
+}

packages/database-interface/user.ts

@@ -10,7 +10,6 @@ import { EntityValidator } from "@lysand-org/federation";
 import type { Entity, User as LysandUser } from "@lysand-org/federation/types";
 import {
     type InferInsertModel,
-    type InferSelectModel,
     type SQL,
     and,
     count,
@@ -25,7 +24,6 @@ import {
 } from "drizzle-orm";
 import { htmlToText } from "html-to-text";
 import { objectToInboxRequest } from "~/classes/functions/federation";
-import { addInstanceIfNotExists } from "~/classes/functions/instance";
 import {
     type UserWithRelations,
     findManyUsers,
@@ -34,7 +32,6 @@ import { searchManager } from "~/classes/search/search-manager";
 import { db } from "~/drizzle/db";
 import {
     EmojiToUser,
-    type Instances,
     NoteToMentions,
     Notes,
     type RolePermissions,
@@ -44,6 +41,7 @@ import {
 import { type Config, config } from "~/packages/config-manager";
 import { BaseInterface } from "./base";
 import { Emoji } from "./emoji";
+import { Instance } from "./instance";
 import type { Note } from "./note";
 import { Role } from "./role";
 
@@ -247,39 +245,51 @@ export class User extends BaseInterface<typeof Users, UserWithRelations> {
 
     static async saveFromRemote(uri: string): Promise<User> {
         if (!URL.canParse(uri)) {
-            throw new Error(`Invalid URI to parse ${uri}`);
+            throw new Error(`Invalid URI: ${uri}`);
         }
 
+        const instance = await Instance.resolve(uri);
+
+        if (instance.data.protocol === "lysand") {
+            return await User.saveFromLysand(uri, instance);
+        }
+
+        if (instance.data.protocol === "activitypub") {
+            // Placeholder for ActivityPub user fetching
+            throw new Error("ActivityPub user fetching not implemented");
+        }
+
+        throw new Error(`Unsupported protocol: ${instance.data.protocol}`);
+    }
+
+    private static async saveFromLysand(
+        uri: string,
+        instance: Instance,
+    ): Promise<User> {
         const response = await fetch(uri, {
             method: "GET",
-            headers: {
+            headers: { Accept: "application/json" },
-                Accept: "application/json",
-            },
             proxy: config.http.proxy.address,
         });
 
+        if (!response.ok) {
+            throw new Error(`HTTP error! status: ${response.status}`);
+        }
+
         const json = (await response.json()) as Partial<LysandUser>;
-
         const validator = new EntityValidator();
-
         const data = await validator.User(json);
 
-        // Parse emojis and add them to database
+        const user = await User.fromLysand(data, instance);
+
         const userEmojis =
             data.extensions?.["org.lysand:custom_emojis"]?.emojis ?? [];
-
-        const instance = await addInstanceIfNotExists(data.uri);
-
         const emojis = await Promise.all(
             userEmojis.map((emoji) => Emoji.fromLysand(emoji, instance.id)),
         );
 
-        const user = await User.fromLysand(data, instance);
-
-        // Add emojis to user
         if (emojis.length > 0) {
             await db.delete(EmojiToUser).where(eq(EmojiToUser.userId, user.id));
-
             await db.insert(EmojiToUser).values(
                 emojis.map((emoji) => ({
                     emojiId: emoji.id,
@@ -289,12 +299,10 @@ export class User extends BaseInterface<typeof Users, UserWithRelations> {
         }
 
         const finalUser = await User.fromId(user.id);
-
         if (!finalUser) {
             throw new Error("Failed to save user from remote");
         }
 
-        // Add to search index
         await searchManager.addUser(finalUser);
 
         return finalUser;
@@ -302,7 +310,7 @@ export class User extends BaseInterface<typeof Users, UserWithRelations> {
 
     static async fromLysand(
         user: LysandUser,
-        instance: InferSelectModel<typeof Instances>,
+        instance: Instance,
     ): Promise<User> {
         const data = {
             username: user.username,