More work on converting to the Lysand protocol

database/entities/Emoji.ts

@@ -2,11 +2,13 @@ import {
 	BaseEntity,
 	Column,
 	Entity,
+	IsNull,
 	ManyToOne,
 	PrimaryGeneratedColumn,
 } from "typeorm";
 import { APIEmoji } from "~types/entities/emoji";
 import { Instance } from "./Instance";
+import { Emoji as LysandEmoji } from "~types/lysand/extensions/org.lysand/custom_emojis";
 
 /**
  * Represents an emoji entity in the database.
@@ -42,12 +44,69 @@ export class Emoji extends BaseEntity {
 	@Column("varchar")
 	url!: string;
 
+	/**
+	 * The alt text for the emoji.
+	 */
+	@Column("varchar", {
+		nullable: true,
+	})
+	alt!: string | null;
+
+	/**
+	 * The content type of the emoji.
+	 */
+	@Column("varchar")
+	content_type!: string;
+
 	/**
 	 * Whether the emoji is visible in the picker.
 	 */
 	@Column("boolean")
 	visible_in_picker!: boolean;
 
+	/**
+	 * Used for parsing emojis from local text
+	 * @param text The text to parse
+	 * @returns An array of emojis
+	 */
+	static async parseEmojis(text: string): Promise<Emoji[]> {
+		const regex = /:[a-zA-Z0-9_]+:/g;
+		const matches = text.match(regex);
+		if (!matches) return [];
+		return (
+			await Promise.all(
+				matches.map(match =>
+					Emoji.findOne({
+						where: {
+							shortcode: match.slice(1, -1),
+							instance: IsNull(),
+						},
+						relations: ["instance"],
+					})
+				)
+			)
+		).filter(emoji => emoji !== null) as Emoji[];
+	}
+
+	static async addIfNotExists(emoji: LysandEmoji) {
+		const existingEmoji = await Emoji.findOne({
+			where: {
+				shortcode: emoji.name,
+				instance: IsNull(),
+			},
+		});
+		if (existingEmoji) return existingEmoji;
+		const newEmoji = new Emoji();
+		newEmoji.shortcode = emoji.name;
+		// TODO: Content types
+		newEmoji.url = emoji.url[0].content;
+		newEmoji.alt = emoji.alt || null;
+		newEmoji.content_type = emoji.url[0].content_type;
+		newEmoji.visible_in_picker = true;
+		await newEmoji.save();
+		return newEmoji;
+	}
+
 	/**
 	 * Converts the emoji to an APIEmoji object.
 	 * @returns The APIEmoji object.
@@ -62,4 +121,17 @@ export class Emoji extends BaseEntity {
 			category: undefined,
 		};
 	}
+
+	toLysand(): LysandEmoji {
+		return {
+			name: this.shortcode,
+			url: [
+				{
+					content: this.url,
+					content_type: this.content_type,
+				},
+			],
+			alt: this.alt || undefined,
+		};
+	}
 }

database/entities/Instance.ts

@@ -1,62 +1,5 @@
 import { BaseEntity, Column, Entity, PrimaryGeneratedColumn } from "typeorm";
-import { APIInstance } from "~types/entities/instance";
-import { APIAccount } from "~types/entities/account";
-
-export interface NodeInfo {
-	software: {
-		name: string;
-		version: string;
-	};
-	protocols: string[];
-	version: string;
-	services: {
-		inbound: string[];
-		outbound: string[];
-	};
-	openRegistrations: boolean;
-	usage: {
-		users: {
-			total: number;
-			activeHalfyear: number;
-			activeMonth: number;
-		};
-		localPosts: number;
-		localComments?: number;
-		remotePosts?: number;
-		remoteComments?: number;
-	};
-	metadata: Partial<{
-		nodeName: string;
-		nodeDescription: string;
-		maintainer: {
-			name: string;
-			email: string;
-		};
-		langs: string[];
-		tosUrl: string;
-		repositoryUrl: string;
-		feedbackUrl: string;
-		disableRegistration: boolean;
-		disableLocalTimeline: boolean;
-		disableRecommendedTimeline: boolean;
-		disableGlobalTimeline: boolean;
-		emailRequiredForSignup: boolean;
-		searchFilters: boolean;
-		postEditing: boolean;
-		postImports: boolean;
-		enableHcaptcha: boolean;
-		enableRecaptcha: boolean;
-		maxNoteTextLength: number;
-		maxCaptionTextLength: number;
-		enableTwitterIntegration: boolean;
-		enableGithubIntegration: boolean;
-		enableDiscordIntegration: boolean;
-		enableEmail: boolean;
-		enableServiceWorker: boolean;
-		proxyAccountName: string | null;
-		themeColor: string;
-	}>;
-}
+import { ContentFormat, ServerMetadata } from "~types/lysand/Object";
 
 /**
  * Represents an instance in the database.
@@ -79,18 +22,27 @@ export class Instance extends BaseEntity {
 	base_url!: string;
 
 	/**
-	 * The configuration of the instance.
+	 * The name of the instance.
 	 */
-	@Column("jsonb", {
-		nullable: true,
-	})
-	instance_data?: APIInstance;
+	@Column("varchar")
+	name!: string;
 
 	/**
-	 * Instance nodeinfo data
+	 * The description of the instance.
+	 */
+	@Column("varchar")
+	version!: string;
+
+	/**
+	 * The logo of the instance.
 	 */
 	@Column("jsonb")
-	nodeinfo!: NodeInfo;
+	logo?: ContentFormat[];
+
+	/**
+	 * The banner of the instance.
+	 */
+	banner?: ContentFormat[];
 
 	/**
 	 * Adds an instance to the database if it doesn't already exist.
@@ -114,80 +66,25 @@ export class Instance extends BaseEntity {
 		instance.base_url = hostname;
 
 		// Fetch the instance configuration
-		const nodeinfo: NodeInfo = await fetch(`${origin}/nodeinfo/2.0`).then(
+		const metadata = (await fetch(`${origin}/.well-known/lysand`).then(
 			res => res.json()
-		);
+		)) as Partial<ServerMetadata>;
 
-		// Try to fetch configuration from Mastodon-compatible instances
-		if (
-			["firefish", "iceshrimp", "mastodon", "akkoma", "pleroma"].includes(
-				nodeinfo.software.name
-			)
-		) {
-			const instanceData: APIInstance = await fetch(
-				`${origin}/api/v1/instance`
-			).then(res => res.json());
-
-			instance.instance_data = instanceData;
+		if (metadata.type !== "ServerMetadata") {
+			throw new Error("Invalid instance metadata");
 		}
 
-		instance.nodeinfo = nodeinfo;
+		if (!(metadata.name && metadata.version)) {
+			throw new Error("Invalid instance metadata");
+		}
+
+		instance.name = metadata.name;
+		instance.version = metadata.version;
+		instance.logo = metadata.logo;
+		instance.banner = metadata.banner;
 
 		await instance.save();
 
 		return instance;
 	}
-
-	/**
-	 * Converts the instance to an API instance.
-	 * @returns The API instance.
-	 */
-	// eslint-disable-next-line @typescript-eslint/require-await
-	async toAPI(): Promise<APIInstance> {
-		return {
-			uri: this.instance_data?.uri || this.base_url,
-			approval_required: this.instance_data?.approval_required || false,
-			email: this.instance_data?.email || "",
-			thumbnail: this.instance_data?.thumbnail || "",
-			title: this.instance_data?.title || "",
-			version: this.instance_data?.version || "",
-			configuration: this.instance_data?.configuration || {
-				media_attachments: {
-					image_matrix_limit: 0,
-					image_size_limit: 0,
-					supported_mime_types: [],
-					video_frame_limit: 0,
-					video_matrix_limit: 0,
-					video_size_limit: 0,
-				},
-				polls: {
-					max_characters_per_option: 0,
-					max_expiration: 0,
-					max_options: 0,
-					min_expiration: 0,
-				},
-				statuses: {
-					characters_reserved_per_url: 0,
-					max_characters: 0,
-					max_media_attachments: 0,
-				},
-			},
-			contact_account:
-				this.instance_data?.contact_account || ({} as APIAccount),
-			description: this.instance_data?.description || "",
-			invites_enabled: this.instance_data?.invites_enabled || false,
-			languages: this.instance_data?.languages || [],
-			registrations: this.instance_data?.registrations || false,
-			rules: this.instance_data?.rules || [],
-			stats: {
-				domain_count: this.instance_data?.stats.domain_count || 0,
-				status_count: this.instance_data?.stats.status_count || 0,
-				user_count: this.instance_data?.stats.user_count || 0,
-			},
-			urls: {
-				streaming_api: this.instance_data?.urls.streaming_api || "",
-			},
-			max_toot_chars: this.instance_data?.max_toot_chars || 0,
-		};
-	}
 }

database/entities/Like.ts

@@ -7,6 +7,8 @@ import {
 } from "typeorm";
 import { User } from "./User";
 import { Status } from "./Status";
+import { Like as LysandLike } from "~types/lysand/Object";
+import { getConfig } from "@config";
 
 /**
  * Represents a Like entity in the database.
@@ -29,4 +31,15 @@ export class Like extends BaseEntity {
 
 	@CreateDateColumn()
 	created_at!: Date;
+
+	toLysand(): LysandLike {
+		return {
+			id: this.id,
+			author: this.liker.uri,
+			type: "Like",
+			created_at: new Date(this.created_at).toISOString(),
+			object: this.liked.toLysand().uri,
+			uri: `${getConfig().http.base_url}/actions/${this.id}`,
+		};
+	}
 }

database/entities/Object.ts

@@ -0,0 +1,92 @@
+import {
+	BaseEntity,
+	Column,
+	Entity,
+	ManyToOne,
+	PrimaryGeneratedColumn,
+} from "typeorm";
+import { LysandObjectType } from "~types/lysand/Object";
+
+/**
+ * Represents a Lysand object in the database.
+ */
+@Entity({
+	name: "objects",
+})
+export class LysandObject extends BaseEntity {
+	/**
+	 * The unique identifier for the object. If local, same as `remote_id`
+	 */
+	@PrimaryGeneratedColumn("uuid")
+	id!: string;
+
+	/**
+	 * UUID of the object across the network. If the object is local, same as `id`
+	 */
+	remote_id!: string;
+
+	/**
+	 * Any valid Lysand type, such as `Note`, `Like`, `Follow`, etc.
+	 */
+	@Column("varchar")
+	type!: string;
+
+	/**
+	 * Remote URI for the object
+	 * Example: `https://example.com/publications/ef235cc6-d68c-4756-b0df-4e6623c4d51c`
+	 */
+	@Column("varchar")
+	uri!: string;
+
+	@Column("timestamp")
+	created_at!: string;
+
+	/**
+	 * References an Actor object by URI
+	 */
+	@ManyToOne(() => LysandObject, object => object.uri, {
+		nullable: true,
+	})
+	author!: LysandObject;
+
+	@Column("jsonb")
+	extra_data!: Omit<
+		Omit<Omit<Omit<LysandObjectType, "created_at">, "id">, "uri">,
+		"type"
+	>;
+
+	@Column("jsonb")
+	extensions!: Record<string, any>;
+
+	static new(type: string, uri: string): LysandObject {
+		const object = new LysandObject();
+		object.type = type;
+		object.uri = uri;
+		object.created_at = new Date().toISOString();
+		return object;
+	}
+
+	isPublication(): boolean {
+		return this.type === "Note" || this.type === "Patch";
+	}
+
+	isAction(): boolean {
+		return [
+			"Like",
+			"Follow",
+			"Dislike",
+			"FollowAccept",
+			"FollowReject",
+			"Undo",
+			"Announce",
+		].includes(this.type);
+	}
+
+	isActor(): boolean {
+		return this.type === "User";
+	}
+
+	isExtension(): boolean {
+		return this.type === "Extension";
+	}
+}

database/entities/Status.ts

@@ -21,6 +21,8 @@ import { Emoji } from "./Emoji";
 import { Instance } from "./Instance";
 import { Like } from "./Like";
 import { AppDataSource } from "~database/datasource";
+import { Note } from "~types/lysand/Object";
+import { htmlToText } from "html-to-text";
 
 const config = getConfig();
 
@@ -95,6 +97,14 @@ export class Status extends BaseEntity {
 	})
 	content!: string;
 
+	/**
+	 * The content type of this status.
+	 */
+	@Column("varchar", {
+		default: "text/plain",
+	})
+	content_type!: string;
+
 	/**
 	 * The visibility of this status.
 	 */
@@ -289,6 +299,8 @@ export class Status extends BaseEntity {
 		sensitive: boolean;
 		spoiler_text: string;
 		emojis: Emoji[];
+		content_type?: string;
+		mentions?: User[];
 		reply?: {
 			status: Status;
 			user: User;
@@ -299,6 +311,7 @@ export class Status extends BaseEntity {
 		newStatus.account = data.account;
 		newStatus.application = data.application ?? null;
 		newStatus.content = data.content;
+		newStatus.content_type = data.content_type ?? "text/plain";
 		newStatus.visibility = data.visibility;
 		newStatus.sensitive = data.sensitive;
 		newStatus.spoiler_text = data.spoiler_text;
@@ -318,40 +331,44 @@ export class Status extends BaseEntity {
 		});
 
 		// Get list of mentioned users
-		await Promise.all(
-			mentionedPeople.map(async person => {
-				// Check if post is in format @username or @username@instance.com
-				// If is @username, the user is a local user
-				const instanceUrl =
-					person.split("@").length === 3
-						? person.split("@")[2]
-						: null;
+		if (!data.mentions) {
+			await Promise.all(
+				mentionedPeople.map(async person => {
+					// Check if post is in format @username or @username@instance.com
+					// If is @username, the user is a local user
+					const instanceUrl =
+						person.split("@").length === 3
+							? person.split("@")[2]
+							: null;
 
-				if (instanceUrl) {
-					const user = await User.findOne({
-						where: {
-							username: person.split("@")[1],
-							// If contains instanceUrl
-							instance: {
-								base_url: instanceUrl,
+					if (instanceUrl) {
+						const user = await User.findOne({
+							where: {
+								username: person.split("@")[1],
+								// If contains instanceUrl
+								instance: {
+									base_url: instanceUrl,
+								},
 							},
-						},
-						relations: userRelations,
-					});
+							relations: userRelations,
+						});
 
-					newStatus.mentions.push(user as User);
-				} else {
-					const user = await User.findOne({
-						where: {
-							username: person.split("@")[1],
-						},
-						relations: userRelations,
-					});
+						newStatus.mentions.push(user as User);
+					} else {
+						const user = await User.findOne({
+							where: {
+								username: person.split("@")[1],
+							},
+							relations: userRelations,
+						});
 
-					newStatus.mentions.push(user as User);
-				}
-			})
-		);
+						newStatus.mentions.push(user as User);
+					}
+				})
+			);
+		} else {
+			newStatus.mentions = data.mentions;
+		}
 
 		await newStatus.save();
 		return newStatus;
@@ -442,4 +459,41 @@ export class Status extends BaseEntity {
 			quote_id: undefined,
 		};
 	}
+
+	toLysand(): Note {
+		return {
+			type: "Note",
+			created_at: new Date(this.created_at).toISOString(),
+			id: this.id,
+			author: this.account.uri,
+			uri: `${config.http.base_url}/users/${this.account.id}/statuses/${this.id}`,
+			contents: [
+				{
+					content: this.content,
+					content_type: "text/html",
+				},
+				{
+					// Content converted to plaintext
+					content: htmlToText(this.content),
+					content_type: "text/plain",
+				},
+			],
+			// TODO: Add attachments
+			attachments: [],
+			is_sensitive: this.sensitive,
+			mentions: this.mentions.map(mention => mention.id),
+			// TODO: Add quotes
+			quotes: [],
+			replies_to: this.in_reply_to_post?.id
+				? [this.in_reply_to_post.id]
+				: [],
+			subject: this.spoiler_text,
+			extensions: {
+				"org.lysand:custom_emojis": {
+					emojis: this.emojis.map(emoji => emoji.toLysand()),
+				},
+				// TODO: Add polls and reactions
+			},
+		};
+	}
 }

database/entities/User.ts

@@ -18,6 +18,9 @@ import { Status, statusRelations } from "./Status";
 import { APISource } from "~types/entities/source";
 import { Relationship } from "./Relationship";
 import { Instance } from "./Instance";
+import { User as LysandUser } from "~types/lysand/Object";
+import { htmlToText } from "html-to-text";
+import { Emoji } from "./Emoji";
 
 export const userRelations = ["relationships", "pinned_notes", "instance"];
 
@@ -35,6 +38,12 @@ export class User extends BaseEntity {
 	@PrimaryGeneratedColumn("uuid")
 	id!: string;
 
+	/**
+	 * The user URI on the global network
+	 */
+	@Column("varchar")
+	uri!: string;
+
 	/**
 	 * The username for the user.
 	 */
@@ -82,6 +91,19 @@ export class User extends BaseEntity {
 	})
 	is_admin!: boolean;
 
+	@Column("jsonb", {
+		nullable: true,
+	})
+	endpoints!: {
+		liked: string;
+		disliked: string;
+		featured: string;
+		followers: string;
+		following: string;
+		inbox: string;
+		outbox: string;
+	} | null;
+
 	/**
 	 * The source for the user.
 	 */
@@ -147,6 +169,13 @@ export class User extends BaseEntity {
 	@JoinTable()
 	pinned_notes!: Status[];
 
+	/**
+	 * The emojis for the user.
+	 */
+	@ManyToMany(() => Emoji, emoji => emoji.id)
+	@JoinTable()
+	emojis!: Emoji[];
+
 	/**
 	 * Get the user's avatar in raw URL format
 	 * @param config The config to use
@@ -180,6 +209,76 @@ export class User extends BaseEntity {
 		return { user: await User.retrieveFromToken(token), token };
 	}
 
+	static async fetchRemoteUser(uri: string) {
+		const response = await fetch(uri, {
+			method: "GET",
+			headers: {
+				"Content-Type": "application/json",
+				Accept: "application/json",
+			},
+		});
+
+		const data = (await response.json()) as Partial<LysandUser>;
+
+		const user = new User();
+
+		if (
+			!(
+				data.id &&
+				data.username &&
+				data.uri &&
+				data.created_at &&
+				data.disliked &&
+				data.featured &&
+				data.liked &&
+				data.followers &&
+				data.following &&
+				data.inbox &&
+				data.outbox &&
+				data.public_key
+			)
+		) {
+			throw new Error("Invalid user data");
+		}
+
+		user.id = data.id;
+		user.username = data.username;
+		user.uri = data.uri;
+		user.created_at = new Date(data.created_at);
+		user.endpoints = {
+			disliked: data.disliked,
+			featured: data.featured,
+			liked: data.liked,
+			followers: data.followers,
+			following: data.following,
+			inbox: data.inbox,
+			outbox: data.outbox,
+		};
+
+		user.avatar = (data.avatar && data.avatar[0].content) || "";
+		user.header = (data.header && data.header[0].content) || "";
+		user.display_name = data.display_name ?? "";
+		// TODO: Add bio content types
+		user.note = data.bio?.[0].content ?? "";
+
+		// Parse emojis and add them to database
+		const emojis =
+			data.extensions?.["org.lysand:custom_emojis"]?.emojis ?? [];
+
+		for (const emoji of emojis) {
+			user.emojis.push(await Emoji.addIfNotExists(emoji));
+		}
+
+		user.public_key = data.public_key.public_key;
+
+		const uriData = new URL(data.uri);
+
+		user.instance = await Instance.addIfNotExists(uriData.origin);
+
+		await user.save();
+		return user;
+	}
+
 	/**
 	 * Fetches the list of followers associated with the actor and updates the user's followers
 	 */
@@ -229,6 +328,7 @@ export class User extends BaseEntity {
 		user.note = data.bio ?? "";
 		user.avatar = data.avatar ?? config.defaults.avatar;
 		user.header = data.header ?? config.defaults.avatar;
+		user.uri = `${config.http.base_url}/users/${user.id}`;
 
 		user.relationships = [];
 		user.instance = null;
@@ -348,15 +448,10 @@ export class User extends BaseEntity {
 	 * Generates keys for the user.
 	 */
 	async generateKeys(): Promise<void> {
-		// openssl genrsa -out private.pem 2048
-		// openssl rsa -in private.pem -outform PEM -pubout -out public.pem
-
 		const keys = await crypto.subtle.generateKey(
 			{
-				name: "RSASSA-PKCS1-v1_5",
-				hash: "SHA-256",
-				modulusLength: 4096,
-				publicExponent: new Uint8Array([0x01, 0x00, 0x01]),
+				name: "ed25519",
+				namedCurve: "ed25519",
 			},
 			true,
 			["sign", "verify"]
@@ -366,7 +461,7 @@ export class User extends BaseEntity {
 			String.fromCharCode.apply(null, [
 				...new Uint8Array(
 					// jesus help me what do these letters mean
-					await crypto.subtle.exportKey("pkcs8", keys.privateKey)
+					await crypto.subtle.exportKey("raw", keys.privateKey)
 				),
 			])
 		);
@@ -374,7 +469,7 @@ export class User extends BaseEntity {
 			String.fromCharCode(
 				...new Uint8Array(
 					// why is exporting a key so hard
-					await crypto.subtle.exportKey("spki", keys.publicKey)
+					await crypto.subtle.exportKey("raw", keys.publicKey)
 				)
 			)
 		);
@@ -431,7 +526,7 @@ export class User extends BaseEntity {
 			followers_count: follower_count,
 			following_count: following_count,
 			statuses_count: statusCount,
-			emojis: [],
+			emojis: await Promise.all(this.emojis.map(emoji => emoji.toAPI())),
 			fields: [],
 			bot: false,
 			source: isOwnAccount ? this.source : undefined,
@@ -451,4 +546,83 @@ export class User extends BaseEntity {
 			role: undefined,
 		};
 	}
+
+	/**
+	 * Should only return local users
+	 */
+	toLysand(): LysandUser {
+		if (this.instance !== null) {
+			throw new Error("Cannot convert remote user to Lysand format");
+		}
+
+		return {
+			id: this.id,
+			type: "User",
+			uri: this.uri,
+			bio: [
+				{
+					content: this.note,
+					content_type: "text/html",
+				},
+				{
+					content: htmlToText(this.note),
+					content_type: "text/plain",
+				},
+			],
+			created_at: new Date(this.created_at).toISOString(),
+			disliked: `${this.uri}/disliked`,
+			featured: `${this.uri}/featured`,
+			liked: `${this.uri}/liked`,
+			followers: `${this.uri}/followers`,
+			following: `${this.uri}/following`,
+			inbox: `${this.uri}/inbox`,
+			outbox: `${this.uri}/outbox`,
+			indexable: false,
+			username: this.username,
+			avatar: [
+				{
+					content: this.getAvatarUrl(getConfig()) || "",
+					content_type: `image/${this.avatar.split(".")[1]}`,
+				},
+			],
+			header: [
+				{
+					content: this.getHeaderUrl(getConfig()) || "",
+					content_type: `image/${this.header.split(".")[1]}`,
+				},
+			],
+			display_name: this.display_name,
+			fields: this.source.fields.map(field => ({
+				key: [
+					{
+						content: field.name,
+						content_type: "text/html",
+					},
+					{
+						content: htmlToText(field.name),
+						content_type: "text/plain",
+					},
+				],
+				value: [
+					{
+						content: field.value,
+						content_type: "text/html",
+					},
+					{
+						content: htmlToText(field.value),
+						content_type: "text/plain",
+					},
+				],
+			})),
+			public_key: {
+				actor: `${getConfig().http.base_url}/users/${this.id}`,
+				public_key: this.public_key,
+			},
+			extensions: {
+				"org.lysand:custom_emojis": {
+					emojis: this.emojis.map(emoji => emoji.toLysand()),
+				},
+			},
+		};
+	}
 }