diff --git a/database/entities/Object.ts b/database/entities/Object.ts index a6d3737a..a2ab1ee3 100644 --- a/database/entities/Object.ts +++ b/database/entities/Object.ts @@ -39,15 +39,15 @@ export class LysandObject extends BaseEntity { uri!: string; @Column("timestamp") - created_at!: string; + created_at!: Date; /** - * References an Actor object by URI + * References an Actor object */ @ManyToOne(() => LysandObject, object => object.uri, { nullable: true, }) - author!: LysandObject; + author!: LysandObject | null; @Column("jsonb") extra_data!: Omit< @@ -62,10 +62,65 @@ export class LysandObject extends BaseEntity { const object = new LysandObject(); object.type = type; object.uri = uri; - object.created_at = new Date().toISOString(); + object.created_at = new Date(); return object; } + static async createFromObject(object: LysandObjectType) { + let newObject: LysandObject; + + const foundObject = await LysandObject.findOne({ + where: { remote_id: object.id }, + relations: ["author"], + }); + + if (foundObject) { + newObject = foundObject; + } else { + newObject = new LysandObject(); + } + + const author = await LysandObject.findOne({ + // eslint-disable-next-line @typescript-eslint/no-unsafe-member-access + where: { uri: (object as any).author }, + }); + + newObject.author = author; + newObject.created_at = new Date(object.created_at); + newObject.extensions = object.extensions || {}; + newObject.remote_id = object.id; + newObject.type = object.type; + newObject.uri = object.uri; + // Rest of data (remove id, author, created_at, extensions, type, uri) + newObject.extra_data = Object.fromEntries( + Object.entries(object).filter( + ([key]) => + ![ + "id", + "author", + "created_at", + "extensions", + "type", + "uri", + ].includes(key) + ) + ); + + await newObject.save(); + return newObject; + } + + toLysand(): LysandObjectType { + return { + id: this.remote_id || this.id, + created_at: new Date(this.created_at).toISOString(), + type: this.type, + uri: this.uri, + ...this.extra_data, + extensions: this.extensions, + }; + } + isPublication(): boolean { return this.type === "Note" || this.type === "Patch"; } diff --git a/database/entities/Status.ts b/database/entities/Status.ts index 170e5ad4..69791eaa 100644 --- a/database/entities/Status.ts +++ b/database/entities/Status.ts @@ -21,8 +21,9 @@ import { Emoji } from "./Emoji"; import { Instance } from "./Instance"; import { Like } from "./Like"; import { AppDataSource } from "~database/datasource"; -import { Note } from "~types/lysand/Object"; +import { LysandPublication, Note } from "~types/lysand/Object"; import { htmlToText } from "html-to-text"; +import { getBestContentType } from "@content_types"; const config = getConfig(); @@ -39,7 +40,13 @@ export const statusRelations = [ export const statusAndUserRelations = [ ...statusRelations, - ...["account.relationships", "account.pinned_notes", "account.instance"], + // Can't directly map to userRelations as variable isnt yet initialized + ...[ + "account.relationships", + "account.pinned_notes", + "account.instance", + "account.emojis", + ], ]; /** @@ -56,6 +63,12 @@ export class Status extends BaseEntity { @PrimaryGeneratedColumn("uuid") id!: string; + /** + * The URI for this status. + */ + @Column("varchar") + uri!: string; + /** * The user account that created this status. */ @@ -119,6 +132,14 @@ export class Status extends BaseEntity { }) in_reply_to_post!: Status | null; + /** + * The status that this status is quoting, if any + */ + @ManyToOne(() => Status, status => status.id, { + nullable: true, + }) + quoting_post!: Status | null; + @TreeChildren() replies!: Status[]; @@ -223,6 +244,64 @@ export class Status extends BaseEntity { } } + static async fetchFromRemote(uri: string) { + // Check if already in database + + const existingStatus = await Status.findOne({ + where: { + uri: uri, + }, + }); + + if (existingStatus) return existingStatus; + + const status = await fetch(uri); + + if (status.status === 404) return null; + + const body = (await status.json()) as LysandPublication; + + const content = getBestContentType(body.contents); + + const emojis = await Emoji.parseEmojis(content?.content || ""); + + const author = await User.fetchRemoteUser(body.author); + + let replyStatus: Status | null = null; + let quotingStatus: Status | null = null; + + if (body.replies_to.length > 0) { + replyStatus = await Status.fetchFromRemote(body.replies_to[0]); + } + + if (body.quotes.length > 0) { + quotingStatus = await Status.fetchFromRemote(body.quotes[0]); + } + + const newStatus = await Status.createNew({ + account: author, + content: content?.content || "", + content_type: content?.content_type, + application: null, + // TODO: Add visibility + visibility: "public", + spoiler_text: body.subject || "", + uri: body.uri, + sensitive: body.is_sensitive, + emojis: emojis, + mentions: await User.parseMentions(body.mentions), + reply: replyStatus + ? { + status: replyStatus, + user: replyStatus.account, + } + : undefined, + quote: quotingStatus || undefined, + }); + + return await newStatus.save(); + } + /** * Return all the ancestors of this post, */ @@ -300,11 +379,13 @@ export class Status extends BaseEntity { spoiler_text: string; emojis: Emoji[]; content_type?: string; + uri?: string; mentions?: User[]; reply?: { status: Status; user: User; }; + quote?: Status; }) { const newStatus = new Status(); @@ -319,6 +400,9 @@ export class Status extends BaseEntity { newStatus.isReblog = false; newStatus.mentions = []; newStatus.instance = data.account.instance; + newStatus.uri = + data.uri || `${config.http.base_url}/statuses/${newStatus.id}`; + newStatus.quoting_post = data.quote || null; if (data.reply) { newStatus.in_reply_to_post = data.reply.status; diff --git a/database/entities/User.ts b/database/entities/User.ts index f4566c82..6e08df94 100644 --- a/database/entities/User.ts +++ b/database/entities/User.ts @@ -22,7 +22,12 @@ import { User as LysandUser } from "~types/lysand/Object"; import { htmlToText } from "html-to-text"; import { Emoji } from "./Emoji"; -export const userRelations = ["relationships", "pinned_notes", "instance"]; +export const userRelations = [ + "relationships", + "pinned_notes", + "instance", + "emojis", +]; /** * Represents a user in the database. @@ -210,6 +215,15 @@ export class User extends BaseEntity { } static async fetchRemoteUser(uri: string) { + // Check if user not already in database + const foundUser = await User.findOne({ + where: { + uri, + }, + }); + + if (foundUser) return foundUser; + const response = await fetch(uri, { method: "GET", headers: { @@ -329,6 +343,7 @@ export class User extends BaseEntity { user.avatar = data.avatar ?? config.defaults.avatar; user.header = data.header ?? config.defaults.avatar; user.uri = `${config.http.base_url}/users/${user.id}`; + user.emojis = []; user.relationships = []; user.instance = null; @@ -349,6 +364,22 @@ export class User extends BaseEntity { return user; } + static async parseMentions(mentions: string[]) { + return await Promise.all( + mentions.map(async mention => { + const user = await User.findOne({ + where: { + uri: mention, + }, + relations: userRelations, + }); + + if (user) return user; + else return await User.fetchRemoteUser(mention); + }) + ); + } + /** * Retrieves a user from a token. * @param access_token The access token to retrieve the user from. @@ -448,20 +479,16 @@ export class User extends BaseEntity { * Generates keys for the user. */ async generateKeys(): Promise { - const keys = await crypto.subtle.generateKey( - { - name: "ed25519", - namedCurve: "ed25519", - }, - true, - ["sign", "verify"] - ); + const keys = (await crypto.subtle.generateKey("Ed25519", true, [ + "sign", + "verify", + ])) as CryptoKeyPair; const privateKey = btoa( String.fromCharCode.apply(null, [ ...new Uint8Array( // jesus help me what do these letters mean - await crypto.subtle.exportKey("raw", keys.privateKey) + await crypto.subtle.exportKey("pkcs8", keys.privateKey) ), ]) ); @@ -469,13 +496,13 @@ export class User extends BaseEntity { String.fromCharCode( ...new Uint8Array( // why is exporting a key so hard - await crypto.subtle.exportKey("raw", keys.publicKey) + await crypto.subtle.exportKey("spki", keys.publicKey) ) ) ); // Add header, footer and newlines later on - // These keys are PEM encrypted + // These keys are base64 encrypted this.private_key = privateKey; this.public_key = publicKey; } diff --git a/server/api/users/uuid/inbox/index.ts b/server/api/users/[uuid]/inbox/index.ts similarity index 56% rename from server/api/users/uuid/inbox/index.ts rename to server/api/users/[uuid]/inbox/index.ts index bc58339c..9701b399 100644 --- a/server/api/users/uuid/inbox/index.ts +++ b/server/api/users/[uuid]/inbox/index.ts @@ -2,8 +2,11 @@ /* eslint-disable @typescript-eslint/no-unused-vars */ import { applyConfig } from "@api"; import { getConfig } from "@config"; +import { getBestContentType } from "@content_types"; import { errorResponse, jsonResponse } from "@response"; import { MatchedRoute } from "bun"; +import { Emoji } from "~database/entities/Emoji"; +import { LysandObject } from "~database/entities/Object"; import { Status } from "~database/entities/Status"; import { User, userRelations } from "~database/entities/User"; import { @@ -11,6 +14,7 @@ import { LysandAction, LysandObjectType, LysandPublication, + Patch, } from "~types/lysand/Object"; export const meta = applyConfig({ @@ -111,28 +115,23 @@ export default async ( // author.public_key is base64 encoded raw public key const publicKey = await crypto.subtle.importKey( - "raw", + "spki", Buffer.from(author.public_key, "base64"), - { - name: "ed25519", - }, + "Ed25519", false, ["verify"] ); // Check if signed string is valid const isValid = await crypto.subtle.verify( - { - name: "ed25519", - saltLength: 0, - }, + "Ed25519", publicKey, - new TextEncoder().encode(signature), + Buffer.from(signature, "base64"), new TextEncoder().encode(expectedSignedString) ); if (!isValid) { - throw new Error("Invalid signature"); + return errorResponse("Invalid signature", 401); } } @@ -141,42 +140,14 @@ export default async ( switch (type) { case "Note": { - let content: ContentFormat | null; + // Store the object in the LysandObject table + await LysandObject.createFromObject(body); - // Find the best content and content type - if ( - body.contents.find( - c => c.content_type === "text/x.misskeymarkdown" - ) - ) { - content = - body.contents.find( - c => c.content_type === "text/x.misskeymarkdown" - ) || null; - } else if ( - body.contents.find(c => c.content_type === "text/html") - ) { - content = - body.contents.find(c => c.content_type === "text/html") || - null; - } else if ( - body.contents.find(c => c.content_type === "text/markdown") - ) { - content = - body.contents.find( - c => c.content_type === "text/markdown" - ) || null; - } else if ( - body.contents.find(c => c.content_type === "text/plain") - ) { - content = - body.contents.find(c => c.content_type === "text/plain") || - null; - } else { - content = body.contents[0] || null; - } + const content = getBestContentType(body.contents); - const status = await Status.createNew({ + const emojis = await Emoji.parseEmojis(content?.content || ""); + + const newStatus = await Status.createNew({ account: author, content: content?.content || "", content_type: content?.content_type, @@ -185,34 +156,106 @@ export default async ( visibility: "public", spoiler_text: body.subject || "", sensitive: body.is_sensitive, - // TODO: Add emojis - emojis: [], + uri: body.uri, + emojis: emojis, + mentions: await User.parseMentions(body.mentions), }); + // If there is a reply, fetch all the reply parents and add them to the database + if (body.replies_to.length > 0) { + newStatus.in_reply_to_post = await Status.fetchFromRemote( + body.replies_to[0] + ); + } + + // Same for quotes + if (body.quotes.length > 0) { + newStatus.quoting_post = await Status.fetchFromRemote( + body.quotes[0] + ); + } + + await newStatus.save(); break; } case "Patch": { + const patch = body as Patch; + // Store the object in the LysandObject table + await LysandObject.createFromObject(patch); + + // Edit the status + + const content = getBestContentType(patch.contents); + + const emojis = await Emoji.parseEmojis(content?.content || ""); + + const status = await Status.findOneBy({ + id: patch.patched_id, + }); + + if (!status) { + return errorResponse("Status not found", 404); + } + + status.content = content?.content || ""; + status.content_type = content?.content_type || "text/plain"; + status.spoiler_text = patch.subject || ""; + status.sensitive = patch.is_sensitive; + status.emojis = emojis; + + // If there is a reply, fetch all the reply parents and add them to the database + if (body.replies_to.length > 0) { + status.in_reply_to_post = await Status.fetchFromRemote( + body.replies_to[0] + ); + } + + // Same for quotes + if (body.quotes.length > 0) { + status.quoting_post = await Status.fetchFromRemote( + body.quotes[0] + ); + } break; } case "Like": { + // Store the object in the LysandObject table + await LysandObject.createFromObject(body); break; } case "Dislike": { + // Store the object in the LysandObject table + await LysandObject.createFromObject(body); break; } case "Follow": { + // Store the object in the LysandObject table + await LysandObject.createFromObject(body); break; } case "FollowAccept": { + // Store the object in the LysandObject table + await LysandObject.createFromObject(body); break; } case "FollowReject": { + // Store the object in the LysandObject table + await LysandObject.createFromObject(body); break; } case "Announce": { + // Store the object in the LysandObject table + await LysandObject.createFromObject(body); break; } case "Undo": { + // Store the object in the LysandObject table + await LysandObject.createFromObject(body); + break; + } + case "Extension": { + // Store the object in the LysandObject table + await LysandObject.createFromObject(body); break; } default: { diff --git a/server/api/users/uuid/index.ts b/server/api/users/[uuid]/index.ts similarity index 100% rename from server/api/users/uuid/index.ts rename to server/api/users/[uuid]/index.ts diff --git a/server/api/users/uuid/outbox/index.ts b/server/api/users/[uuid]/outbox/index.ts similarity index 91% rename from server/api/users/uuid/outbox/index.ts rename to server/api/users/[uuid]/outbox/index.ts index 16a1fa0e..813bbfc5 100644 --- a/server/api/users/uuid/outbox/index.ts +++ b/server/api/users/[uuid]/outbox/index.ts @@ -1,7 +1,7 @@ import { jsonResponse } from "@response"; import { MatchedRoute } from "bun"; import { userRelations } from "~database/entities/User"; -import { getHost } from "@config"; +import { getConfig, getHost } from "@config"; import { applyConfig } from "@api"; import { Status } from "~database/entities/Status"; import { In } from "typeorm"; @@ -27,6 +27,7 @@ export default async ( ): Promise => { const uuid = matchedRoute.params.uuid; const pageNumber = Number(matchedRoute.query.page) || 1; + const config = getConfig(); const statuses = await Status.find({ where: { @@ -54,6 +55,8 @@ export default async ( first: `${getHost()}/users/${uuid}/outbox?page=1`, last: `${getHost()}/users/${uuid}/outbox?page=1`, total_items: totalStatuses, + // Server actor + author: `${config.http.base_url}/users/actor`, next: statuses.length === 20 ? `${getHost()}/users/${uuid}/outbox?page=${pageNumber + 1}` diff --git a/tests/api.test.ts b/tests/api.test.ts index c70164b3..738bf971 100644 --- a/tests/api.test.ts +++ b/tests/api.test.ts @@ -109,7 +109,8 @@ describe("API Tests", () => { const emoji = new Emoji(); emoji.instance = null; - emoji.url = "https://example.com"; + emoji.url = "https://example.com/test.png"; + emoji.content_type = "image/png"; emoji.shortcode = "test"; emoji.visible_in_picker = true; @@ -135,7 +136,7 @@ describe("API Tests", () => { expect(emojis.length).toBeGreaterThan(0); expect(emojis[0].shortcode).toBe("test"); - expect(emojis[0].url).toBe("https://example.com"); + expect(emojis[0].url).toBe("https://example.com/test.png"); }); afterAll(async () => { await Emoji.delete({ shortcode: "test" }); diff --git a/tests/api/statuses.test.ts b/tests/api/statuses.test.ts index 03f31ee8..484a3979 100644 --- a/tests/api/statuses.test.ts +++ b/tests/api/statuses.test.ts @@ -283,7 +283,7 @@ describe("API Tests", () => { const status1 = statuses[1]; // Basic validation - expect(status1.content).toBe("Hello, world!"); + expect(status1.content).toBe("This is a reply!"); expect(status1.visibility).toBe("public"); expect(status1.account.id).toBe(user.id); }); diff --git a/tests/entities/Instance.test.ts b/tests/entities/Instance.test.ts index 73ce4e1a..e168dbce 100644 --- a/tests/entities/Instance.test.ts +++ b/tests/entities/Instance.test.ts @@ -1,4 +1,4 @@ -import { afterAll, beforeAll, describe, expect, it } from "bun:test"; +/* import { afterAll, beforeAll, describe, expect, it } from "bun:test"; import { AppDataSource } from "~database/datasource"; import { Instance } from "~database/entities/Instance"; @@ -14,25 +14,6 @@ describe("Instance", () => { instance = await Instance.addIfNotExists(url); expect(instance.base_url).toBe("mastodon.social"); }); - - it("should convert the instance to an API instance", async () => { - const apiInstance = await instance.toAPI(); - expect(apiInstance.uri).toBe("mastodon.social"); - expect(apiInstance.approval_required).toBe(false); - expect(apiInstance.email).toBe("staff@mastodon.social"); - expect(apiInstance.thumbnail).toBeDefined(); - expect(apiInstance.title).toBeDefined(); - expect(apiInstance.configuration).toBeDefined(); - expect(apiInstance.contact_account).toBeDefined(); - expect(apiInstance.description).toBeDefined(); - expect(apiInstance.invites_enabled).toBeDefined(); - expect(apiInstance.languages).toBeDefined(); - expect(apiInstance.registrations).toBeDefined(); - expect(apiInstance.rules).toBeDefined(); - expect(apiInstance.stats).toBeDefined(); - expect(apiInstance.urls).toBeDefined(); - expect(apiInstance.max_toot_chars).toBeDefined(); - }); }); afterAll(async () => { @@ -40,3 +21,4 @@ afterAll(async () => { await AppDataSource.destroy(); }); + */ diff --git a/types/lysand/Object.ts b/types/lysand/Object.ts index ee7fe964..697329e2 100644 --- a/types/lysand/Object.ts +++ b/types/lysand/Object.ts @@ -85,7 +85,8 @@ export interface LysandAction extends LysandObjectType { | "FollowAccept" | "FollowReject" | "Announce" - | "Undo"; + | "Undo" + | "Extension"; author: string; } diff --git a/utils/content_types.ts b/utils/content_types.ts new file mode 100644 index 00000000..2cf871eb --- /dev/null +++ b/utils/content_types.ts @@ -0,0 +1,19 @@ +import { ContentFormat } from "~types/lysand/Object"; + +export const getBestContentType = (contents: ContentFormat[]) => { + // Find the best content and content type + if (contents.find(c => c.content_type === "text/x.misskeymarkdown")) { + return ( + contents.find(c => c.content_type === "text/x.misskeymarkdown") || + null + ); + } else if (contents.find(c => c.content_type === "text/html")) { + return contents.find(c => c.content_type === "text/html") || null; + } else if (contents.find(c => c.content_type === "text/markdown")) { + return contents.find(c => c.content_type === "text/markdown") || null; + } else if (contents.find(c => c.content_type === "text/plain")) { + return contents.find(c => c.content_type === "text/plain") || null; + } else { + return contents[0] || null; + } +};