diff --git a/.github/config.workflow.toml b/.github/config.workflow.toml index 447a42f6..01323099 100644 --- a/.github/config.workflow.toml +++ b/.github/config.workflow.toml @@ -19,10 +19,10 @@ password = "" database = 1 enabled = false -[meilisearch] +[sonic] host = "localhost" port = 40007 -api_key = "" +password = "" enabled = false [signups] diff --git a/CHANGELOG.md b/CHANGELOG.md index 59585295..d9e9a692 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,7 @@ Lysand Server `0.7.0` is backwards compatible with `0.6.0`. However, some new fe ## Features - Upgrade Bun to `1.1.17`. This brings performance upgrades and better stability. +- Added support for the [Sonic](https://github.com/valeriansaliou/sonic) search indexer. - Note deletions are now federated. - Note edits are now federated. - Added option for more federation debug logging. @@ -38,6 +39,7 @@ Lysand Server `0.7.0` is backwards compatible with `0.6.0`. However, some new fe ## Removals - Remove old logging system, to be replaced by a new one. +- Removed Meilisearch support, in favor of Sonic. Follow instructions in the [installation guide](docs/installation.md) to set up Sonic. ## Miscellaneous diff --git a/bun.lockb b/bun.lockb index 31acd855..89d216c0 100755 Binary files a/bun.lockb and b/bun.lockb differ diff --git a/classes/search/search-manager.ts b/classes/search/search-manager.ts new file mode 100644 index 00000000..f1f5b601 --- /dev/null +++ b/classes/search/search-manager.ts @@ -0,0 +1,283 @@ +/** + * @file search-manager.ts + * @description Sonic search integration for indexing and searching accounts and statuses + */ + +import { getLogger } from "@logtape/logtape"; +import { + Ingest as SonicChannelIngest, + Search as SonicChannelSearch, +} from "sonic-channel"; +import { db } from "~/drizzle/db"; +import { type Config, config } from "~/packages/config-manager"; +import { Note } from "~/packages/database-interface/note"; +import { User } from "~/packages/database-interface/user"; + +/** + * Enum for Sonic index types + */ +export enum SonicIndexType { + Accounts = "accounts", + Statuses = "statuses", +} + +/** + * Class for managing Sonic search operations + */ +export class SonicSearchManager { + private searchChannel: SonicChannelSearch; + private ingestChannel: SonicChannelIngest; + private logger = getLogger("sonic"); + + /** + * @param config Configuration for Sonic + */ + constructor(private config: Config) { + this.searchChannel = new SonicChannelSearch({ + host: config.sonic.host, + port: config.sonic.port, + auth: config.sonic.password, + }); + + this.ingestChannel = new SonicChannelIngest({ + host: config.sonic.host, + port: config.sonic.port, + auth: config.sonic.password, + }); + } + + /** + * Connect to Sonic + */ + async connect(): Promise { + if (!this.config.sonic.enabled) { + this.logger.info`Sonic search is disabled`; + return; + } + + this.logger.info`Connecting to Sonic...`; + + // Connect to Sonic + await new Promise((resolve, reject) => { + this.searchChannel.connect({ + connected: () => { + this.logger.info`Connected to Sonic Search Channel`; + resolve(true); + }, + disconnected: () => + this.logger + .error`Disconnected from Sonic Search Channel. You might be using an incorrect password.`, + timeout: () => + this.logger + .error`Sonic Search Channel connection timed out`, + retrying: () => + this.logger + .warn`Retrying connection to Sonic Search Channel`, + error: (error) => { + this.logger + .error`Failed to connect to Sonic Search Channel: ${error}`; + reject(error); + }, + }); + }); + + await new Promise((resolve, reject) => { + this.ingestChannel.connect({ + connected: () => { + this.logger.info`Connected to Sonic Ingest Channel`; + resolve(true); + }, + disconnected: () => + this.logger.error`Disconnected from Sonic Ingest Channel`, + timeout: () => + this.logger + .error`Sonic Ingest Channel connection timed out`, + retrying: () => + this.logger + .warn`Retrying connection to Sonic Ingest Channel`, + error: (error) => { + this.logger + .error`Failed to connect to Sonic Ingest Channel: ${error}`; + reject(error); + }, + }); + }); + + try { + await Promise.all([ + this.searchChannel.ping(), + this.ingestChannel.ping(), + ]); + this.logger.info`Connected to Sonic`; + } catch (error) { + this.logger.fatal`Error while connecting to Sonic: ${error}`; + throw error; + } + } + + /** + * Add a user to Sonic + * @param user User to add + */ + async addUser(user: User): Promise { + if (!this.config.sonic.enabled) { + return; + } + + try { + await this.ingestChannel.push( + SonicIndexType.Accounts, + "users", + user.id, + `${user.data.username} ${user.data.displayName} ${user.data.note}`, + ); + } catch (error) { + this.logger.error`Failed to add user to Sonic: ${error}`; + } + } + + /** + * Get a batch of accounts from the database + * @param n Batch number + * @param batchSize Size of the batch + */ + private async getNthDatabaseAccountBatch( + n: number, + batchSize = 1000, + ): Promise[]> { + return db.query.Users.findMany({ + offset: n * batchSize, + limit: batchSize, + columns: { + id: true, + username: true, + displayName: true, + note: true, + createdAt: true, + }, + orderBy: (user, { asc }) => asc(user.createdAt), + }); + } + + /** + * Get a batch of statuses from the database + * @param n Batch number + * @param batchSize Size of the batch + */ + private async getNthDatabaseStatusBatch( + n: number, + batchSize = 1000, + ): Promise[]> { + return db.query.Notes.findMany({ + offset: n * batchSize, + limit: batchSize, + columns: { + id: true, + content: true, + createdAt: true, + }, + orderBy: (status, { asc }) => asc(status.createdAt), + }); + } + + /** + * Rebuild search indexes + * @param indexes Indexes to rebuild + * @param batchSize Size of each batch + */ + async rebuildSearchIndexes( + indexes: SonicIndexType[], + batchSize = 100, + ): Promise { + for (const index of indexes) { + if (index === SonicIndexType.Accounts) { + await this.rebuildAccountsIndex(batchSize); + } else if (index === SonicIndexType.Statuses) { + await this.rebuildStatusesIndex(batchSize); + } + } + } + + /** + * Rebuild accounts index + * @param batchSize Size of each batch + */ + private async rebuildAccountsIndex(batchSize: number): Promise { + const accountCount = await User.getCount(); + const batchCount = Math.ceil(accountCount / batchSize); + + for (let i = 0; i < batchCount; i++) { + const accounts = await this.getNthDatabaseAccountBatch( + i, + batchSize, + ); + await Promise.all( + accounts.map((account) => + this.ingestChannel.push( + SonicIndexType.Accounts, + "users", + account.id as string, + `${account.username} ${account.displayName} ${account.note}`, + ), + ), + ); + this.logger.info`Indexed accounts batch ${i + 1}/${batchCount}`; + } + } + + /** + * Rebuild statuses index + * @param batchSize Size of each batch + */ + private async rebuildStatusesIndex(batchSize: number): Promise { + const statusCount = await Note.getCount(); + const batchCount = Math.ceil(statusCount / batchSize); + + for (let i = 0; i < batchCount; i++) { + const statuses = await this.getNthDatabaseStatusBatch(i, batchSize); + await Promise.all( + statuses.map((status) => + this.ingestChannel.push( + SonicIndexType.Statuses, + "notes", + status.id as string, + status.content as string, + ), + ), + ); + this.logger.info`Indexed statuses batch ${i + 1}/${batchCount}`; + } + } + + /** + * Search for accounts + * @param query Search query + * @param limit Maximum number of results + * @param offset Offset for pagination + */ + searchAccounts(query: string, limit = 10, offset = 0): Promise { + return this.searchChannel.query( + SonicIndexType.Accounts, + "users", + query, + { limit, offset }, + ); + } + + /** + * Search for statuses + * @param query Search query + * @param limit Maximum number of results + * @param offset Offset for pagination + */ + searchStatuses(query: string, limit = 10, offset = 0): Promise { + return this.searchChannel.query( + SonicIndexType.Statuses, + "notes", + query, + { limit, offset }, + ); + } +} + +export const searchManager = new SonicSearchManager(config); diff --git a/config/config.example.toml b/config/config.example.toml index fc9854dd..c80b4d80 100644 --- a/config/config.example.toml +++ b/config/config.example.toml @@ -24,11 +24,11 @@ password = "" database = 1 enabled = false -[meilisearch] -# If Meilisearch is not configured, search will not be enabled +[sonic] +# If Sonic is not configured, search will not be enabled host = "localhost" port = 40007 -api_key = "" +password = "" enabled = true [signups] diff --git a/docker-compose.yml b/docker-compose.yml index c6535641..f8bbda8e 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -14,7 +14,7 @@ services: depends_on: - db - redis - - meilisearch + - sonic - fe fe: @@ -48,18 +48,11 @@ services: networks: - lysand-net - meilisearch: - stdin_open: true - environment: - - MEILI_MASTER_KEY=__________________ - tty: true - networks: - - lysand-net + sonic: volumes: - - ./meili-data:/meili_data - image: getmeili/meilisearch:v1.7 - container_name: lysand-meilisearch - restart: unless-stopped + - ./config.cfg:/etc/sonic.cfg + - ./store/:/var/lib/sonic/store/ + image: valeriansaliou/sonic:v1.4.9 networks: lysand-net: \ No newline at end of file diff --git a/docs/installation.md b/docs/installation.md index f1a2ea35..6b7cdf36 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -6,7 +6,7 @@ - Lysand will work on lower versions than 1.1.17, but only the latest version is supported - A PostgreSQL database - (Optional but recommended) A Linux-based operating system -- (Optional if you want search) A working Meilisearch instance +- (Optional if you want search) A working [Sonic](https://github.com/valeriansaliou/sonic) instance > [!WARNING] > Lysand has not been tested on Windows or macOS. It is recommended to use a Linux-based operating system to run Lysand. @@ -59,7 +59,7 @@ bun install 1. Set up a PostgreSQL database (you need a special extension, please look at [the database documentation](database.md)) 2. (If you want search) -Create a Meilisearch instance (using Docker is recommended). For a [`docker-compose`] file, copy the `meilisearch` service from the [`docker-compose.yml`](../docker-compose.yml) file. +Create a [Sonic](https://github.com/valeriansaliou/sonic) instance (using Docker is recommended). For a [`docker-compose`] file, copy the `sonic` service from the [`docker-compose.yml`](../docker-compose.yml) file. Don't forget to fill in the `config.cfg` for Sonic! 1. Build everything: diff --git a/package.json b/package.json index b461201b..4e7252d8 100644 --- a/package.json +++ b/package.json @@ -130,13 +130,13 @@ "markdown-it-anchor": "^9.0.1", "markdown-it-container": "^4.0.0", "markdown-it-toc-done-right": "^4.2.0", - "meilisearch": "^0.40.0", "mime-types": "^2.1.35", "oauth4webapi": "^2.11.1", "ora": "^8.0.1", "pg": "^8.12.0", "qs": "^6.12.1", "sharp": "^0.33.4", + "sonic-channel": "^1.3.1", "string-comparison": "^1.3.0", "stringify-entities": "^4.0.4", "strip-ansi": "^7.1.0", diff --git a/packages/config-manager/config.type.ts b/packages/config-manager/config.type.ts index 7bc4867e..af9d202d 100644 --- a/packages/config-manager/config.type.ts +++ b/packages/config-manager/config.type.ts @@ -76,7 +76,7 @@ export const configValidator = z.object({ enabled: false, }), }), - meilisearch: z.object({ + sonic: z.object({ host: z.string().min(1).default("localhost"), port: z .number() @@ -84,7 +84,7 @@ export const configValidator = z.object({ .min(1) .max(2 ** 16 - 1) .default(7700), - api_key: z.string(), + password: z.string(), enabled: z.boolean().default(false), }), signups: z.object({ diff --git a/packages/database-interface/user.ts b/packages/database-interface/user.ts index 9469b106..10da1395 100644 --- a/packages/database-interface/user.ts +++ b/packages/database-interface/user.ts @@ -1,7 +1,6 @@ import { idValidator } from "@/api"; import { getBestContentType, urlToContentFormat } from "@/content_types"; import { randomString } from "@/math"; -import { addUserToMeilisearch } from "@/meilisearch"; import { proxyUrl } from "@/response"; import type { Account as ApiAccount, @@ -31,6 +30,7 @@ import { type UserWithRelations, findManyUsers, } from "~/classes/functions/user"; +import { searchManager } from "~/classes/search/search-manager"; import { db } from "~/drizzle/db"; import { EmojiToUser, @@ -294,8 +294,8 @@ export class User extends BaseInterface { throw new Error("Failed to save user from remote"); } - // Add to Meilisearch - await addUserToMeilisearch(finalUser); + // Add to search index + await searchManager.addUser(finalUser); return finalUser; } @@ -477,8 +477,8 @@ export class User extends BaseInterface { throw new Error("Failed to create user"); } - // Add to Meilisearch - await addUserToMeilisearch(finalUser); + // Add to search index + await searchManager.addUser(finalUser); return finalUser; } diff --git a/server/api/api/v2/search/index.ts b/server/api/api/v2/search/index.ts index 57952c9d..b2b0636a 100644 --- a/server/api/api/v2/search/index.ts +++ b/server/api/api/v2/search/index.ts @@ -1,5 +1,4 @@ import { applyConfig, auth, handleZodError, userAddressValidator } from "@/api"; -import { MeiliIndexType, meilisearch } from "@/meilisearch"; import { errorResponse, jsonResponse } from "@/response"; import { zValidator } from "@hono/zod-validator"; import { getLogger } from "@logtape/logtape"; @@ -7,6 +6,7 @@ import { and, eq, inArray, sql } from "drizzle-orm"; import type { Hono } from "hono"; import { z } from "zod"; import { resolveWebFinger } from "~/classes/functions/user"; +import { searchManager } from "~/classes/search/search-manager"; import { db } from "~/drizzle/db"; import { Instances, Notes, RolePermissions, Users } from "~/drizzle/schema"; import { config } from "~/packages/config-manager"; @@ -65,12 +65,19 @@ export default (app: Hono) => ); } - if (!config.meilisearch.enabled) { - return errorResponse("Meilisearch is not enabled", 501); + if (!q) { + return errorResponse("Query is required", 400); } - let accountResults: { id: string }[] = []; - let statusResults: { id: string }[] = []; + if (!config.sonic.enabled) { + return errorResponse( + "Search is not enabled by your server administrator", + 501, + ); + } + + let accountResults: string[] = []; + let statusResults: string[] = []; if (!type || type === "accounts") { // Check if q is matching format username@domain.com or @username@domain.com @@ -132,34 +139,26 @@ export default (app: Hono) => } } - accountResults = ( - await meilisearch.index(MeiliIndexType.Accounts).search<{ - id: string; - }>(q, { - limit: Number(limit) || 10, - offset: Number(offset) || 0, - sort: ["createdAt:desc"], - }) - ).hits; + accountResults = await searchManager.searchAccounts( + q, + Number(limit) || 10, + Number(offset) || 0, + ); } if (!type || type === "statuses") { - statusResults = ( - await meilisearch.index(MeiliIndexType.Statuses).search<{ - id: string; - }>(q, { - limit: Number(limit) || 10, - offset: Number(offset) || 0, - sort: ["createdAt:desc"], - }) - ).hits; + statusResults = await searchManager.searchStatuses( + q, + Number(limit) || 10, + Number(offset) || 0, + ); } const accounts = await User.manyFromSql( and( inArray( Users.id, - accountResults.map((hit) => hit.id), + accountResults.map((hit) => hit), ), self ? sql`EXISTS (SELECT 1 FROM Relationships WHERE Relationships.subjectId = ${ @@ -175,7 +174,7 @@ export default (app: Hono) => and( inArray( Notes.id, - statusResults.map((hit) => hit.id), + statusResults.map((hit) => hit), ), account_id ? eq(Notes.authorId, account_id) : undefined, self diff --git a/setup.ts b/setup.ts index f468bec3..4f7bf6a1 100644 --- a/setup.ts +++ b/setup.ts @@ -1,10 +1,10 @@ import { checkConfig } from "@/init"; import { configureLoggers } from "@/loggers"; -import { connectMeili } from "@/meilisearch"; import { getLogger } from "@logtape/logtape"; import { config } from "config-manager"; import { setupDatabase } from "~/drizzle/db"; import { Note } from "~/packages/database-interface/note"; +import { searchManager } from "./classes/search/search-manager"; const timeAtStart = performance.now(); @@ -16,8 +16,8 @@ serverLogger.info`Starting Lysand...`; await setupDatabase(); -if (config.meilisearch.enabled) { - await connectMeili(); +if (config.sonic.enabled) { + await searchManager.connect(); } process.on("SIGINT", () => { diff --git a/tests/utils.ts b/tests/utils.ts index 07de9a5b..31e0e331 100644 --- a/tests/utils.ts +++ b/tests/utils.ts @@ -4,13 +4,18 @@ import { solveChallenge } from "altcha-lib"; import { asc, inArray, like } from "drizzle-orm"; import { appFactory } from "~/app"; import type { Status } from "~/classes/functions/status"; +import { searchManager } from "~/classes/search/search-manager"; import { db } from "~/drizzle/db"; import { setupDatabase } from "~/drizzle/db"; import { Notes, Tokens, Users } from "~/drizzle/schema"; +import { config } from "~/packages/config-manager"; import { Note } from "~/packages/database-interface/note"; import { User } from "~/packages/database-interface/user"; await setupDatabase(); +if (config.sonic.enabled) { + await searchManager.connect(); +} /** * This allows us to send a test request to the server even when it isnt running diff --git a/utils/loggers.ts b/utils/loggers.ts index 0e372a31..c4377f8d 100644 --- a/utils/loggers.ts +++ b/utils/loggers.ts @@ -221,7 +221,7 @@ export const configureLoggers = (silent = false) => filters: ["configFilter"], }, { - category: "meilisearch", + category: "sonic", sinks: ["console", "file"], filters: ["configFilter"], }, diff --git a/utils/meilisearch.ts b/utils/meilisearch.ts deleted file mode 100644 index 5a75f205..00000000 --- a/utils/meilisearch.ts +++ /dev/null @@ -1,155 +0,0 @@ -import { getLogger } from "@logtape/logtape"; -import { config } from "config-manager"; -import { count } from "drizzle-orm"; -import { Meilisearch } from "meilisearch"; -import { db } from "~/drizzle/db"; -import { Notes, Users } from "~/drizzle/schema"; -import type { User } from "~/packages/database-interface/user"; - -export const meilisearch = new Meilisearch({ - host: `${config.meilisearch.host}:${config.meilisearch.port}`, - apiKey: config.meilisearch.api_key, -}); - -export const connectMeili = async () => { - const logger = getLogger("meilisearch"); - if (!config.meilisearch.enabled) { - return; - } - - if (await meilisearch.isHealthy()) { - await meilisearch - .index(MeiliIndexType.Accounts) - .updateSortableAttributes(["createdAt"]); - - await meilisearch - .index(MeiliIndexType.Accounts) - .updateSearchableAttributes(["username", "displayName", "note"]); - - await meilisearch - .index(MeiliIndexType.Statuses) - .updateSortableAttributes(["createdAt"]); - - await meilisearch - .index(MeiliIndexType.Statuses) - .updateSearchableAttributes(["content"]); - - logger.info`Connected to Meilisearch`; - } else { - logger.fatal`Error while connecting to Meilisearch`; - // Hang until Ctrl+C is pressed - await Bun.sleep(Number.POSITIVE_INFINITY); - } -}; - -export enum MeiliIndexType { - Accounts = "accounts", - Statuses = "statuses", -} - -export const addUserToMeilisearch = async (user: User) => { - if (!config.meilisearch.enabled) { - return; - } - - await meilisearch.index(MeiliIndexType.Accounts).addDocuments([ - { - id: user.id, - username: user.data.username, - displayName: user.data.displayName, - note: user.data.note, - createdAt: user.data.createdAt, - }, - ]); -}; - -export const getNthDatabaseAccountBatch = ( - n: number, - batchSize = 1000, -): Promise[]> => { - return db.query.Users.findMany({ - offset: n * batchSize, - limit: batchSize, - columns: { - id: true, - username: true, - displayName: true, - note: true, - createdAt: true, - }, - orderBy: (user, { asc }) => asc(user.createdAt), - }); -}; - -export const getNthDatabaseStatusBatch = ( - n: number, - batchSize = 1000, -): Promise[]> => { - return db.query.Notes.findMany({ - offset: n * batchSize, - limit: batchSize, - columns: { - id: true, - content: true, - createdAt: true, - }, - orderBy: (status, { asc }) => asc(status.createdAt), - }); -}; - -export const rebuildSearchIndexes = async ( - indexes: MeiliIndexType[], - batchSize = 100, -) => { - if (indexes.includes(MeiliIndexType.Accounts)) { - const accountCount = ( - await db - .select({ - count: count(), - }) - .from(Users) - )[0].count; - - for (let i = 0; i < accountCount / batchSize; i++) { - const accounts = await getNthDatabaseAccountBatch(i, batchSize); - - /* const _progress = Math.round( - (i / (accountCount / batchSize)) * 100, - ); - */ - // Sync with Meilisearch - await meilisearch - .index(MeiliIndexType.Accounts) - .addDocuments(accounts); - } - - /* const _meiliAccountCount = ( - await meilisearch.index(MeiliIndexType.Accounts).getStats() - ).numberOfDocuments; */ - } - - if (indexes.includes(MeiliIndexType.Statuses)) { - const statusCount = ( - await db - .select({ - count: count(), - }) - .from(Notes) - )[0].count; - - for (let i = 0; i < statusCount / batchSize; i++) { - const statuses = await getNthDatabaseStatusBatch(i, batchSize); - - /* const _progress = Math.round((i / (statusCount / batchSize)) * 100); */ - - // Sync with Meilisearch - await meilisearch - .index(MeiliIndexType.Statuses) - .addDocuments(statuses); - } - - /* const _meiliStatusCount = ( - await meilisearch.index(MeiliIndexType.Statuses).getStats() - ).numberOfDocuments; */ - } -};