refactor: ♻️ Replace Meilisearch with Sonic

.github/config.workflow.toml

@@ -19,10 +19,10 @@ password = ""
 database = 1
 enabled = false
 
-[meilisearch]
+[sonic]
 host = "localhost"
 port = 40007
-api_key = ""
+password = ""
 enabled = false
 
 [signups]

CHANGELOG.md

@@ -7,6 +7,7 @@ Lysand Server `0.7.0` is backwards compatible with `0.6.0`. However, some new fe
 ## Features
 
 - Upgrade Bun to `1.1.17`. This brings performance upgrades and better stability.
+- Added support for the [Sonic](https://github.com/valeriansaliou/sonic) search indexer.
 - Note deletions are now federated.
 - Note edits are now federated.
 - Added option for more federation debug logging.
@@ -38,6 +39,7 @@ Lysand Server `0.7.0` is backwards compatible with `0.6.0`. However, some new fe
 ## Removals
 
 - Remove old logging system, to be replaced by a new one.
+- Removed Meilisearch support, in favor of Sonic. Follow instructions in the [installation guide](docs/installation.md) to set up Sonic.
 
 ## Miscellaneous
 

classes/search/search-manager.ts

@@ -0,0 +1,283 @@
+/**
+ * @file search-manager.ts
+ * @description Sonic search integration for indexing and searching accounts and statuses
+ */
+
+import { getLogger } from "@logtape/logtape";
+import {
+    Ingest as SonicChannelIngest,
+    Search as SonicChannelSearch,
+} from "sonic-channel";
+import { db } from "~/drizzle/db";
+import { type Config, config } from "~/packages/config-manager";
+import { Note } from "~/packages/database-interface/note";
+import { User } from "~/packages/database-interface/user";
+
+/**
+ * Enum for Sonic index types
+ */
+export enum SonicIndexType {
+    Accounts = "accounts",
+    Statuses = "statuses",
+}
+
+/**
+ * Class for managing Sonic search operations
+ */
+export class SonicSearchManager {
+    private searchChannel: SonicChannelSearch;
+    private ingestChannel: SonicChannelIngest;
+    private logger = getLogger("sonic");
+
+    /**
+     * @param config Configuration for Sonic
+     */
+    constructor(private config: Config) {
+        this.searchChannel = new SonicChannelSearch({
+            host: config.sonic.host,
+            port: config.sonic.port,
+            auth: config.sonic.password,
+        });
+
+        this.ingestChannel = new SonicChannelIngest({
+            host: config.sonic.host,
+            port: config.sonic.port,
+            auth: config.sonic.password,
+        });
+    }
+
+    /**
+     * Connect to Sonic
+     */
+    async connect(): Promise<void> {
+        if (!this.config.sonic.enabled) {
+            this.logger.info`Sonic search is disabled`;
+            return;
+        }
+
+        this.logger.info`Connecting to Sonic...`;
+
+        // Connect to Sonic
+        await new Promise<boolean>((resolve, reject) => {
+            this.searchChannel.connect({
+                connected: () => {
+                    this.logger.info`Connected to Sonic Search Channel`;
+                    resolve(true);
+                },
+                disconnected: () =>
+                    this.logger
+                        .error`Disconnected from Sonic Search Channel. You might be using an incorrect password.`,
+                timeout: () =>
+                    this.logger
+                        .error`Sonic Search Channel connection timed out`,
+                retrying: () =>
+                    this.logger
+                        .warn`Retrying connection to Sonic Search Channel`,
+                error: (error) => {
+                    this.logger
+                        .error`Failed to connect to Sonic Search Channel: ${error}`;
+                    reject(error);
+                },
+            });
+        });
+
+        await new Promise<boolean>((resolve, reject) => {
+            this.ingestChannel.connect({
+                connected: () => {
+                    this.logger.info`Connected to Sonic Ingest Channel`;
+                    resolve(true);
+                },
+                disconnected: () =>
+                    this.logger.error`Disconnected from Sonic Ingest Channel`,
+                timeout: () =>
+                    this.logger
+                        .error`Sonic Ingest Channel connection timed out`,
+                retrying: () =>
+                    this.logger
+                        .warn`Retrying connection to Sonic Ingest Channel`,
+                error: (error) => {
+                    this.logger
+                        .error`Failed to connect to Sonic Ingest Channel: ${error}`;
+                    reject(error);
+                },
+            });
+        });
+
+        try {
+            await Promise.all([
+                this.searchChannel.ping(),
+                this.ingestChannel.ping(),
+            ]);
+            this.logger.info`Connected to Sonic`;
+        } catch (error) {
+            this.logger.fatal`Error while connecting to Sonic: ${error}`;
+            throw error;
+        }
+    }
+
+    /**
+     * Add a user to Sonic
+     * @param user User to add
+     */
+    async addUser(user: User): Promise<void> {
+        if (!this.config.sonic.enabled) {
+            return;
+        }
+
+        try {
+            await this.ingestChannel.push(
+                SonicIndexType.Accounts,
+                "users",
+                user.id,
+                `${user.data.username} ${user.data.displayName} ${user.data.note}`,
+            );
+        } catch (error) {
+            this.logger.error`Failed to add user to Sonic: ${error}`;
+        }
+    }
+
+    /**
+     * Get a batch of accounts from the database
+     * @param n Batch number
+     * @param batchSize Size of the batch
+     */
+    private async getNthDatabaseAccountBatch(
+        n: number,
+        batchSize = 1000,
+    ): Promise<Record<string, string | Date>[]> {
+        return db.query.Users.findMany({
+            offset: n * batchSize,
+            limit: batchSize,
+            columns: {
+                id: true,
+                username: true,
+                displayName: true,
+                note: true,
+                createdAt: true,
+            },
+            orderBy: (user, { asc }) => asc(user.createdAt),
+        });
+    }
+
+    /**
+     * Get a batch of statuses from the database
+     * @param n Batch number
+     * @param batchSize Size of the batch
+     */
+    private async getNthDatabaseStatusBatch(
+        n: number,
+        batchSize = 1000,
+    ): Promise<Record<string, string | Date>[]> {
+        return db.query.Notes.findMany({
+            offset: n * batchSize,
+            limit: batchSize,
+            columns: {
+                id: true,
+                content: true,
+                createdAt: true,
+            },
+            orderBy: (status, { asc }) => asc(status.createdAt),
+        });
+    }
+
+    /**
+     * Rebuild search indexes
+     * @param indexes Indexes to rebuild
+     * @param batchSize Size of each batch
+     */
+    async rebuildSearchIndexes(
+        indexes: SonicIndexType[],
+        batchSize = 100,
+    ): Promise<void> {
+        for (const index of indexes) {
+            if (index === SonicIndexType.Accounts) {
+                await this.rebuildAccountsIndex(batchSize);
+            } else if (index === SonicIndexType.Statuses) {
+                await this.rebuildStatusesIndex(batchSize);
+            }
+        }
+    }
+
+    /**
+     * Rebuild accounts index
+     * @param batchSize Size of each batch
+     */
+    private async rebuildAccountsIndex(batchSize: number): Promise<void> {
+        const accountCount = await User.getCount();
+        const batchCount = Math.ceil(accountCount / batchSize);
+
+        for (let i = 0; i < batchCount; i++) {
+            const accounts = await this.getNthDatabaseAccountBatch(
+                i,
+                batchSize,
+            );
+            await Promise.all(
+                accounts.map((account) =>
+                    this.ingestChannel.push(
+                        SonicIndexType.Accounts,
+                        "users",
+                        account.id as string,
+                        `${account.username} ${account.displayName} ${account.note}`,
+                    ),
+                ),
+            );
+            this.logger.info`Indexed accounts batch ${i + 1}/${batchCount}`;
+        }
+    }
+
+    /**
+     * Rebuild statuses index
+     * @param batchSize Size of each batch
+     */
+    private async rebuildStatusesIndex(batchSize: number): Promise<void> {
+        const statusCount = await Note.getCount();
+        const batchCount = Math.ceil(statusCount / batchSize);
+
+        for (let i = 0; i < batchCount; i++) {
+            const statuses = await this.getNthDatabaseStatusBatch(i, batchSize);
+            await Promise.all(
+                statuses.map((status) =>
+                    this.ingestChannel.push(
+                        SonicIndexType.Statuses,
+                        "notes",
+                        status.id as string,
+                        status.content as string,
+                    ),
+                ),
+            );
+            this.logger.info`Indexed statuses batch ${i + 1}/${batchCount}`;
+        }
+    }
+
+    /**
+     * Search for accounts
+     * @param query Search query
+     * @param limit Maximum number of results
+     * @param offset Offset for pagination
+     */
+    searchAccounts(query: string, limit = 10, offset = 0): Promise<string[]> {
+        return this.searchChannel.query(
+            SonicIndexType.Accounts,
+            "users",
+            query,
+            { limit, offset },
+        );
+    }
+
+    /**
+     * Search for statuses
+     * @param query Search query
+     * @param limit Maximum number of results
+     * @param offset Offset for pagination
+     */
+    searchStatuses(query: string, limit = 10, offset = 0): Promise<string[]> {
+        return this.searchChannel.query(
+            SonicIndexType.Statuses,
+            "notes",
+            query,
+            { limit, offset },
+        );
+    }
+}
+
+export const searchManager = new SonicSearchManager(config);

config/config.example.toml

@@ -24,11 +24,11 @@ password = ""
 database = 1
 enabled = false
 
-[meilisearch]
+[sonic]
-# If Meilisearch is not configured, search will not be enabled
+# If Sonic is not configured, search will not be enabled
 host = "localhost"
 port = 40007
-api_key = ""
+password = ""
 enabled = true
 
 [signups]

docker-compose.yml

@@ -14,7 +14,7 @@ services:
         depends_on:
             - db
             - redis
-            - meilisearch
+            - sonic
             - fe
     
     fe:
@@ -48,18 +48,11 @@ services:
         networks:
             - lysand-net
 
-    meilisearch:
+    sonic:
-        stdin_open: true
-        environment:
-            - MEILI_MASTER_KEY=__________________
-        tty: true
-        networks:
-            - lysand-net
         volumes:
-            - ./meili-data:/meili_data
+          - ./config.cfg:/etc/sonic.cfg
-        image: getmeili/meilisearch:v1.7
+          - ./store/:/var/lib/sonic/store/
-        container_name: lysand-meilisearch
+        image: valeriansaliou/sonic:v1.4.9
-        restart: unless-stopped
 
 networks:
     lysand-net:

docs/installation.md

@@ -6,7 +6,7 @@
   - Lysand will work on lower versions than 1.1.17, but only the latest version is supported
 - A PostgreSQL database
 - (Optional but recommended) A Linux-based operating system
-- (Optional if you want search) A working Meilisearch instance
+- (Optional if you want search) A working [Sonic](https://github.com/valeriansaliou/sonic) instance
 
 > [!WARNING]
 > Lysand has not been tested on Windows or macOS. It is recommended to use a Linux-based operating system to run Lysand.
@@ -59,7 +59,7 @@ bun install
 1. Set up a PostgreSQL database (you need a special extension, please look at [the database documentation](database.md))
 
 2. (If you want search)
-Create a Meilisearch instance (using Docker is recommended). For a [`docker-compose`] file, copy the `meilisearch` service from the [`docker-compose.yml`](../docker-compose.yml) file.
+Create a [Sonic](https://github.com/valeriansaliou/sonic) instance (using Docker is recommended). For a [`docker-compose`] file, copy the `sonic` service from the [`docker-compose.yml`](../docker-compose.yml) file. Don't forget to fill in the `config.cfg` for Sonic!
 
 1. Build everything:
 

package.json

@@ -130,13 +130,13 @@
         "markdown-it-anchor": "^9.0.1",
         "markdown-it-container": "^4.0.0",
         "markdown-it-toc-done-right": "^4.2.0",
-        "meilisearch": "^0.40.0",
         "mime-types": "^2.1.35",
         "oauth4webapi": "^2.11.1",
         "ora": "^8.0.1",
         "pg": "^8.12.0",
         "qs": "^6.12.1",
         "sharp": "^0.33.4",
+        "sonic-channel": "^1.3.1",
         "string-comparison": "^1.3.0",
         "stringify-entities": "^4.0.4",
         "strip-ansi": "^7.1.0",

packages/config-manager/config.type.ts

@@ -76,7 +76,7 @@ export const configValidator = z.object({
                 enabled: false,
             }),
     }),
-    meilisearch: z.object({
+    sonic: z.object({
         host: z.string().min(1).default("localhost"),
         port: z
             .number()
@@ -84,7 +84,7 @@ export const configValidator = z.object({
             .min(1)
             .max(2 ** 16 - 1)
             .default(7700),
-        api_key: z.string(),
+        password: z.string(),
         enabled: z.boolean().default(false),
     }),
     signups: z.object({

packages/database-interface/user.ts

@@ -1,7 +1,6 @@
 import { idValidator } from "@/api";
 import { getBestContentType, urlToContentFormat } from "@/content_types";
 import { randomString } from "@/math";
-import { addUserToMeilisearch } from "@/meilisearch";
 import { proxyUrl } from "@/response";
 import type {
     Account as ApiAccount,
@@ -31,6 +30,7 @@ import {
     type UserWithRelations,
     findManyUsers,
 } from "~/classes/functions/user";
+import { searchManager } from "~/classes/search/search-manager";
 import { db } from "~/drizzle/db";
 import {
     EmojiToUser,
@@ -294,8 +294,8 @@ export class User extends BaseInterface<typeof Users, UserWithRelations> {
             throw new Error("Failed to save user from remote");
         }
 
-        // Add to Meilisearch
+        // Add to search index
-        await addUserToMeilisearch(finalUser);
+        await searchManager.addUser(finalUser);
 
         return finalUser;
     }
@@ -477,8 +477,8 @@ export class User extends BaseInterface<typeof Users, UserWithRelations> {
             throw new Error("Failed to create user");
         }
 
-        // Add to Meilisearch
+        // Add to search index
-        await addUserToMeilisearch(finalUser);
+        await searchManager.addUser(finalUser);
 
         return finalUser;
     }

server/api/api/v2/search/index.ts

@@ -1,5 +1,4 @@
 import { applyConfig, auth, handleZodError, userAddressValidator } from "@/api";
-import { MeiliIndexType, meilisearch } from "@/meilisearch";
 import { errorResponse, jsonResponse } from "@/response";
 import { zValidator } from "@hono/zod-validator";
 import { getLogger } from "@logtape/logtape";
@@ -7,6 +6,7 @@ import { and, eq, inArray, sql } from "drizzle-orm";
 import type { Hono } from "hono";
 import { z } from "zod";
 import { resolveWebFinger } from "~/classes/functions/user";
+import { searchManager } from "~/classes/search/search-manager";
 import { db } from "~/drizzle/db";
 import { Instances, Notes, RolePermissions, Users } from "~/drizzle/schema";
 import { config } from "~/packages/config-manager";
@@ -65,12 +65,19 @@ export default (app: Hono) =>
                 );
             }
 
-            if (!config.meilisearch.enabled) {
+            if (!q) {
-                return errorResponse("Meilisearch is not enabled", 501);
+                return errorResponse("Query is required", 400);
             }
 
-            let accountResults: { id: string }[] = [];
+            if (!config.sonic.enabled) {
-            let statusResults: { id: string }[] = [];
+                return errorResponse(
+                    "Search is not enabled by your server administrator",
+                    501,
+                );
+            }
+
+            let accountResults: string[] = [];
+            let statusResults: string[] = [];
 
             if (!type || type === "accounts") {
                 // Check if q is matching format username@domain.com or @username@domain.com
@@ -132,34 +139,26 @@ export default (app: Hono) =>
                     }
                 }
 
-                accountResults = (
+                accountResults = await searchManager.searchAccounts(
-                    await meilisearch.index(MeiliIndexType.Accounts).search<{
+                    q,
-                        id: string;
+                    Number(limit) || 10,
-                    }>(q, {
+                    Number(offset) || 0,
-                        limit: Number(limit) || 10,
+                );
-                        offset: Number(offset) || 0,
-                        sort: ["createdAt:desc"],
-                    })
-                ).hits;
             }
 
             if (!type || type === "statuses") {
-                statusResults = (
+                statusResults = await searchManager.searchStatuses(
-                    await meilisearch.index(MeiliIndexType.Statuses).search<{
+                    q,
-                        id: string;
+                    Number(limit) || 10,
-                    }>(q, {
+                    Number(offset) || 0,
-                        limit: Number(limit) || 10,
+                );
-                        offset: Number(offset) || 0,
-                        sort: ["createdAt:desc"],
-                    })
-                ).hits;
             }
 
             const accounts = await User.manyFromSql(
                 and(
                     inArray(
                         Users.id,
-                        accountResults.map((hit) => hit.id),
+                        accountResults.map((hit) => hit),
                     ),
                     self
                         ? sql`EXISTS (SELECT 1 FROM Relationships WHERE Relationships.subjectId = ${
@@ -175,7 +174,7 @@ export default (app: Hono) =>
                 and(
                     inArray(
                         Notes.id,
-                        statusResults.map((hit) => hit.id),
+                        statusResults.map((hit) => hit),
                     ),
                     account_id ? eq(Notes.authorId, account_id) : undefined,
                     self

setup.ts

@@ -1,10 +1,10 @@
 import { checkConfig } from "@/init";
 import { configureLoggers } from "@/loggers";
-import { connectMeili } from "@/meilisearch";
 import { getLogger } from "@logtape/logtape";
 import { config } from "config-manager";
 import { setupDatabase } from "~/drizzle/db";
 import { Note } from "~/packages/database-interface/note";
+import { searchManager } from "./classes/search/search-manager";
 
 const timeAtStart = performance.now();
 
@@ -16,8 +16,8 @@ serverLogger.info`Starting Lysand...`;
 
 await setupDatabase();
 
-if (config.meilisearch.enabled) {
+if (config.sonic.enabled) {
-    await connectMeili();
+    await searchManager.connect();
 }
 
 process.on("SIGINT", () => {

tests/utils.ts

@@ -4,13 +4,18 @@ import { solveChallenge } from "altcha-lib";
 import { asc, inArray, like } from "drizzle-orm";
 import { appFactory } from "~/app";
 import type { Status } from "~/classes/functions/status";
+import { searchManager } from "~/classes/search/search-manager";
 import { db } from "~/drizzle/db";
 import { setupDatabase } from "~/drizzle/db";
 import { Notes, Tokens, Users } from "~/drizzle/schema";
+import { config } from "~/packages/config-manager";
 import { Note } from "~/packages/database-interface/note";
 import { User } from "~/packages/database-interface/user";
 
 await setupDatabase();
+if (config.sonic.enabled) {
+    await searchManager.connect();
+}
 
 /**
  * This allows us to send a test request to the server even when it isnt running

utils/loggers.ts

@@ -221,7 +221,7 @@ export const configureLoggers = (silent = false) =>
                 filters: ["configFilter"],
             },
             {
-                category: "meilisearch",
+                category: "sonic",
                 sinks: ["console", "file"],
                 filters: ["configFilter"],
             },

utils/meilisearch.ts

@@ -1,155 +0,0 @@
-import { getLogger } from "@logtape/logtape";
-import { config } from "config-manager";
-import { count } from "drizzle-orm";
-import { Meilisearch } from "meilisearch";
-import { db } from "~/drizzle/db";
-import { Notes, Users } from "~/drizzle/schema";
-import type { User } from "~/packages/database-interface/user";
-
-export const meilisearch = new Meilisearch({
-    host: `${config.meilisearch.host}:${config.meilisearch.port}`,
-    apiKey: config.meilisearch.api_key,
-});
-
-export const connectMeili = async () => {
-    const logger = getLogger("meilisearch");
-    if (!config.meilisearch.enabled) {
-        return;
-    }
-
-    if (await meilisearch.isHealthy()) {
-        await meilisearch
-            .index(MeiliIndexType.Accounts)
-            .updateSortableAttributes(["createdAt"]);
-
-        await meilisearch
-            .index(MeiliIndexType.Accounts)
-            .updateSearchableAttributes(["username", "displayName", "note"]);
-
-        await meilisearch
-            .index(MeiliIndexType.Statuses)
-            .updateSortableAttributes(["createdAt"]);
-
-        await meilisearch
-            .index(MeiliIndexType.Statuses)
-            .updateSearchableAttributes(["content"]);
-
-        logger.info`Connected to Meilisearch`;
-    } else {
-        logger.fatal`Error while connecting to Meilisearch`;
-        // Hang until Ctrl+C is pressed
-        await Bun.sleep(Number.POSITIVE_INFINITY);
-    }
-};
-
-export enum MeiliIndexType {
-    Accounts = "accounts",
-    Statuses = "statuses",
-}
-
-export const addUserToMeilisearch = async (user: User) => {
-    if (!config.meilisearch.enabled) {
-        return;
-    }
-
-    await meilisearch.index(MeiliIndexType.Accounts).addDocuments([
-        {
-            id: user.id,
-            username: user.data.username,
-            displayName: user.data.displayName,
-            note: user.data.note,
-            createdAt: user.data.createdAt,
-        },
-    ]);
-};
-
-export const getNthDatabaseAccountBatch = (
-    n: number,
-    batchSize = 1000,
-): Promise<Record<string, string | Date>[]> => {
-    return db.query.Users.findMany({
-        offset: n * batchSize,
-        limit: batchSize,
-        columns: {
-            id: true,
-            username: true,
-            displayName: true,
-            note: true,
-            createdAt: true,
-        },
-        orderBy: (user, { asc }) => asc(user.createdAt),
-    });
-};
-
-export const getNthDatabaseStatusBatch = (
-    n: number,
-    batchSize = 1000,
-): Promise<Record<string, string | Date>[]> => {
-    return db.query.Notes.findMany({
-        offset: n * batchSize,
-        limit: batchSize,
-        columns: {
-            id: true,
-            content: true,
-            createdAt: true,
-        },
-        orderBy: (status, { asc }) => asc(status.createdAt),
-    });
-};
-
-export const rebuildSearchIndexes = async (
-    indexes: MeiliIndexType[],
-    batchSize = 100,
-) => {
-    if (indexes.includes(MeiliIndexType.Accounts)) {
-        const accountCount = (
-            await db
-                .select({
-                    count: count(),
-                })
-                .from(Users)
-        )[0].count;
-
-        for (let i = 0; i < accountCount / batchSize; i++) {
-            const accounts = await getNthDatabaseAccountBatch(i, batchSize);
-
-            /* const _progress = Math.round(
-                (i / (accountCount / batchSize)) * 100,
-            );
- */
-            // Sync with Meilisearch
-            await meilisearch
-                .index(MeiliIndexType.Accounts)
-                .addDocuments(accounts);
-        }
-
-        /* const _meiliAccountCount = (
-            await meilisearch.index(MeiliIndexType.Accounts).getStats()
-        ).numberOfDocuments; */
-    }
-
-    if (indexes.includes(MeiliIndexType.Statuses)) {
-        const statusCount = (
-            await db
-                .select({
-                    count: count(),
-                })
-                .from(Notes)
-        )[0].count;
-
-        for (let i = 0; i < statusCount / batchSize; i++) {
-            const statuses = await getNthDatabaseStatusBatch(i, batchSize);
-
-            /* const _progress = Math.round((i / (statusCount / batchSize)) * 100); */
-
-            // Sync with Meilisearch
-            await meilisearch
-                .index(MeiliIndexType.Statuses)
-                .addDocuments(statuses);
-        }
-
-        /* const _meiliStatusCount = (
-            await meilisearch.index(MeiliIndexType.Statuses).getStats()
-        ).numberOfDocuments; */
-    }
-};