versia-pub/server
2
0
Fork 0
mirror of https://github.com/versia-pub/server.git synced 2025-12-06 08:28:19 +01:00
Code Issues Actions Packages Projects Releases Wiki Activity

refactor: ♻️ Replace Meilisearch with Sonic

Browse source
This commit is contained in:
Jesse Wierzbinski 2024-06-28 23:40:44 -10:00
parent 2cf1537a7e
commit 19c15f7e96
No known key found for this signature in database
15 changed files with 338 additions and 211 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
.github/config.workflow.toml vendored
View file

@ -19,10 +19,10 @@ password = ""
database = 1
enabled = false
[meilisearch]
[sonic]
host = "localhost"
port = 40007
api_key = ""
password = ""
enabled = false
[signups]

2
CHANGELOG.md
View file

@ -7,6 +7,7 @@ Lysand Server `0.7.0` is backwards compatible with `0.6.0`. However, some new fe
## Features
- Upgrade Bun to `1.1.17`. This brings performance upgrades and better stability.
- Added support for the [Sonic](https://github.com/valeriansaliou/sonic) search indexer.
- Note deletions are now federated.
- Note edits are now federated.
- Added option for more federation debug logging.
@ -38,6 +39,7 @@ Lysand Server `0.7.0` is backwards compatible with `0.6.0`. However, some new fe
## Removals
- Remove old logging system, to be replaced by a new one.
- Removed Meilisearch support, in favor of Sonic. Follow instructions in the [installation guide](docs/installation.md) to set up Sonic.
## Miscellaneous

BIN
bun.lockb
View file

Binary file not shown.

283
classes/search/search-manager.ts Normal file
View file

@ -0,0 +1,283 @@
/**
* @file search-manager.ts
* @description Sonic search integration for indexing and searching accounts and statuses
*/
import { getLogger } from "@logtape/logtape";
import {
Ingest as SonicChannelIngest,
Search as SonicChannelSearch,
} from "sonic-channel";
import { db } from "~/drizzle/db";
import { type Config, config } from "~/packages/config-manager";
import { Note } from "~/packages/database-interface/note";
import { User } from "~/packages/database-interface/user";
/**
* Enum for Sonic index types
*/
export enum SonicIndexType {
Accounts = "accounts",
Statuses = "statuses",
}
/**
* Class for managing Sonic search operations
*/
export class SonicSearchManager {
private searchChannel: SonicChannelSearch;
private ingestChannel: SonicChannelIngest;
private logger = getLogger("sonic");
/**
* @param config Configuration for Sonic
*/
constructor(private config: Config) {
this.searchChannel = new SonicChannelSearch({
host: config.sonic.host,
port: config.sonic.port,
auth: config.sonic.password,
});
this.ingestChannel = new SonicChannelIngest({
host: config.sonic.host,
port: config.sonic.port,
auth: config.sonic.password,
});
}
/**
* Connect to Sonic
*/
async connect(): Promise<void> {
if (!this.config.sonic.enabled) {
this.logger.info`Sonic search is disabled`;
return;
}
this.logger.info`Connecting to Sonic...`;
// Connect to Sonic
await new Promise<boolean>((resolve, reject) => {
this.searchChannel.connect({
connected: () => {
this.logger.info`Connected to Sonic Search Channel`;
resolve(true);
},
disconnected: () =>
this.logger
.error`Disconnected from Sonic Search Channel. You might be using an incorrect password.`,
timeout: () =>
this.logger
.error`Sonic Search Channel connection timed out`,
retrying: () =>
this.logger
.warn`Retrying connection to Sonic Search Channel`,
error: (error) => {
this.logger
.error`Failed to connect to Sonic Search Channel: ${error}`;
reject(error);
},
});
});
await new Promise<boolean>((resolve, reject) => {
this.ingestChannel.connect({
connected: () => {
this.logger.info`Connected to Sonic Ingest Channel`;
resolve(true);
},
disconnected: () =>
this.logger.error`Disconnected from Sonic Ingest Channel`,
timeout: () =>
this.logger
.error`Sonic Ingest Channel connection timed out`,
retrying: () =>
this.logger
.warn`Retrying connection to Sonic Ingest Channel`,
error: (error) => {
this.logger
.error`Failed to connect to Sonic Ingest Channel: ${error}`;
reject(error);
},
});
});
try {
await Promise.all([
this.searchChannel.ping(),
this.ingestChannel.ping(),
]);
this.logger.info`Connected to Sonic`;
} catch (error) {
this.logger.fatal`Error while connecting to Sonic: ${error}`;
throw error;
}
}
/**
* Add a user to Sonic
* @param user User to add
*/
async addUser(user: User): Promise<void> {
if (!this.config.sonic.enabled) {
return;
}
try {
await this.ingestChannel.push(
SonicIndexType.Accounts,
"users",
user.id,
`${user.data.username} ${user.data.displayName} ${user.data.note}`,
);
} catch (error) {
this.logger.error`Failed to add user to Sonic: ${error}`;
}
}
/**
* Get a batch of accounts from the database
* @param n Batch number
* @param batchSize Size of the batch
*/
private async getNthDatabaseAccountBatch(
n: number,
batchSize = 1000,
): Promise<Record<string, string | Date>[]> {
return db.query.Users.findMany({
offset: n * batchSize,
limit: batchSize,
columns: {
id: true,
username: true,
displayName: true,
note: true,
createdAt: true,
},
orderBy: (user, { asc }) => asc(user.createdAt),
});
}
/**
* Get a batch of statuses from the database
* @param n Batch number
* @param batchSize Size of the batch
*/
private async getNthDatabaseStatusBatch(
n: number,
batchSize = 1000,
): Promise<Record<string, string | Date>[]> {
return db.query.Notes.findMany({
offset: n * batchSize,
limit: batchSize,
columns: {
id: true,
content: true,
createdAt: true,
},
orderBy: (status, { asc }) => asc(status.createdAt),
});
}
/**
* Rebuild search indexes
* @param indexes Indexes to rebuild
* @param batchSize Size of each batch
*/
async rebuildSearchIndexes(
indexes: SonicIndexType[],
batchSize = 100,
): Promise<void> {
for (const index of indexes) {
if (index === SonicIndexType.Accounts) {
await this.rebuildAccountsIndex(batchSize);
} else if (index === SonicIndexType.Statuses) {
await this.rebuildStatusesIndex(batchSize);
}
}
}
/**
* Rebuild accounts index
* @param batchSize Size of each batch
*/
private async rebuildAccountsIndex(batchSize: number): Promise<void> {
const accountCount = await User.getCount();
const batchCount = Math.ceil(accountCount / batchSize);
for (let i = 0; i < batchCount; i++) {
const accounts = await this.getNthDatabaseAccountBatch(
i,
batchSize,
);
await Promise.all(
accounts.map((account) =>
this.ingestChannel.push(
SonicIndexType.Accounts,
"users",
account.id as string,
`${account.username} ${account.displayName} ${account.note}`,
),
),
);
this.logger.info`Indexed accounts batch ${i + 1}/${batchCount}`;
}
}
/**
* Rebuild statuses index
* @param batchSize Size of each batch
*/
private async rebuildStatusesIndex(batchSize: number): Promise<void> {
const statusCount = await Note.getCount();
const batchCount = Math.ceil(statusCount / batchSize);
for (let i = 0; i < batchCount; i++) {
const statuses = await this.getNthDatabaseStatusBatch(i, batchSize);
await Promise.all(
statuses.map((status) =>
this.ingestChannel.push(
SonicIndexType.Statuses,
"notes",
status.id as string,
status.content as string,
),
),
);
this.logger.info`Indexed statuses batch ${i + 1}/${batchCount}`;
}
}
/**
* Search for accounts
* @param query Search query
* @param limit Maximum number of results
* @param offset Offset for pagination
*/
searchAccounts(query: string, limit = 10, offset = 0): Promise<string[]> {
return this.searchChannel.query(
SonicIndexType.Accounts,
"users",
query,
{ limit, offset },
);
}
/**
* Search for statuses
* @param query Search query
* @param limit Maximum number of results
* @param offset Offset for pagination
*/
searchStatuses(query: string, limit = 10, offset = 0): Promise<string[]> {
return this.searchChannel.query(
SonicIndexType.Statuses,
"notes",
query,
{ limit, offset },
);
}
}
export const searchManager = new SonicSearchManager(config);

6
config/config.example.toml
View file

@ -24,11 +24,11 @@ password = ""
database = 1
enabled = false
[meilisearch]
# If Meilisearch is not configured, search will not be enabled
[sonic]
# If Sonic is not configured, search will not be enabled
host = "localhost"
port = 40007
api_key = ""
password = ""
enabled = true
[signups]

17
docker-compose.yml
View file

@ -14,7 +14,7 @@ services:
depends_on:
- db
- redis
- meilisearch
- sonic
- fe
fe:
@ -48,18 +48,11 @@ services:
networks:
- lysand-net
meilisearch:
stdin_open: true
environment:
- MEILI_MASTER_KEY=__________________
tty: true
networks:
- lysand-net
sonic:
volumes:
- ./meili-data:/meili_data
image: getmeili/meilisearch:v1.7
container_name: lysand-meilisearch
restart: unless-stopped
- ./config.cfg:/etc/sonic.cfg
- ./store/:/var/lib/sonic/store/
image: valeriansaliou/sonic:v1.4.9
networks:
lysand-net:

4
docs/installation.md
View file

@ -6,7 +6,7 @@
- Lysand will work on lower versions than 1.1.17, but only the latest version is supported
- A PostgreSQL database
- (Optional but recommended) A Linux-based operating system
- (Optional if you want search) A working Meilisearch instance
- (Optional if you want search) A working [Sonic](https://github.com/valeriansaliou/sonic) instance
> [!WARNING]
> Lysand has not been tested on Windows or macOS. It is recommended to use a Linux-based operating system to run Lysand.
@ -59,7 +59,7 @@ bun install
1. Set up a PostgreSQL database (you need a special extension, please look at [the database documentation](database.md))
2. (If you want search)
Create a Meilisearch instance (using Docker is recommended). For a [`docker-compose`] file, copy the `meilisearch` service from the [`docker-compose.yml`](../docker-compose.yml) file.
Create a [Sonic](https://github.com/valeriansaliou/sonic) instance (using Docker is recommended). For a [`docker-compose`] file, copy the `sonic` service from the [`docker-compose.yml`](../docker-compose.yml) file. Don't forget to fill in the `config.cfg` for Sonic!
1. Build everything:

2
package.json
View file

@ -130,13 +130,13 @@
"markdown-it-anchor": "^9.0.1",
"markdown-it-container": "^4.0.0",
"markdown-it-toc-done-right": "^4.2.0",
"meilisearch": "^0.40.0",
"mime-types": "^2.1.35",
"oauth4webapi": "^2.11.1",
"ora": "^8.0.1",
"pg": "^8.12.0",
"qs": "^6.12.1",
"sharp": "^0.33.4",
"sonic-channel": "^1.3.1",
"string-comparison": "^1.3.0",
"stringify-entities": "^4.0.4",
"strip-ansi": "^7.1.0",

4
packages/config-manager/config.type.ts
View file

@ -76,7 +76,7 @@ export const configValidator = z.object({
enabled: false,
}),
}),
meilisearch: z.object({
sonic: z.object({
host: z.string().min(1).default("localhost"),
port: z
.number()
@ -84,7 +84,7 @@ export const configValidator = z.object({
.min(1)
.max(2 ** 16 - 1)
.default(7700),
api_key: z.string(),
password: z.string(),
enabled: z.boolean().default(false),
}),
signups: z.object({

10
packages/database-interface/user.ts
View file

@ -1,7 +1,6 @@
import { idValidator } from "@/api";
import { getBestContentType, urlToContentFormat } from "@/content_types";
import { randomString } from "@/math";
import { addUserToMeilisearch } from "@/meilisearch";
import { proxyUrl } from "@/response";
import type {
Account as ApiAccount,
@ -31,6 +30,7 @@ import {
type UserWithRelations,
findManyUsers,
} from "~/classes/functions/user";
import { searchManager } from "~/classes/search/search-manager";
import { db } from "~/drizzle/db";
import {
EmojiToUser,
@ -294,8 +294,8 @@ export class User extends BaseInterface<typeof Users, UserWithRelations> {
throw new Error("Failed to save user from remote");
}
// Add to Meilisearch
await addUserToMeilisearch(finalUser);
// Add to search index
await searchManager.addUser(finalUser);
return finalUser;
}
@ -477,8 +477,8 @@ export class User extends BaseInterface<typeof Users, UserWithRelations> {
throw new Error("Failed to create user");
}
// Add to Meilisearch
await addUserToMeilisearch(finalUser);
// Add to search index
await searchManager.addUser(finalUser);
return finalUser;
}

49
server/api/api/v2/search/index.ts
View file

@ -1,5 +1,4 @@
import { applyConfig, auth, handleZodError, userAddressValidator } from "@/api";
import { MeiliIndexType, meilisearch } from "@/meilisearch";
import { errorResponse, jsonResponse } from "@/response";
import { zValidator } from "@hono/zod-validator";
import { getLogger } from "@logtape/logtape";
@ -7,6 +6,7 @@ import { and, eq, inArray, sql } from "drizzle-orm";
import type { Hono } from "hono";
import { z } from "zod";
import { resolveWebFinger } from "~/classes/functions/user";
import { searchManager } from "~/classes/search/search-manager";
import { db } from "~/drizzle/db";
import { Instances, Notes, RolePermissions, Users } from "~/drizzle/schema";
import { config } from "~/packages/config-manager";
@ -65,12 +65,19 @@ export default (app: Hono) =>
);
}
if (!config.meilisearch.enabled) {
return errorResponse("Meilisearch is not enabled", 501);
if (!q) {
return errorResponse("Query is required", 400);
}
let accountResults: { id: string }[] = [];
let statusResults: { id: string }[] = [];
if (!config.sonic.enabled) {
return errorResponse(
"Search is not enabled by your server administrator",
501,
);
}
let accountResults: string[] = [];
let statusResults: string[] = [];
if (!type || type === "accounts") {
// Check if q is matching format username@domain.com or @username@domain.com
@ -132,34 +139,26 @@ export default (app: Hono) =>
}
}
accountResults = (
await meilisearch.index(MeiliIndexType.Accounts).search<{
id: string;
}>(q, {
limit: Number(limit) || 10,
offset: Number(offset) || 0,
sort: ["createdAt:desc"],
})
).hits;
accountResults = await searchManager.searchAccounts(
q,
Number(limit) || 10,
Number(offset) || 0,
);
}
if (!type || type === "statuses") {
statusResults = (
await meilisearch.index(MeiliIndexType.Statuses).search<{
id: string;
}>(q, {
limit: Number(limit) || 10,
offset: Number(offset) || 0,
sort: ["createdAt:desc"],
})
).hits;
statusResults = await searchManager.searchStatuses(
q,
Number(limit) || 10,
Number(offset) || 0,
);
}
const accounts = await User.manyFromSql(
and(
inArray(
Users.id,
accountResults.map((hit) => hit.id),
accountResults.map((hit) => hit),
),
self
? sql`EXISTS (SELECT 1 FROM Relationships WHERE Relationships.subjectId = ${
@ -175,7 +174,7 @@ export default (app: Hono) =>
and(
inArray(
Notes.id,
statusResults.map((hit) => hit.id),
statusResults.map((hit) => hit),
),
account_id ? eq(Notes.authorId, account_id) : undefined,
self

6
setup.ts
View file

@ -1,10 +1,10 @@
import { checkConfig } from "@/init";
import { configureLoggers } from "@/loggers";
import { connectMeili } from "@/meilisearch";
import { getLogger } from "@logtape/logtape";
import { config } from "config-manager";
import { setupDatabase } from "~/drizzle/db";
import { Note } from "~/packages/database-interface/note";
import { searchManager } from "./classes/search/search-manager";
const timeAtStart = performance.now();
@ -16,8 +16,8 @@ serverLogger.info`Starting Lysand...`;
await setupDatabase();
if (config.meilisearch.enabled) {
await connectMeili();
if (config.sonic.enabled) {
await searchManager.connect();
}
process.on("SIGINT", () => {

5
tests/utils.ts
View file

@ -4,13 +4,18 @@ import { solveChallenge } from "altcha-lib";
import { asc, inArray, like } from "drizzle-orm";
import { appFactory } from "~/app";
import type { Status } from "~/classes/functions/status";
import { searchManager } from "~/classes/search/search-manager";
import { db } from "~/drizzle/db";
import { setupDatabase } from "~/drizzle/db";
import { Notes, Tokens, Users } from "~/drizzle/schema";
import { config } from "~/packages/config-manager";
import { Note } from "~/packages/database-interface/note";
import { User } from "~/packages/database-interface/user";
await setupDatabase();
if (config.sonic.enabled) {
await searchManager.connect();
}
/**
* This allows us to send a test request to the server even when it isnt running

2
utils/loggers.ts
View file

@ -221,7 +221,7 @@ export const configureLoggers = (silent = false) =>
filters: ["configFilter"],
},
{
category: "meilisearch",
category: "sonic",
sinks: ["console", "file"],
filters: ["configFilter"],
},

155
utils/meilisearch.ts
View file

@ -1,155 +0,0 @@
import { getLogger } from "@logtape/logtape";
import { config } from "config-manager";
import { count } from "drizzle-orm";
import { Meilisearch } from "meilisearch";
import { db } from "~/drizzle/db";
import { Notes, Users } from "~/drizzle/schema";
import type { User } from "~/packages/database-interface/user";
export const meilisearch = new Meilisearch({
host: `${config.meilisearch.host}:${config.meilisearch.port}`,
apiKey: config.meilisearch.api_key,
});
export const connectMeili = async () => {
const logger = getLogger("meilisearch");
if (!config.meilisearch.enabled) {
return;
}
if (await meilisearch.isHealthy()) {
await meilisearch
.index(MeiliIndexType.Accounts)
.updateSortableAttributes(["createdAt"]);
await meilisearch
.index(MeiliIndexType.Accounts)
.updateSearchableAttributes(["username", "displayName", "note"]);
await meilisearch
.index(MeiliIndexType.Statuses)
.updateSortableAttributes(["createdAt"]);
await meilisearch
.index(MeiliIndexType.Statuses)
.updateSearchableAttributes(["content"]);
logger.info`Connected to Meilisearch`;
} else {
logger.fatal`Error while connecting to Meilisearch`;
// Hang until Ctrl+C is pressed
await Bun.sleep(Number.POSITIVE_INFINITY);
}
};
export enum MeiliIndexType {
Accounts = "accounts",
Statuses = "statuses",
}
export const addUserToMeilisearch = async (user: User) => {
if (!config.meilisearch.enabled) {
return;
}
await meilisearch.index(MeiliIndexType.Accounts).addDocuments([
{
id: user.id,
username: user.data.username,
displayName: user.data.displayName,
note: user.data.note,
createdAt: user.data.createdAt,
},
]);
};
export const getNthDatabaseAccountBatch = (
n: number,
batchSize = 1000,
): Promise<Record<string, string | Date>[]> => {
return db.query.Users.findMany({
offset: n * batchSize,
limit: batchSize,
columns: {
id: true,
username: true,
displayName: true,
note: true,
createdAt: true,
},
orderBy: (user, { asc }) => asc(user.createdAt),
});
};
export const getNthDatabaseStatusBatch = (
n: number,
batchSize = 1000,
): Promise<Record<string, string | Date>[]> => {
return db.query.Notes.findMany({
offset: n * batchSize,
limit: batchSize,
columns: {
id: true,
content: true,
createdAt: true,
},
orderBy: (status, { asc }) => asc(status.createdAt),
});
};
export const rebuildSearchIndexes = async (
indexes: MeiliIndexType[],
batchSize = 100,
) => {
if (indexes.includes(MeiliIndexType.Accounts)) {
const accountCount = (
await db
.select({
count: count(),
})
.from(Users)
)[0].count;
for (let i = 0; i < accountCount / batchSize; i++) {
const accounts = await getNthDatabaseAccountBatch(i, batchSize);
/* const _progress = Math.round(
(i / (accountCount / batchSize)) * 100,
);
*/
// Sync with Meilisearch
await meilisearch
.index(MeiliIndexType.Accounts)
.addDocuments(accounts);
}
/* const _meiliAccountCount = (
await meilisearch.index(MeiliIndexType.Accounts).getStats()
).numberOfDocuments; */
}
if (indexes.includes(MeiliIndexType.Statuses)) {
const statusCount = (
await db
.select({
count: count(),
})
.from(Notes)
)[0].count;
for (let i = 0; i < statusCount / batchSize; i++) {
const statuses = await getNthDatabaseStatusBatch(i, batchSize);
/* const _progress = Math.round((i / (statusCount / batchSize)) * 100); */
// Sync with Meilisearch
await meilisearch
.index(MeiliIndexType.Statuses)
.addDocuments(statuses);
}
/* const _meiliStatusCount = (
await meilisearch.index(MeiliIndexType.Statuses).getStats()
).numberOfDocuments; */
}
};