versia-pub/server
2
0
Fork 0
mirror of https://github.com/versia-pub/server.git synced 2026-03-13 05:49:16 +01:00
Code Issues Projects Releases Packages Wiki Activity Actions

refactor: ♻️ Replace Meilisearch with Sonic

Browse source
This commit is contained in:
Jesse Wierzbinski 2024-06-28 23:40:44 -10:00
parent 2cf1537a7e
commit 19c15f7e96
No known key found for this signature in database
15 changed files with 338 additions and 211 deletions
Download patch file Download diff file Expand all files Collapse all files

2
utils/loggers.ts
View file

@ -221,7 +221,7 @@ export const configureLoggers = (silent = false) =>
filters: ["configFilter"],
},
{
category: "meilisearch",
category: "sonic",
sinks: ["console", "file"],
filters: ["configFilter"],
},

155
utils/meilisearch.ts
View file

@ -1,155 +0,0 @@
import { getLogger } from "@logtape/logtape";
import { config } from "config-manager";
import { count } from "drizzle-orm";
import { Meilisearch } from "meilisearch";
import { db } from "~/drizzle/db";
import { Notes, Users } from "~/drizzle/schema";
import type { User } from "~/packages/database-interface/user";
export const meilisearch = new Meilisearch({
host: `${config.meilisearch.host}:${config.meilisearch.port}`,
apiKey: config.meilisearch.api_key,
});
export const connectMeili = async () => {
const logger = getLogger("meilisearch");
if (!config.meilisearch.enabled) {
return;
}
if (await meilisearch.isHealthy()) {
await meilisearch
.index(MeiliIndexType.Accounts)
.updateSortableAttributes(["createdAt"]);
await meilisearch
.index(MeiliIndexType.Accounts)
.updateSearchableAttributes(["username", "displayName", "note"]);
await meilisearch
.index(MeiliIndexType.Statuses)
.updateSortableAttributes(["createdAt"]);
await meilisearch
.index(MeiliIndexType.Statuses)
.updateSearchableAttributes(["content"]);
logger.info`Connected to Meilisearch`;
} else {
logger.fatal`Error while connecting to Meilisearch`;
// Hang until Ctrl+C is pressed
await Bun.sleep(Number.POSITIVE_INFINITY);
}
};
export enum MeiliIndexType {
Accounts = "accounts",
Statuses = "statuses",
}
export const addUserToMeilisearch = async (user: User) => {
if (!config.meilisearch.enabled) {
return;
}
await meilisearch.index(MeiliIndexType.Accounts).addDocuments([
{
id: user.id,
username: user.data.username,
displayName: user.data.displayName,
note: user.data.note,
createdAt: user.data.createdAt,
},
]);
};
export const getNthDatabaseAccountBatch = (
n: number,
batchSize = 1000,
): Promise<Record<string, string | Date>[]> => {
return db.query.Users.findMany({
offset: n * batchSize,
limit: batchSize,
columns: {
id: true,
username: true,
displayName: true,
note: true,
createdAt: true,
},
orderBy: (user, { asc }) => asc(user.createdAt),
});
};
export const getNthDatabaseStatusBatch = (
n: number,
batchSize = 1000,
): Promise<Record<string, string | Date>[]> => {
return db.query.Notes.findMany({
offset: n * batchSize,
limit: batchSize,
columns: {
id: true,
content: true,
createdAt: true,
},
orderBy: (status, { asc }) => asc(status.createdAt),
});
};
export const rebuildSearchIndexes = async (
indexes: MeiliIndexType[],
batchSize = 100,
) => {
if (indexes.includes(MeiliIndexType.Accounts)) {
const accountCount = (
await db
.select({
count: count(),
})
.from(Users)
)[0].count;
for (let i = 0; i < accountCount / batchSize; i++) {
const accounts = await getNthDatabaseAccountBatch(i, batchSize);
/* const _progress = Math.round(
(i / (accountCount / batchSize)) * 100,
);
*/
// Sync with Meilisearch
await meilisearch
.index(MeiliIndexType.Accounts)
.addDocuments(accounts);
}
/* const _meiliAccountCount = (
await meilisearch.index(MeiliIndexType.Accounts).getStats()
).numberOfDocuments; */
}
if (indexes.includes(MeiliIndexType.Statuses)) {
const statusCount = (
await db
.select({
count: count(),
})
.from(Notes)
)[0].count;
for (let i = 0; i < statusCount / batchSize; i++) {
const statuses = await getNthDatabaseStatusBatch(i, batchSize);
/* const _progress = Math.round((i / (statusCount / batchSize)) * 100); */
// Sync with Meilisearch
await meilisearch
.index(MeiliIndexType.Statuses)
.addDocuments(statuses);
}
/* const _meiliStatusCount = (
await meilisearch.index(MeiliIndexType.Statuses).getStats()
).numberOfDocuments; */
}
};