diff --git a/packages/backend/src/db/meilisearch.ts b/packages/backend/src/db/meilisearch.ts index 40efaa5864..ab21b4677f 100644 --- a/packages/backend/src/db/meilisearch.ts +++ b/packages/backend/src/db/meilisearch.ts @@ -6,6 +6,8 @@ import { Note } from "@/models/entities/note.js"; import * as url from "url"; import { ILocalUser } from "@/models/entities/user.js"; import { Followings, Users } from "@/models/index.js"; +import { userByIdCache } from "@/services/user-cache.js"; +import { ScyllaNote } from "./scylla.js"; const logger = dbLogger.createSubLogger("meilisearch", "gray", false); @@ -347,20 +349,14 @@ export default hasConfig sort: sortRules, }); }, - ingestNote: async (ingestNotes: Note | Note[]) => { - if (ingestNotes instanceof Note) { - ingestNotes = [ingestNotes]; - } - + ingestNote: async (ingestNotes: Note[] | ScyllaNote[]) => { const indexingBatch: MeilisearchNote[] = []; for (const note of ingestNotes) { - if (note.user === undefined) { - note.user = await Users.findOne({ - where: { - id: note.userId, - }, - }); + if (!note.user) { + note.user = await userByIdCache.fetch(note.userId, () => + Users.findOneByOrFail({ id: note.userId }), + ); } let attachmentType = ""; @@ -408,40 +404,15 @@ export default hasConfig return { health: health.status, size: stats.databaseSize, - indexed_count: stats.indexes["posts"].numberOfDocuments, + indexed_count: stats.indexes.posts.numberOfDocuments, }; }, - deleteNotes: async (note: Note | Note[] | string | string[]) => { - if (note instanceof Note) { - note = [note]; - } - if (typeof note === "string") { - note = [note]; - } + deleteNotes: async (notes: Note[] | ScyllaNote[] | string[]) => { + const deletionBatch = notes + .map((n) => (typeof n === "string" ? n : n.id)) + .filter((id) => !!id); - const deletionBatch = note - .map((n) => { - if (n instanceof Note) { - return n.id; - } - - if (n.length > 0) return n; - - logger.error( - `Failed to delete note from Meilisearch, invalid post ID: ${JSON.stringify( - n, - )}`, - ); - - throw new Error( - `Invalid note ID passed to meilisearch deleteNote: ${JSON.stringify( - n, - )}`, - ); - }) - .filter((el) => el !== null); - - await posts.deleteDocuments(deletionBatch as string[]).then(() => { + await posts.deleteDocuments(deletionBatch).then(() => { logger.info( `submitted ${deletionBatch.length} large batch for deletion`, ); diff --git a/packages/backend/src/models/repositories/note.ts b/packages/backend/src/models/repositories/note.ts index 4b0ad88d6e..7744128507 100644 --- a/packages/backend/src/models/repositories/note.ts +++ b/packages/backend/src/models/repositories/note.ts @@ -414,12 +414,11 @@ export const NoteRepository = db.getRepository(Note).extend({ const targets = [...notes.map((n) => n.id), ...renoteIds]; let myReactions: NoteReaction[] = []; if (scyllaClient) { - const result = await scyllaClient.execute( - prepared.reaction.select.byNoteAndUser, - [targets, [meId]], - { prepare: true }, - ); - myReactions = result.rows.map(parseScyllaReaction); + myReactions = await scyllaClient + .execute(prepared.reaction.select.byNoteAndUser, [targets, [meId]], { + prepare: true, + }) + .then((result) => result.rows.map(parseScyllaReaction)); } else { myReactions = await NoteReactions.findBy({ userId: meId, diff --git a/packages/backend/src/server/api/endpoints/notes/search.ts b/packages/backend/src/server/api/endpoints/notes/search.ts index 8c5798b62d..5482bb4d8e 100644 --- a/packages/backend/src/server/api/endpoints/notes/search.ts +++ b/packages/backend/src/server/api/endpoints/notes/search.ts @@ -11,6 +11,7 @@ import { generateVisibilityQuery } from "../../common/generate-visibility-query. import { generateMutedUserQuery } from "../../common/generate-muted-user-query.js"; import { generateBlockedUserQuery } from "../../common/generate-block-query.js"; import { sqlLikeEscape } from "@/misc/sql-like-escape.js"; +import { parseScyllaNote, prepared, scyllaClient } from "@/db/scylla.js"; export const meta = { tags: ["notes"], @@ -68,7 +69,7 @@ export const paramDef = { } as const; export default define(meta, paramDef, async (ps, me) => { - if (es == null && sonic == null && meilisearch == null) { + if (!es && !sonic && !meilisearch && !scyllaClient) { const query = makePaginationQuery( Notes.createQueryBuilder("note"), ps.sinceId, @@ -86,17 +87,8 @@ export default define(meta, paramDef, async (ps, me) => { query .andWhere("note.text ILIKE :q", { q: `%${sqlLikeEscape(ps.query)}%` }) .andWhere("note.visibility = 'public'") - .innerJoinAndSelect("note.user", "user") - .leftJoinAndSelect("user.avatar", "avatar") - .leftJoinAndSelect("user.banner", "banner") .leftJoinAndSelect("note.reply", "reply") - .leftJoinAndSelect("note.renote", "renote") - .leftJoinAndSelect("reply.user", "replyUser") - .leftJoinAndSelect("replyUser.avatar", "replyUserAvatar") - .leftJoinAndSelect("replyUser.banner", "replyUserBanner") - .leftJoinAndSelect("renote.user", "renoteUser") - .leftJoinAndSelect("renoteUser.avatar", "renoteUserAvatar") - .leftJoinAndSelect("renoteUser.banner", "renoteUserBanner"); + .leftJoinAndSelect("note.renote", "renote"); generateVisibilityQuery(query, me); if (me) generateMutedUserQuery(query, me); @@ -110,7 +102,7 @@ export default define(meta, paramDef, async (ps, me) => { const chunkSize = 100; // Use sonic to fetch and step through all search results that could match the requirements - const ids = []; + const ids: string[] = []; while (true) { const results = await sonic.search.query( sonic.collection, @@ -151,18 +143,25 @@ export default define(meta, paramDef, async (ps, me) => { } // Sort all the results by note id DESC (newest first) - ids.sort((a, b) => b - a); + ids.sort().reverse(); // Fetch the notes from the database until we have enough to satisfy the limit start = 0; const found = []; while (found.length < ps.limit && start < ids.length) { const chunk = ids.slice(start, start + chunkSize); - const notes: Note[] = await Notes.find({ - where: { - id: In(chunk), - }, - }); + let notes: Note[] = []; + if (scyllaClient) { + notes = await scyllaClient + .execute(prepared.note.select.byIds, [chunk], { prepare: true }) + .then((result) => result.rows.map(parseScyllaNote)); + } else { + notes = await Notes.find({ + where: { + id: In(chunk), + }, + }); + } // The notes are checked for visibility and muted/blocked users when packed found.push(...(await Notes.packMany(notes, me))); @@ -239,13 +238,14 @@ export default define(meta, paramDef, async (ps, me) => { while (found.length < ps.limit && start < noteIDs.length) { const chunk = noteIDs.slice(start, start + chunkSize); - let query: FindManyOptions = { - where: { - id: In(chunk), - }, - }; - - const notes: Note[] = await Notes.find(query); + let notes: Note[] = []; + if (scyllaClient) { + notes = await scyllaClient + .execute(prepared.note.select.byIds, [chunk], { prepare: true }) + .then((result) => result.rows.map(parseScyllaNote)); + } else { + notes = await Notes.findBy({ id: In(chunk) }); + } // Re-order the note result according to the noteIDs array (cannot be undefined, we map this earlier) // @ts-ignore @@ -262,7 +262,7 @@ export default define(meta, paramDef, async (ps, me) => { } return found; - } else { + } else if (es && !scyllaClient) { const userQuery = ps.userId != null ? [ @@ -329,7 +329,7 @@ export default define(meta, paramDef, async (ps, me) => { const hits = result.body.hits.hits.map((hit: any) => hit._id); - if (hits.length === 0) return []; + if (hits.length === 0) return await Notes.packMany([]); // Fetch found notes const notes = await Notes.find({ @@ -343,4 +343,6 @@ export default define(meta, paramDef, async (ps, me) => { return await Notes.packMany(notes, me); } + + return await Notes.packMany([]); }); diff --git a/packages/backend/src/services/note/create.ts b/packages/backend/src/services/note/create.ts index c70722c8c3..ca9e6f62ad 100644 --- a/packages/backend/src/services/note/create.ts +++ b/packages/backend/src/services/note/create.ts @@ -1007,7 +1007,7 @@ export async function index(note: Note, reindexing: boolean): Promise { } if (meilisearch && !reindexing) { - await meilisearch.ingestNote(note); + await meilisearch.ingestNote([note]); } } diff --git a/packages/backend/src/services/note/delete.ts b/packages/backend/src/services/note/delete.ts index da2593d6d7..697e1c1699 100644 --- a/packages/backend/src/services/note/delete.ts +++ b/packages/backend/src/services/note/delete.ts @@ -326,7 +326,7 @@ export default async function ( } if (meilisearch) { - await meilisearch.deleteNotes(note.id); + await meilisearch.deleteNotes([note.id]); } }