Merge branch 'gh-fa55fa5e/10491/unknown/feat/relevance-search' into 'develop'

[PR]: Meilisearch relevancy search

See merge request firefish/firefish!10491

packages/backend/src/db/meilisearch.ts

@@ -121,6 +121,19 @@ if (hasConfig) {
 			),
 		);
 
+	posts
+		.updateRankingRules([
+			"sort",
+			"words",
+			"typo",
+			"proximity",
+			"attribute",
+			"exactness",
+		])
+		.catch((e) => {
+			logger.error("Failed to set ranking rules, sorting won't work properly.");
+		});
+
 	logger.info("Connected to MeiliSearch");
 }
 
@@ -160,6 +173,7 @@ export default hasConfig
 				limit: number,
 				offset: number,
 				userCtx: ILocalUser | null,
+				overrideSort: string | null,
 			) => {
 				/// Advanced search syntax
 				/// from:user => filter by user + optional domain
@@ -170,8 +184,10 @@ export default hasConfig
 				/// "text" => get posts with exact text between quotes
 				/// filter:following => show results only from users you follow
 				/// filter:followers => show results only from followers
+				/// order:desc/asc => order results ascending or descending
 
 				const constructedFilters: string[] = [];
+				let sortRules: string[] = [];
 
 				const splitSearch = query.split(" ");
 
@@ -278,6 +294,14 @@ export default hasConfig
 									);
 								}
 
+								return null;
+							} else if (term.startsWith("order:desc")) {
+								sortRules.push("createdAt:desc");
+
+								return null;
+							} else if (term.startsWith("order:asc")) {
+								sortRules.push("createdAt:asc");
+
 								return null;
 							}
 
@@ -286,14 +310,27 @@ export default hasConfig
 					)
 				).filter((term) => term !== null);
 
-				const sortRules = [];
-
 				// An empty search term with defined filters means we have a placeholder search => https://www.meilisearch.com/docs/reference/api/search#placeholder-search
 				// These have to be ordered manually, otherwise the *oldest* posts are returned first, which we don't want
-				if (filteredSearchTerms.length === 0 && constructedFilters.length > 0) {
+				// If the user has defined a sort rule, don't mess with it
+				if (
+					filteredSearchTerms.length === 0 &&
+					constructedFilters.length > 0 &&
+					sortRules.length === 0
+				) {
 					sortRules.push("createdAt:desc");
 				}
 
+				// More than one sorting rule doesn't make sense. We only keep the first one, otherwise weird stuff may happen.
+				if (sortRules.length > 1) {
+					sortRules = [sortRules[0]];
+				}
+
+				// An override sort takes precedence, user sorting is ignored here
+				if (overrideSort) {
+					sortRules = [overrideSort];
+				}
+
 				logger.info(`Searching for ${filteredSearchTerms.join(" ")}`);
 				logger.info(`Limit: ${limit}`);
 				logger.info(`Offset: ${offset}`);

packages/backend/src/server/api/endpoints/notes/search.ts

@@ -1,4 +1,4 @@
-import { In } from "typeorm";
+import { FindManyOptions, In } from "typeorm";
 import { Notes } from "@/models/index.js";
 import { Note } from "@/models/entities/note.js";
 import config from "@/config/index.js";
@@ -58,6 +58,11 @@ export const paramDef = {
 			nullable: true,
 			default: null,
 		},
+		order: {
+			type: "string",
+			default: "chronological",
+			nullable: true,
+		},
 	},
 	required: ["query"],
 } as const;
@@ -156,9 +161,6 @@ export default define(meta, paramDef, async (ps, me) => {
 				where: {
 					id: In(chunk),
 				},
-				order: {
-					id: "DESC",
-				},
 			});
 
 			// The notes are checked for visibility and muted/blocked users when packed
@@ -175,19 +177,31 @@ export default define(meta, paramDef, async (ps, me) => {
 	} else if (meilisearch) {
 		let start = 0;
 		const chunkSize = 100;
+		const sortByDate = ps.order !== "relevancy";
+
+		type NoteResult = {
+			id: string;
+			createdAt: number;
+		};
+		const extractedNotes: NoteResult[] = [];
 
-		// Use meilisearch to fetch and step through all search results that could match the requirements
-		const ids = [];
 		while (true) {
-			const results = await meilisearch.search(ps.query, chunkSize, start, me);
+			const searchRes = await meilisearch.search(
+				ps.query,
+				chunkSize,
+				start,
+				me,
+				sortByDate ? "createdAt:desc" : null,
+			);
+			const results: MeilisearchNote[] = searchRes.hits as MeilisearchNote[];
 
 			start += chunkSize;
 
-			if (results.hits.length === 0) {
+			if (results.length === 0) {
 				break;
 			}
 
-			const res = results.hits
+			const res = results
 				.filter((key: MeilisearchNote) => {
 					if (ps.userId && key.userId !== ps.userId) {
 						return false;
@@ -203,34 +217,45 @@ export default define(meta, paramDef, async (ps, me) => {
 					}
 					return true;
 				})
-				.map((key) => key.id);
+				.map((key) => {
+					return {
+						id: key.id,
+						createdAt: key.createdAt,
+					};
+				});
 
-			ids.push(...res);
+			extractedNotes.push(...res);
 		}
 
-		// Sort all the results by note id DESC (newest first)
-		ids.sort((a, b) => b - a);
-
 		// Fetch the notes from the database until we have enough to satisfy the limit
 		start = 0;
 		const found = [];
-		while (found.length < ps.limit && start < ids.length) {
-			const chunk = ids.slice(start, start + chunkSize);
-			const notes: Note[] = await Notes.find({
+		const noteIDs = extractedNotes.map((note) => note.id);
+
+		// Index the ID => index number into a map, so we can restore the array ordering efficiently later
+		const idIndexMap = new Map(noteIDs.map((id, index) => [id, index]));
+
+		while (found.length < ps.limit && start < noteIDs.length) {
+			const chunk = noteIDs.slice(start, start + chunkSize);
+
+			let query: FindManyOptions = {
 				where: {
 					id: In(chunk),
 				},
-				order: {
-					id: "DESC",
-				},
-			});
+			};
+
+			const notes: Note[] = await Notes.find(query);
+
+			// Re-order the note result according to the noteIDs array (cannot be undefined, we map this earlier)
+			// @ts-ignore
+			notes.sort((a, b) => idIndexMap.get(a.id) - idIndexMap.get(b.id));
 
 			// The notes are checked for visibility and muted/blocked users when packed
 			found.push(...(await Notes.packMany(notes, me)));
 			start += chunkSize;
 		}
 
-		// If we have more results than the limit, trim them
+		// If we have more results than the limit, trim the results down
 		if (found.length > ps.limit) {
 			found.length = ps.limit;
 		}