perf: read timelines from scylla

packages/backend/src/server/api/endpoints/notes/global-timeline.ts

@@ -1,5 +1,5 @@
 import { fetchMeta } from "@/misc/fetch-meta.js";
-import { Notes } from "@/models/index.js";
+import { Notes, UserProfiles } from "@/models/index.js";
 import { activeUsersChart } from "@/services/chart/index.js";
 import define from "../../define.js";
 import { ApiError } from "../../error.js";
@@ -9,6 +9,23 @@ import { generateRepliesQuery } from "../../common/generate-replies-query.js";
 import { generateMutedNoteQuery } from "../../common/generate-muted-note-query.js";
 import { generateBlockedUserQuery } from "../../common/generate-block-query.js";
 import { generateMutedUserRenotesQueryForNotes } from "../../common/generated-muted-renote-query.js";
+import {
+	ScyllaNote,
+	execTimelineQuery,
+	filterBlockedUser,
+	filterMutedNote,
+	filterMutedRenotes,
+	filterMutedUser,
+	filterReply,
+	scyllaClient,
+} from "@/db/scylla.js";
+import {
+	InstanceMutingsCache,
+	RenoteMutingsCache,
+	UserBlockedCache,
+	UserMutingsCache,
+	userWordMuteCache,
+} from "@/misc/cache.js";
 
 export const meta = {
 	tags: ["notes"],
@@ -70,6 +87,63 @@ export default define(meta, paramDef, async (ps, user) => {
 		}
 	}
 
+	process.nextTick(() => {
+		if (user) {
+			activeUsersChart.read(user);
+		}
+	});
+
+	if (scyllaClient) {
+		let [mutedUserIds, mutedInstances, blockerIds, renoteMutedIds]: string[][] =
+			[];
+		let mutedWords: string[][];
+		if (user) {
+			[mutedUserIds, mutedInstances, mutedWords, blockerIds, renoteMutedIds] =
+				await Promise.all([
+					UserMutingsCache.init(user.id).then((cache) => cache.getAll()),
+					InstanceMutingsCache.init(user.id).then((cache) => cache.getAll()),
+					userWordMuteCache
+						.fetchMaybe(user.id, () =>
+							UserProfiles.findOne({
+								select: ["mutedWords"],
+								where: { userId: user.id },
+							}).then((profile) => profile?.mutedWords),
+						)
+						.then((words) => words ?? []),
+					UserBlockedCache.init(user.id).then((cache) => cache.getAll()),
+					RenoteMutingsCache.init(user.id).then((cache) => cache.getAll()),
+				]);
+		}
+
+		const filter = async (notes: ScyllaNote[]) => {
+			let filtered = notes.filter(
+				(n) => n.visibility === "public" && !n.channelId,
+			);
+			filtered = await filterReply(filtered, ps.withReplies, user);
+			if (user) {
+				filtered = await filterMutedUser(
+					filtered,
+					user,
+					mutedUserIds,
+					mutedInstances,
+				);
+				filtered = await filterMutedNote(filtered, user, mutedWords);
+				filtered = await filterBlockedUser(filtered, user, blockerIds);
+				filtered = await filterMutedRenotes(filtered, user, renoteMutedIds);
+			}
+			if (ps.withFiles) {
+				filtered = filtered.filter((n) => n.files.length > 0);
+			}
+			filtered = filtered.filter((n) => n.visibility !== "hidden");
+			return filtered;
+		};
+
+		const foundNotes = await execTimelineQuery(ps, filter);
+		return await Notes.packMany(foundNotes.slice(0, ps.limit), user, {
+			scyllaNote: true,
+		});
+	}
+
 	//#region Construct query
 	const query = makePaginationQuery(
 		Notes.createQueryBuilder("note"),
@@ -95,12 +169,6 @@ export default define(meta, paramDef, async (ps, user) => {
 	query.andWhere("note.visibility != 'hidden'");
 	//#endregion
 
-	process.nextTick(() => {
-		if (user) {
-			activeUsersChart.read(user);
-		}
-	});
-
 	// We fetch more than requested because some may be filtered out, and if there's less than
 	// requested, the pagination stops.
 	const found = [];

packages/backend/src/server/api/endpoints/notes/hybrid-timeline.ts

@@ -1,6 +1,6 @@
 import { Brackets } from "typeorm";
 import { fetchMeta } from "@/misc/fetch-meta.js";
-import { Followings, Notes } from "@/models/index.js";
+import { Followings, Notes, UserProfiles } from "@/models/index.js";
 import { activeUsersChart } from "@/services/chart/index.js";
 import define from "../../define.js";
 import { ApiError } from "../../error.js";
@@ -12,6 +12,27 @@ import { generateMutedNoteQuery } from "../../common/generate-muted-note-query.j
 import { generateChannelQuery } from "../../common/generate-channel-query.js";
 import { generateBlockedUserQuery } from "../../common/generate-block-query.js";
 import { generateMutedUserRenotesQueryForNotes } from "../../common/generated-muted-renote-query.js";
+import {
+	ScyllaNote,
+	execTimelineQuery,
+	filterBlockedUser,
+	filterChannel,
+	filterMutedNote,
+	filterMutedRenotes,
+	filterMutedUser,
+	filterReply,
+	filterVisibility,
+	scyllaClient,
+} from "@/db/scylla.js";
+import {
+	ChannelFollowingsCache,
+	InstanceMutingsCache,
+	LocalFollowingsCache,
+	RenoteMutingsCache,
+	UserBlockedCache,
+	UserMutingsCache,
+	userWordMuteCache,
+} from "@/misc/cache.js";
 
 export const meta = {
 	tags: ["notes"],
@@ -75,6 +96,81 @@ export default define(meta, paramDef, async (ps, user) => {
 		throw new ApiError(meta.errors.stlDisabled);
 	}
 
+	process.nextTick(() => {
+		activeUsersChart.read(user);
+	});
+
+	if (scyllaClient) {
+		const [
+			followingChannelIds,
+			followingUserIds,
+			mutedUserIds,
+			mutedInstances,
+			mutedWords,
+			blockerIds,
+			renoteMutedIds,
+		] = await Promise.all([
+			ChannelFollowingsCache.init(user.id).then((cache) => cache.getAll()),
+			LocalFollowingsCache.init(user.id).then((cache) => cache.getAll()),
+			UserMutingsCache.init(user.id).then((cache) => cache.getAll()),
+			InstanceMutingsCache.init(user.id).then((cache) => cache.getAll()),
+			userWordMuteCache
+				.fetchMaybe(user.id, () =>
+					UserProfiles.findOne({
+						select: ["mutedWords"],
+						where: { userId: user.id },
+					}).then((profile) => profile?.mutedWords),
+				)
+				.then((words) => words ?? []),
+			UserBlockedCache.init(user.id).then((cache) => cache.getAll()),
+			RenoteMutingsCache.init(user.id).then((cache) => cache.getAll()),
+		]);
+		const validUserIds = [user.id].concat(followingUserIds);
+		const optFilter = (n: ScyllaNote) =>
+			!n.renoteId || !!n.text || n.files.length > 0 || n.hasPoll;
+
+		const filter = async (notes: ScyllaNote[]) => {
+			let filtered = notes.filter(
+				(n) =>
+					validUserIds.includes(n.userId) ||
+					(n.visibility === "public" && !n.userHost),
+			);
+			filtered = await filterChannel(filtered, user, followingChannelIds);
+			filtered = await filterReply(filtered, ps.withReplies, user);
+			filtered = await filterVisibility(filtered, user, followingUserIds);
+			filtered = await filterMutedUser(
+				filtered,
+				user,
+				mutedUserIds,
+				mutedInstances,
+			);
+			filtered = await filterMutedNote(filtered, user, mutedWords);
+			filtered = await filterBlockedUser(filtered, user, blockerIds);
+			filtered = await filterMutedRenotes(filtered, user, renoteMutedIds);
+			if (!ps.includeMyRenotes) {
+				filtered = filtered.filter((n) => n.userId !== user.id || optFilter(n));
+			}
+			if (!ps.includeRenotedMyNotes) {
+				filtered = filtered.filter(
+					(n) => n.renoteUserId !== user.id || optFilter(n),
+				);
+			}
+			if (!ps.includeLocalRenotes) {
+				filtered = filtered.filter((n) => n.renoteUserHost || optFilter(n));
+			}
+			if (ps.withFiles) {
+				filtered = filtered.filter((n) => n.files.length > 0);
+			}
+			filtered = filtered.filter((n) => n.visibility !== "hidden");
+			return filtered;
+		};
+
+		const foundNotes = await execTimelineQuery(ps, filter);
+		return await Notes.packMany(foundNotes.slice(0, ps.limit), user, {
+			scyllaNote: true,
+		});
+	}
+
 	//#region Construct query
 	const followingQuery = Followings.createQueryBuilder("following")
 		.select("following.followeeId")
@@ -154,10 +250,6 @@ export default define(meta, paramDef, async (ps, user) => {
 	query.andWhere("note.visibility != 'hidden'");
 	//#endregion
 
-	process.nextTick(() => {
-		activeUsersChart.read(user);
-	});
-
 	// We fetch more than requested because some may be filtered out, and if there's less than
 	// requested, the pagination stops.
 	const found = [];

packages/backend/src/server/api/endpoints/notes/recommended-timeline.ts

@@ -1,6 +1,6 @@
 import { Brackets } from "typeorm";
 import { fetchMeta } from "@/misc/fetch-meta.js";
-import { Notes } from "@/models/index.js";
+import { Notes, UserProfiles } from "@/models/index.js";
 import { activeUsersChart } from "@/services/chart/index.js";
 import define from "../../define.js";
 import { ApiError } from "../../error.js";
@@ -12,6 +12,25 @@ import { generateMutedNoteQuery } from "../../common/generate-muted-note-query.j
 import { generateChannelQuery } from "../../common/generate-channel-query.js";
 import { generateBlockedUserQuery } from "../../common/generate-block-query.js";
 import { generateMutedUserRenotesQueryForNotes } from "../../common/generated-muted-renote-query.js";
+import {
+	ScyllaNote,
+	execTimelineQuery,
+	filterBlockedUser,
+	filterMutedNote,
+	filterMutedRenotes,
+	filterMutedUser,
+	filterReply,
+	filterVisibility,
+	scyllaClient,
+} from "@/db/scylla.js";
+import {
+	InstanceMutingsCache,
+	LocalFollowingsCache,
+	RenoteMutingsCache,
+	UserBlockedCache,
+	UserMutingsCache,
+	userWordMuteCache,
+} from "@/misc/cache.js";
 
 export const meta = {
 	tags: ["notes"],
@@ -80,6 +99,90 @@ export default define(meta, paramDef, async (ps, user) => {
 		}
 	}
 
+	process.nextTick(() => {
+		if (user) {
+			activeUsersChart.read(user);
+		}
+	});
+
+	if (scyllaClient) {
+		let [
+			followingUserIds,
+			mutedUserIds,
+			mutedInstances,
+			blockerIds,
+			renoteMutedIds,
+		]: string[][] = [];
+		let mutedWords: string[][];
+		if (user) {
+			[
+				followingUserIds,
+				mutedUserIds,
+				mutedInstances,
+				mutedWords,
+				blockerIds,
+				renoteMutedIds,
+			] = await Promise.all([
+				LocalFollowingsCache.init(user.id).then((cache) => cache.getAll()),
+				UserMutingsCache.init(user.id).then((cache) => cache.getAll()),
+				InstanceMutingsCache.init(user.id).then((cache) => cache.getAll()),
+				userWordMuteCache
+					.fetchMaybe(user.id, () =>
+						UserProfiles.findOne({
+							select: ["mutedWords"],
+							where: { userId: user.id },
+						}).then((profile) => profile?.mutedWords),
+					)
+					.then((words) => words ?? []),
+				UserBlockedCache.init(user.id).then((cache) => cache.getAll()),
+				RenoteMutingsCache.init(user.id).then((cache) => cache.getAll()),
+			]);
+		}
+
+		const filter = async (notes: ScyllaNote[]) => {
+			let filtered = notes.filter(
+				(n) =>
+					n.visibility === "public" &&
+					n.userHost &&
+					m.recommendedInstances.includes(n.userHost) &&
+					!n.channelId,
+			);
+			filtered = await filterReply(filtered, ps.withReplies, user);
+			filtered = await filterVisibility(filtered, user, followingUserIds);
+			if (user) {
+				filtered = await filterMutedUser(
+					filtered,
+					user,
+					mutedUserIds,
+					mutedInstances,
+				);
+				filtered = await filterMutedNote(filtered, user, mutedWords);
+				filtered = await filterBlockedUser(filtered, user, blockerIds);
+				filtered = await filterMutedRenotes(filtered, user, renoteMutedIds);
+			}
+			if (ps.withFiles) {
+				filtered = filtered.filter((n) => n.files.length > 0);
+			}
+			if (ps.fileType) {
+				filtered = filtered.filter((n) =>
+					n.files.some((f) => ps.fileType?.includes(f.type)),
+				);
+			}
+			if (ps.excludeNsfw) {
+				filtered = filtered.filter(
+					(n) => !n.cw && n.files.every((f) => !f.isSensitive),
+				);
+			}
+			filtered = filtered.filter((n) => n.visibility !== "hidden");
+			return filtered;
+		};
+
+		const foundNotes = await execTimelineQuery(ps, filter);
+		return await Notes.packMany(foundNotes.slice(0, ps.limit), user, {
+			scyllaNote: true,
+		});
+	}
+
 	//#region Construct query
 	const query = makePaginationQuery(
 		Notes.createQueryBuilder("note"),
@@ -91,12 +194,7 @@ export default define(meta, paramDef, async (ps, user) => {
 		.andWhere(
 			`(note.userHost = ANY ('{"${m.recommendedInstances.join('","')}"}'))`,
 		)
-		.andWhere("(note.visibility = 'public')")
+		.andWhere("(note.visibility = 'public')");
-		.innerJoinAndSelect("note.user", "user")
-		.leftJoinAndSelect("note.reply", "reply")
-		.leftJoinAndSelect("note.renote", "renote")
-		.leftJoinAndSelect("reply.user", "replyUser")
-		.leftJoinAndSelect("renote.user", "renoteUser");
 
 	generateChannelQuery(query, user);
 	generateRepliesQuery(query, ps.withReplies, user);
@@ -133,12 +231,6 @@ export default define(meta, paramDef, async (ps, user) => {
 	query.andWhere("note.visibility != 'hidden'");
 	//#endregion
 
-	process.nextTick(() => {
-		if (user) {
-			activeUsersChart.read(user);
-		}
-	});
-
 	// We fetch more than requested because some may be filtered out, and if there's less than
 	// requested, the pagination stops.
 	const found = [];